From 8f7d2e38f7e685bdbc9f2d0b8246eea32b24c8d5 Mon Sep 17 00:00:00 2001 From: topjohnwu Date: Mon, 26 Feb 2024 17:34:17 -0800 Subject: [PATCH] Make crt0 an external submodule --- .gitmodules | 3 + native/src/Android.mk | 3 - native/src/Application.mk | 1 - native/src/crt0/Android.mk | 24 - native/src/crt0/bionic/dirent.cpp | 168 - .../src/crt0/bionic/private/ErrnoRestorer.h | 36 - native/src/crt0/bionic/private/bionic_asm.h | 115 - .../src/crt0/bionic/private/bionic_asm_arm.h | 48 - .../crt0/bionic/private/bionic_asm_arm64.h | 79 - .../crt0/bionic/private/bionic_asm_riscv64.h | 43 - .../src/crt0/bionic/private/bionic_asm_x86.h | 51 - .../crt0/bionic/private/bionic_asm_x86_64.h | 42 - .../src/crt0/bionic/private/bionic_errdefs.h | 171 - native/src/crt0/bionic/strerror.cpp | 100 - native/src/crt0/bionic/syscall-arm.S | 51 - native/src/crt0/bionic/syscall-arm64.S | 51 - native/src/crt0/bionic/syscall-x86.S | 66 - native/src/crt0/bionic/syscall-x86_64.S | 63 - native/src/crt0/dlmalloc/malloc.c | 6291 ----------------- native/src/crt0/dlmalloc/malloc.h | 637 -- native/src/crt0/linux_syscall_support.h | 5375 -------------- native/src/crt0/malloc.c | 16 - native/src/crt0/mem.c | 361 - native/src/crt0/misc.c | 201 - native/src/crt0/nolibc.c | 15 - native/src/crt0/nolibc/arch-aarch64.h | 159 - native/src/crt0/nolibc/arch-arm.h | 200 - native/src/crt0/nolibc/arch-i386.h | 179 - native/src/crt0/nolibc/arch-x86_64.h | 177 - native/src/crt0/nolibc/arch.h | 28 - native/src/crt0/nolibc/crt.h | 80 - native/src/crt0/stdio.c | 222 - native/src/crt0/syscall.c | 221 - native/src/crt0/tinystdio/tinystdio.c | 834 --- native/src/crt0/tinystdio/tinystdio.h | 204 - native/src/external/Android.mk | 4 + native/src/external/crt0 | 1 + 37 files changed, 8 insertions(+), 16312 deletions(-) delete mode 100644 native/src/crt0/Android.mk delete mode 100644 native/src/crt0/bionic/dirent.cpp delete mode 100644 native/src/crt0/bionic/private/ErrnoRestorer.h delete mode 100644 native/src/crt0/bionic/private/bionic_asm.h delete mode 100644 native/src/crt0/bionic/private/bionic_asm_arm.h delete mode 100644 native/src/crt0/bionic/private/bionic_asm_arm64.h delete mode 100644 native/src/crt0/bionic/private/bionic_asm_riscv64.h delete mode 100644 native/src/crt0/bionic/private/bionic_asm_x86.h delete mode 100644 native/src/crt0/bionic/private/bionic_asm_x86_64.h delete mode 100644 native/src/crt0/bionic/private/bionic_errdefs.h delete mode 100644 native/src/crt0/bionic/strerror.cpp delete mode 100644 native/src/crt0/bionic/syscall-arm.S delete mode 100644 native/src/crt0/bionic/syscall-arm64.S delete mode 100644 native/src/crt0/bionic/syscall-x86.S delete mode 100644 native/src/crt0/bionic/syscall-x86_64.S delete mode 100644 native/src/crt0/dlmalloc/malloc.c delete mode 100644 native/src/crt0/dlmalloc/malloc.h delete mode 100644 native/src/crt0/linux_syscall_support.h delete mode 100644 native/src/crt0/malloc.c delete mode 100644 native/src/crt0/mem.c delete mode 100644 native/src/crt0/misc.c delete mode 100644 native/src/crt0/nolibc.c delete mode 100644 native/src/crt0/nolibc/arch-aarch64.h delete mode 100644 native/src/crt0/nolibc/arch-arm.h delete mode 100644 native/src/crt0/nolibc/arch-i386.h delete mode 100644 native/src/crt0/nolibc/arch-x86_64.h delete mode 100644 native/src/crt0/nolibc/arch.h delete mode 100644 native/src/crt0/nolibc/crt.h delete mode 100644 native/src/crt0/stdio.c delete mode 100644 native/src/crt0/syscall.c delete mode 100644 native/src/crt0/tinystdio/tinystdio.c delete mode 100644 native/src/crt0/tinystdio/tinystdio.h create mode 160000 native/src/external/crt0 diff --git a/.gitmodules b/.gitmodules index 4473536d4..07558620e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -34,6 +34,9 @@ [submodule "system_properties"] path = native/src/external/system_properties url = https://github.com/topjohnwu/system_properties.git +[submodule "crt0"] + path = native/src/external/crt0 + url = https://github.com/topjohnwu/crt0.git [submodule "termux-elf-cleaner"] path = tools/termux-elf-cleaner url = https://github.com/termux/termux-elf-cleaner.git diff --git a/native/src/Android.mk b/native/src/Android.mk index 20b9c3084..889794e73 100644 --- a/native/src/Android.mk +++ b/native/src/Android.mk @@ -79,8 +79,6 @@ LOCAL_SRC_FILES := \ LOCAL_LDFLAGS := -static -T src/lto_fix.lds ifdef B_CRT0 -# -lc -lm is hardcoded in this variable, disable it -TARGET_LDLIBS := LOCAL_STATIC_LIBRARIES += crt0 LOCAL_LDFLAGS := endif @@ -172,7 +170,6 @@ include $(BUILD_STATIC_LIBRARY) include src/Android-rs.mk include src/base/Android.mk include src/external/Android.mk -include src/crt0/Android.mk ifdef B_BB diff --git a/native/src/Application.mk b/native/src/Application.mk index d0123ed47..69cf63bc3 100644 --- a/native/src/Application.mk +++ b/native/src/Application.mk @@ -12,7 +12,6 @@ ifdef B_CRT0 # Disable all security and debugging features APP_CFLAGS += -fno-unwind-tables -fno-asynchronous-unwind-tables -fno-stack-protector -U_FORTIFY_SOURCE -APP_LDFLAGS += -Wl,-wrap,abort_message # Override output folder to make sure all dependencies are rebuilt with new CFLAGS NDK_APP_OUT := ./obj/nolibc diff --git a/native/src/crt0/Android.mk b/native/src/crt0/Android.mk deleted file mode 100644 index 5f13c046a..000000000 --- a/native/src/crt0/Android.mk +++ /dev/null @@ -1,24 +0,0 @@ -LOCAL_PATH := $(call my-dir) - -include $(CLEAR_VARS) -LOCAL_MODULE := crt0 - -# Manually link the compiler runtime library -LOCAL_compiler_rt := $(shell $(TARGET_CC) -target $(LLVM_TRIPLE)$(TARGET_PLATFORM_LEVEL) --print-libgcc-file-name) - -LOCAL_EXPORT_LDFLAGS := -static -nostartfiles -nodefaultlibs $(LOCAL_compiler_rt) -Wl,--error-limit=0 -LOCAL_CFLAGS := -Wno-c99-designator - -LOCAL_SRC_FILES := \ - malloc.c \ - mem.c \ - misc.c \ - nolibc.c \ - stdio.c \ - syscall.c \ - bionic/dirent.cpp \ - bionic/strerror.cpp \ - bionic/syscall-$(TARGET_ARCH).S \ - tinystdio/tinystdio.c - -include $(BUILD_STATIC_LIBRARY) diff --git a/native/src/crt0/bionic/dirent.cpp b/native/src/crt0/bionic/dirent.cpp deleted file mode 100644 index 6d4c1bf21..000000000 --- a/native/src/crt0/bionic/dirent.cpp +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright (C) 2008 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include - -#include "private/ErrnoRestorer.h" - -extern "C" int sys_getdents64(unsigned int, dirent*, unsigned int); - -// Apportable decided to copy the data structure from this file -// and use it in their own code, but they also call into readdir. -// In order to avoid a lockup, the structure must be maintained in -// the exact same order as in L and below. New structure members -// need to be added to the end of this structure. -// See b/21037208 for more details. -struct DIR { - int fd_; - size_t available_bytes_; - dirent* next_; - dirent buff_[15]; - long current_pos_; -}; - -static DIR* __allocate_DIR(int fd) { - DIR* d = reinterpret_cast(malloc(sizeof(DIR))); - if (d == nullptr) { - return nullptr; - } - d->fd_ = fd; - d->available_bytes_ = 0; - d->next_ = nullptr; - d->current_pos_ = 0L; - return d; -} - -int dirfd(DIR* d) { - return d->fd_; -} - -DIR* fdopendir(int fd) { - // Is 'fd' actually a directory? - struct stat sb; - if (fstat(fd, &sb) == -1) { - return nullptr; - } - if (!S_ISDIR(sb.st_mode)) { - errno = ENOTDIR; - return nullptr; - } - - return __allocate_DIR(fd); -} - -DIR* opendir(const char* path) { - int fd = open(path, O_CLOEXEC | O_DIRECTORY | O_RDONLY); - return (fd != -1) ? __allocate_DIR(fd) : nullptr; -} - -static bool __fill_DIR(DIR* d) { - int rc = TEMP_FAILURE_RETRY(sys_getdents64(d->fd_, d->buff_, sizeof(d->buff_))); - if (rc <= 0) { - return false; - } - d->available_bytes_ = rc; - d->next_ = d->buff_; - return true; -} - -static dirent* __readdir_locked(DIR* d) { - if (d->available_bytes_ == 0 && !__fill_DIR(d)) { - return nullptr; - } - - dirent* entry = d->next_; - d->next_ = reinterpret_cast(reinterpret_cast(entry) + entry->d_reclen); - d->available_bytes_ -= entry->d_reclen; - // The directory entry offset uses 0, 1, 2 instead of real file offset, - // so the value range of long type is enough. - d->current_pos_ = static_cast(entry->d_off); - return entry; -} - -dirent* readdir(DIR* d) { - return __readdir_locked(d); -} -__strong_alias(readdir64, readdir); - -int readdir_r(DIR* d, dirent* entry, dirent** result) { - ErrnoRestorer errno_restorer; - - *result = nullptr; - errno = 0; - - dirent* next = __readdir_locked(d); - if (errno != 0 && next == nullptr) { - return errno; - } - - if (next != nullptr) { - memcpy(entry, next, next->d_reclen); - *result = entry; - } - return 0; -} -__strong_alias(readdir64_r, readdir_r); - -int closedir(DIR* d) { - if (d == nullptr) { - errno = EINVAL; - return -1; - } - - int fd = d->fd_; - int rc = close(fd); - free(d); - return rc; -} - -void rewinddir(DIR* d) { - lseek(d->fd_, 0, SEEK_SET); - d->available_bytes_ = 0; - d->current_pos_ = 0L; -} - -void seekdir(DIR* d, long offset) { - off_t ret = lseek(d->fd_, offset, SEEK_SET); - if (ret != -1L) { - d->available_bytes_ = 0; - d->current_pos_ = ret; - } -} - -long telldir(DIR* d) { - return d->current_pos_; -} diff --git a/native/src/crt0/bionic/private/ErrnoRestorer.h b/native/src/crt0/bionic/private/ErrnoRestorer.h deleted file mode 100644 index e698bb614..000000000 --- a/native/src/crt0/bionic/private/ErrnoRestorer.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once - -#include - -class ErrnoRestorer { - public: - explicit ErrnoRestorer() : saved_errno_(errno) { - } - - ~ErrnoRestorer() { - errno = saved_errno_; - } - - void override(int new_errno) { - saved_errno_ = new_errno; - } - - private: - int saved_errno_; -}; diff --git a/native/src/crt0/bionic/private/bionic_asm.h b/native/src/crt0/bionic/private/bionic_asm.h deleted file mode 100644 index 333f4fa3c..000000000 --- a/native/src/crt0/bionic/private/bionic_asm.h +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#pragma once - -#define __ASSEMBLY__ - -/* https://github.com/android/ndk/issues/1422 */ -#include - -#include /* For system call numbers. */ -#define MAX_ERRNO 4095 /* For recognizing system call error returns. */ - -#define __bionic_asm_custom_entry(f) -#define __bionic_asm_custom_end(f) -#define __bionic_asm_function_type @function -#define __bionic_asm_custom_note_gnu_section() - -#if defined(__aarch64__) -#include "bionic_asm_arm64.h" -#elif defined(__arm__) -#include "bionic_asm_arm.h" -#elif defined(__i386__) -#include "bionic_asm_x86.h" -#elif defined(__riscv) -#include "bionic_asm_riscv64.h" -#elif defined(__x86_64__) -#include "bionic_asm_x86_64.h" -#endif - -// Starts a normal assembler routine. -#define ENTRY(__f) __ENTRY_WITH_BINDING(__f, .globl) - -// Starts an assembler routine with hidden visibility. -#define ENTRY_PRIVATE(__f) \ - __ENTRY_WITH_BINDING(__f, .globl); \ - .hidden __f; - -// Starts an assembler routine that's weak so native bridges can override it. -#define ENTRY_WEAK_FOR_NATIVE_BRIDGE(__f) __ENTRY_WITH_BINDING(__f, .weak) - -// Starts an assembler routine with hidden visibility and no DWARF information. -// Only used for internal functions passed via sa_restorer. -// TODO: can't we just delete all those and let the kernel do its thing? -#define ENTRY_NO_DWARF_PRIVATE(__f) \ - __ENTRY_NO_DWARF(__f, .globl); \ - .hidden __f; - -// (Implementation detail.) -#define __ENTRY_NO_DWARF(__f, __binding) \ - .text; \ - __binding __f; \ - .balign __bionic_asm_align; \ - .type __f, __bionic_asm_function_type; \ - __f: \ - __bionic_asm_custom_entry(__f); - -// (Implementation detail.) -#define __ENTRY_WITH_BINDING(__f, __binding) \ - __ENTRY_NO_DWARF(__f, __binding); \ - .cfi_startproc; - -// Ends a normal assembler routine. -#define END(__f) \ - .cfi_endproc; \ - END_NO_DWARF(__f) - -// Ends an assembler routine with no DWARF information. -// Only used for internal functions passed via sa_restorer. -// TODO: can't we just delete all those and let the kernel do its thing? -#define END_NO_DWARF(__f) \ - .size __f, .- __f; \ - __bionic_asm_custom_end(__f) - -// Creates an alias `alias` for the symbol `original`. -#define ALIAS_SYMBOL(alias, original) \ - .globl alias; \ - .equ alias, original - -// Creates an alias `alias` for the symbol `original` that's weak so it can be -// separately overridden by native bridges. -#define ALIAS_SYMBOL_WEAK_FOR_NATIVE_BRIDGE(alias, original) \ - .weak alias; \ - .equ alias, original - -// Adds a GNU property ELF note. Important on arm64 to declare PAC/BTI support. -#define NOTE_GNU_PROPERTY() __bionic_asm_custom_note_gnu_section() - -// Gives local labels a more convenient and readable syntax. -#define L(__label) .L##__label diff --git a/native/src/crt0/bionic/private/bionic_asm_arm.h b/native/src/crt0/bionic/private/bionic_asm_arm.h deleted file mode 100644 index d8381d32b..000000000 --- a/native/src/crt0/bionic/private/bionic_asm_arm.h +++ /dev/null @@ -1,48 +0,0 @@ -/* $OpenBSD: asm.h,v 1.1 2004/02/01 05:09:49 drahn Exp $ */ -/* $NetBSD: asm.h,v 1.4 2001/07/16 05:43:32 matt Exp $ */ - -/* - * Copyright (c) 1990 The Regents of the University of California. - * All rights reserved. - * - * This code is derived from software contributed to Berkeley by - * William Jolitz. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * from: @(#)asm.h 5.5 (Berkeley) 5/7/91 - */ - -#pragma once - -#define __bionic_asm_align 0 - -#undef __bionic_asm_custom_entry -#undef __bionic_asm_custom_end -#define __bionic_asm_custom_entry(f) .fnstart -#define __bionic_asm_custom_end(f) .fnend - -#undef __bionic_asm_function_type -#define __bionic_asm_function_type #function diff --git a/native/src/crt0/bionic/private/bionic_asm_arm64.h b/native/src/crt0/bionic/private/bionic_asm_arm64.h deleted file mode 100644 index ffc718144..000000000 --- a/native/src/crt0/bionic/private/bionic_asm_arm64.h +++ /dev/null @@ -1,79 +0,0 @@ -/* $OpenBSD: asm.h,v 1.1 2004/02/01 05:09:49 drahn Exp $ */ -/* $NetBSD: asm.h,v 1.4 2001/07/16 05:43:32 matt Exp $ */ - -/* - * Copyright (c) 1990 The Regents of the University of California. - * All rights reserved. - * - * This code is derived from software contributed to Berkeley by - * William Jolitz. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * from: @(#)asm.h 5.5 (Berkeley) 5/7/91 - */ - -#pragma once - -#define __bionic_asm_align 16 - -#undef __bionic_asm_function_type -#define __bionic_asm_function_type %function - -#if defined(__ARM_FEATURE_BTI_DEFAULT) -#define __bionic_asm_aarch64_feature_bti (1 << 0) -#undef __bionic_asm_custom_entry -#define __bionic_asm_custom_entry(f) bti c -#else -#define __bionic_asm_aarch64_feature_bti 0 -#endif - -#if defined(__ARM_FEATURE_PAC_DEFAULT) -#define __bionic_asm_aarch64_feature_pac (1 << 1) -#else -#define __bionic_asm_aarch64_feature_pac 0 -#endif - -#undef __bionic_asm_custom_note_gnu_section -#define __bionic_asm_custom_note_gnu_section() \ - .pushsection .note.gnu.property, "a"; \ - .balign 8; \ - .long 4; \ - .long 0x10; \ - .long 0x5; /* NT_GNU_PROPERTY_TYPE_0 */ \ - .asciz "GNU"; \ - .long 0xc0000000; /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ \ - .long 4; \ - .long (__bionic_asm_aarch64_feature_pac | \ - __bionic_asm_aarch64_feature_bti); \ - .long 0; \ - .popsection; - -#define NT_MEMTAG_LEVEL_MASK 3 -#define NT_MEMTAG_LEVEL_NONE 0 -#define NT_MEMTAG_LEVEL_ASYNC 1 -#define NT_MEMTAG_LEVEL_SYNC 2 -#define NT_MEMTAG_HEAP 4 -#define NT_MEMTAG_STACK 8 diff --git a/native/src/crt0/bionic/private/bionic_asm_riscv64.h b/native/src/crt0/bionic/private/bionic_asm_riscv64.h deleted file mode 100644 index 463ca312a..000000000 --- a/native/src/crt0/bionic/private/bionic_asm_riscv64.h +++ /dev/null @@ -1,43 +0,0 @@ -/* $OpenBSD: asm.h,v 1.1 2004/02/01 05:09:49 drahn Exp $ */ -/* $NetBSD: asm.h,v 1.4 2001/07/16 05:43:32 matt Exp $ */ - -/* - * Copyright (c) 1990 The Regents of the University of California. - * All rights reserved. - * - * This code is derived from software contributed to Berkeley by - * William Jolitz. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * from: @(#)asm.h 5.5 (Berkeley) 5/7/91 - */ - -#pragma once - -#define __bionic_asm_align 16 - -#undef __bionic_asm_function_type -#define __bionic_asm_function_type %function diff --git a/native/src/crt0/bionic/private/bionic_asm_x86.h b/native/src/crt0/bionic/private/bionic_asm_x86.h deleted file mode 100644 index fcec40b5e..000000000 --- a/native/src/crt0/bionic/private/bionic_asm_x86.h +++ /dev/null @@ -1,51 +0,0 @@ -/* $NetBSD: asm.h,v 1.40 2011/06/16 13:16:20 joerg Exp $ */ - -/*- - * Copyright (c) 1990 The Regents of the University of California. - * All rights reserved. - * - * This code is derived from software contributed to Berkeley by - * William Jolitz. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * @(#)asm.h 5.5 (Berkeley) 5/7/91 - */ - -#pragma once - -#define PIC_PROLOGUE \ - pushl %ebx; \ - call 666f; \ -666: \ - popl %ebx; \ - addl $_GLOBAL_OFFSET_TABLE_+[.-666b], %ebx -#define PIC_EPILOGUE \ - popl %ebx -#define PIC_PLT(x) x@PLT -#define PIC_GOT(x) x@GOT(%ebx) -#define PIC_GOTOFF(x) x@GOTOFF(%ebx) - -#define __bionic_asm_align 16 diff --git a/native/src/crt0/bionic/private/bionic_asm_x86_64.h b/native/src/crt0/bionic/private/bionic_asm_x86_64.h deleted file mode 100644 index c553b0c9c..000000000 --- a/native/src/crt0/bionic/private/bionic_asm_x86_64.h +++ /dev/null @@ -1,42 +0,0 @@ -/* $NetBSD: asm.h,v 1.18 2013/09/12 15:36:17 joerg Exp $ */ - -/*- - * Copyright (c) 1990 The Regents of the University of California. - * All rights reserved. - * - * This code is derived from software contributed to Berkeley by - * William Jolitz. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. Neither the name of the University nor the names of its contributors - * may be used to endorse or promote products derived from this software - * without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * @(#)asm.h 5.5 (Berkeley) 5/7/91 - */ - -#pragma once - -#define PIC_PLT(x) x@PLT -#define PIC_GOT(x) x@GOTPCREL(%rip) - -#define __bionic_asm_align 16 diff --git a/native/src/crt0/bionic/private/bionic_errdefs.h b/native/src/crt0/bionic/private/bionic_errdefs.h deleted file mode 100644 index 435d49bc3..000000000 --- a/native/src/crt0/bionic/private/bionic_errdefs.h +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (C) 2023 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * This header is used to define error constants and names; - * it might be included several times. - */ - -#ifndef __BIONIC_ERRDEF -#error __BIONIC_ERRDEF not defined -#endif - -__BIONIC_ERRDEF(0, "Success") -__BIONIC_ERRDEF(EPERM, "Operation not permitted") -__BIONIC_ERRDEF(ENOENT, "No such file or directory") -__BIONIC_ERRDEF(ESRCH, "No such process") -__BIONIC_ERRDEF(EINTR, "Interrupted system call") -__BIONIC_ERRDEF(EIO, "I/O error") -__BIONIC_ERRDEF(ENXIO, "No such device or address") -__BIONIC_ERRDEF(E2BIG, "Argument list too long") -__BIONIC_ERRDEF(ENOEXEC, "Exec format error") -__BIONIC_ERRDEF(EBADF, "Bad file descriptor") -__BIONIC_ERRDEF(ECHILD, "No child processes") -__BIONIC_ERRDEF(EAGAIN, "Try again") -__BIONIC_ERRDEF(ENOMEM, "Out of memory") -__BIONIC_ERRDEF(EACCES, "Permission denied") -__BIONIC_ERRDEF(EFAULT, "Bad address") -__BIONIC_ERRDEF(ENOTBLK, "Block device required") -__BIONIC_ERRDEF(EBUSY, "Device or resource busy") -__BIONIC_ERRDEF(EEXIST, "File exists") -__BIONIC_ERRDEF(EXDEV, "Cross-device link") -__BIONIC_ERRDEF(ENODEV, "No such device") -__BIONIC_ERRDEF(ENOTDIR, "Not a directory") -__BIONIC_ERRDEF(EISDIR, "Is a directory") -__BIONIC_ERRDEF(EINVAL, "Invalid argument") -__BIONIC_ERRDEF(ENFILE, "File table overflow") -__BIONIC_ERRDEF(EMFILE, "Too many open files") -__BIONIC_ERRDEF(ENOTTY, "Inappropriate ioctl for device") -__BIONIC_ERRDEF(ETXTBSY, "Text file busy") -__BIONIC_ERRDEF(EFBIG, "File too large") -__BIONIC_ERRDEF(ENOSPC, "No space left on device") -__BIONIC_ERRDEF(ESPIPE, "Illegal seek") -__BIONIC_ERRDEF(EROFS, "Read-only file system") -__BIONIC_ERRDEF(EMLINK, "Too many links") -__BIONIC_ERRDEF(EPIPE, "Broken pipe") -__BIONIC_ERRDEF(EDOM, "Math argument out of domain of func") -__BIONIC_ERRDEF(ERANGE, "Math result not representable") -__BIONIC_ERRDEF(EDEADLK, "Resource deadlock would occur") -__BIONIC_ERRDEF(ENAMETOOLONG, "File name too long") -__BIONIC_ERRDEF(ENOLCK, "No record locks available") -__BIONIC_ERRDEF(ENOSYS, "Function not implemented") -__BIONIC_ERRDEF(ENOTEMPTY, "Directory not empty") -__BIONIC_ERRDEF(ELOOP, "Too many symbolic links encountered") -__BIONIC_ERRDEF(ENOMSG, "No message of desired type") -__BIONIC_ERRDEF(EIDRM, "Identifier removed") -__BIONIC_ERRDEF(ECHRNG, "Channel number out of range") -__BIONIC_ERRDEF(EL2NSYNC, "Level 2 not synchronized") -__BIONIC_ERRDEF(EL3HLT, "Level 3 halted") -__BIONIC_ERRDEF(EL3RST, "Level 3 reset") -__BIONIC_ERRDEF(ELNRNG, "Link number out of range") -__BIONIC_ERRDEF(EUNATCH, "Protocol driver not attached") -__BIONIC_ERRDEF(ENOCSI, "No CSI structure available") -__BIONIC_ERRDEF(EL2HLT, "Level 2 halted") -__BIONIC_ERRDEF(EBADE, "Invalid exchange") -__BIONIC_ERRDEF(EBADR, "Invalid request descriptor") -__BIONIC_ERRDEF(EXFULL, "Exchange full") -__BIONIC_ERRDEF(ENOANO, "No anode") -__BIONIC_ERRDEF(EBADRQC, "Invalid request code") -__BIONIC_ERRDEF(EBADSLT, "Invalid slot") -__BIONIC_ERRDEF(EBFONT, "Bad font file format") -__BIONIC_ERRDEF(ENOSTR, "Device not a stream") -__BIONIC_ERRDEF(ENODATA, "No data available") -__BIONIC_ERRDEF(ETIME, "Timer expired") -__BIONIC_ERRDEF(ENOSR, "Out of streams resources") -__BIONIC_ERRDEF(ENONET, "Machine is not on the network") -__BIONIC_ERRDEF(ENOPKG, "Package not installed") -__BIONIC_ERRDEF(EREMOTE, "Object is remote") -__BIONIC_ERRDEF(ENOLINK, "Link has been severed") -__BIONIC_ERRDEF(EADV, "Advertise error") -__BIONIC_ERRDEF(ESRMNT, "Srmount error") -__BIONIC_ERRDEF(ECOMM, "Communication error on send") -__BIONIC_ERRDEF(EPROTO, "Protocol error") -__BIONIC_ERRDEF(EMULTIHOP, "Multihop attempted") -__BIONIC_ERRDEF(EDOTDOT, "RFS specific error") -__BIONIC_ERRDEF(EBADMSG, "Not a data message") -__BIONIC_ERRDEF(EOVERFLOW, "Value too large for defined data type") -__BIONIC_ERRDEF(ENOTUNIQ, "Name not unique on network") -__BIONIC_ERRDEF(EBADFD, "File descriptor in bad state") -__BIONIC_ERRDEF(EREMCHG, "Remote address changed") -__BIONIC_ERRDEF(ELIBACC, "Can not access a needed shared library") -__BIONIC_ERRDEF(ELIBBAD, "Accessing a corrupted shared library") -__BIONIC_ERRDEF(ELIBSCN, ".lib section in a.out corrupted") -__BIONIC_ERRDEF(ELIBMAX, "Attempting to link in too many shared libraries") -__BIONIC_ERRDEF(ELIBEXEC, "Cannot exec a shared library directly") -__BIONIC_ERRDEF(EILSEQ, "Illegal byte sequence") -__BIONIC_ERRDEF(ERESTART, "Interrupted system call should be restarted") -__BIONIC_ERRDEF(ESTRPIPE, "Streams pipe error") -__BIONIC_ERRDEF(EUSERS, "Too many users") -__BIONIC_ERRDEF(ENOTSOCK, "Socket operation on non-socket") -__BIONIC_ERRDEF(EDESTADDRREQ, "Destination address required") -__BIONIC_ERRDEF(EMSGSIZE, "Message too long") -__BIONIC_ERRDEF(EPROTOTYPE, "Protocol wrong type for socket") -__BIONIC_ERRDEF(ENOPROTOOPT, "Protocol not available") -__BIONIC_ERRDEF(EPROTONOSUPPORT, "Protocol not supported") -__BIONIC_ERRDEF(ESOCKTNOSUPPORT, "Socket type not supported") -__BIONIC_ERRDEF(EOPNOTSUPP, "Operation not supported on transport endpoint") -__BIONIC_ERRDEF(EPFNOSUPPORT, "Protocol family not supported") -__BIONIC_ERRDEF(EAFNOSUPPORT, "Address family not supported by protocol") -__BIONIC_ERRDEF(EADDRINUSE, "Address already in use") -__BIONIC_ERRDEF(EADDRNOTAVAIL, "Cannot assign requested address") -__BIONIC_ERRDEF(ENETDOWN, "Network is down") -__BIONIC_ERRDEF(ENETUNREACH, "Network is unreachable") -__BIONIC_ERRDEF(ENETRESET, "Network dropped connection because of reset") -__BIONIC_ERRDEF(ECONNABORTED, "Software caused connection abort") -__BIONIC_ERRDEF(ECONNRESET, "Connection reset by peer") -__BIONIC_ERRDEF(ENOBUFS, "No buffer space available") -__BIONIC_ERRDEF(EISCONN, "Transport endpoint is already connected") -__BIONIC_ERRDEF(ENOTCONN, "Transport endpoint is not connected") -__BIONIC_ERRDEF(ESHUTDOWN, "Cannot send after transport endpoint shutdown") -__BIONIC_ERRDEF(ETOOMANYREFS, "Too many references: cannot splice") -__BIONIC_ERRDEF(ETIMEDOUT, "Connection timed out") -__BIONIC_ERRDEF(ECONNREFUSED, "Connection refused") -__BIONIC_ERRDEF(EHOSTDOWN, "Host is down") -__BIONIC_ERRDEF(EHOSTUNREACH, "No route to host") -__BIONIC_ERRDEF(EALREADY, "Operation already in progress") -__BIONIC_ERRDEF(EINPROGRESS, "Operation now in progress") -__BIONIC_ERRDEF(ESTALE, "Stale NFS file handle") -__BIONIC_ERRDEF(EUCLEAN, "Structure needs cleaning") -__BIONIC_ERRDEF(ENOTNAM, "Not a XENIX named type file") -__BIONIC_ERRDEF(ENAVAIL, "No XENIX semaphores available") -__BIONIC_ERRDEF(EISNAM, "Is a named type file") -__BIONIC_ERRDEF(EREMOTEIO, "Remote I/O error") -__BIONIC_ERRDEF(EDQUOT, "Quota exceeded") -__BIONIC_ERRDEF(ENOMEDIUM, "No medium found") -__BIONIC_ERRDEF(EMEDIUMTYPE, "Wrong medium type") -__BIONIC_ERRDEF(ECANCELED, "Operation Canceled") -__BIONIC_ERRDEF(ENOKEY, "Required key not available") -__BIONIC_ERRDEF(EKEYEXPIRED, "Key has expired") -__BIONIC_ERRDEF(EKEYREVOKED, "Key has been revoked") -__BIONIC_ERRDEF(EKEYREJECTED, "Key was rejected by service") -__BIONIC_ERRDEF(EOWNERDEAD, "Owner died") -__BIONIC_ERRDEF(ENOTRECOVERABLE, "State not recoverable") -__BIONIC_ERRDEF(ERFKILL, "Operation not possible due to RF-kill") -__BIONIC_ERRDEF(EHWPOISON, "Memory page has hardware error") - -#undef __BIONIC_ERRDEF diff --git a/native/src/crt0/bionic/strerror.cpp b/native/src/crt0/bionic/strerror.cpp deleted file mode 100644 index b6a8f5e99..000000000 --- a/native/src/crt0/bionic/strerror.cpp +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright (C) 2012 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -// -std=gnu++XX automatically defines _GNU_SOURCE, which then means that -// gives us the GNU variant, which is not what we're defining here. -#undef _GNU_SOURCE - -#include - -#include -#include - -#include "private/ErrnoRestorer.h" - -#include -#include - -// The arraysize(arr) macro returns the # of elements in an array arr. -// The expression is a compile-time constant, and therefore can be -// used in defining new arrays, for example. If you use arraysize on -// a pointer by mistake, you will get a compile-time error. -// -// One caveat is that arraysize() doesn't accept any array of an -// anonymous type or a type defined inside a function. -// -// This template function declaration is used in defining arraysize. -// Note that the function doesn't need an implementation, as we only -// use its type. -template -char (&ArraySizeHelper(T (&array)[N]))[N]; // NOLINT(readability/casting) - -#define arraysize(array) (sizeof(ArraySizeHelper(array))) - -static const char* __sys_error_descriptions[] = { -#define __BIONIC_ERRDEF(error_number, error_description) [error_number] = error_description, -#include "private/bionic_errdefs.h" -}; - -static inline const char* __strerror_lookup(int error_number) { - if (error_number < 0 || error_number >= static_cast(arraysize(__sys_error_descriptions))) { - return nullptr; - } - return __sys_error_descriptions[error_number]; -} - -int strerror_r(int error_number, char* buf, size_t buf_len) { - ErrnoRestorer errno_restorer; - size_t length; - - const char* error_name = __strerror_lookup(error_number); - if (error_name != nullptr) { - length = strlcpy(buf, error_name, buf_len); - } else { - length = snprintf(buf, buf_len, "Unknown error %d", error_number); - } - if (length >= buf_len) { - return ERANGE; - } - - return 0; -} - -char* strerror(int error_number) { - static char strerror_buf[NL_TEXTMAX]; - - // Just return the original constant in the easy cases. - char* result = const_cast(__strerror_lookup(error_number)); - if (result != nullptr) { - return result; - } - - result = strerror_buf; - strerror_r(error_number, result, sizeof(strerror_buf)); - return result; -} diff --git a/native/src/crt0/bionic/syscall-arm.S b/native/src/crt0/bionic/syscall-arm.S deleted file mode 100644 index 8b603835a..000000000 --- a/native/src/crt0/bionic/syscall-arm.S +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2008 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "private/bionic_asm.h" - -ENTRY(syscall) - mov ip, sp - stmfd sp!, {r4, r5, r6, r7} - .cfi_def_cfa_offset 16 - .cfi_rel_offset r4, 0 - .cfi_rel_offset r5, 4 - .cfi_rel_offset r6, 8 - .cfi_rel_offset r7, 12 - mov r7, r0 - mov r0, r1 - mov r1, r2 - mov r2, r3 - ldmfd ip, {r3, r4, r5, r6} - swi #0 - ldmfd sp!, {r4, r5, r6, r7} - .cfi_def_cfa_offset 0 - cmn r0, #(MAX_ERRNO + 1) - bxls lr - neg r0, r0 - b __set_errno_internal -END(syscall) diff --git a/native/src/crt0/bionic/syscall-arm64.S b/native/src/crt0/bionic/syscall-arm64.S deleted file mode 100644 index acd2b4205..000000000 --- a/native/src/crt0/bionic/syscall-arm64.S +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include "private/bionic_asm.h" - -ENTRY(syscall) - /* Move syscall No. from x0 to x8 */ - mov x8, x0 - /* Move syscall parameters from x1 thru x6 to x0 thru x5 */ - mov x0, x1 - mov x1, x2 - mov x2, x3 - mov x3, x4 - mov x4, x5 - mov x5, x6 - svc #0 - - /* check if syscall returned successfully */ - cmn x0, #(MAX_ERRNO + 1) - cneg x0, x0, hi - b.hi __set_errno_internal - - ret -END(syscall) - -NOTE_GNU_PROPERTY() diff --git a/native/src/crt0/bionic/syscall-x86.S b/native/src/crt0/bionic/syscall-x86.S deleted file mode 100644 index c44ce3623..000000000 --- a/native/src/crt0/bionic/syscall-x86.S +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Generic syscall call. - * Upon entry: - * %eax: system call number - caller save - * %ebx: arg0 to system call - callee save - * %ecx: arg1 - caller save - * %edx: arg2 - caller save - * %esi: arg3 - callee save - * %edi: arg4 - callee save - * %ebp: arg5 - callee save - */ - -#include "private/bionic_asm.h" - -ENTRY(syscall) - # Push the callee save registers. - push %ebx - .cfi_adjust_cfa_offset 4 - .cfi_rel_offset ebx, 0 - push %esi - .cfi_adjust_cfa_offset 4 - .cfi_rel_offset esi, 0 - push %edi - .cfi_adjust_cfa_offset 4 - .cfi_rel_offset edi, 0 - push %ebp - .cfi_adjust_cfa_offset 4 - .cfi_rel_offset ebp, 0 - - # Load all the arguments from the calling frame. - # (Not all will be valid, depending on the syscall.) - mov 20(%esp),%eax - mov 24(%esp),%ebx - mov 28(%esp),%ecx - mov 32(%esp),%edx - mov 36(%esp),%esi - mov 40(%esp),%edi - mov 44(%esp),%ebp - - # Make the system call. - int $0x80 - - # Error? - cmpl $-MAX_ERRNO, %eax - jb 1f - # Yes, so set errno. - negl %eax - pushl %eax - call __set_errno_internal - addl $4, %esp -1: - # Restore the callee save registers. - pop %ebp - .cfi_adjust_cfa_offset -4 - .cfi_restore ebp - pop %edi - .cfi_adjust_cfa_offset -4 - .cfi_restore edi - pop %esi - .cfi_adjust_cfa_offset -4 - .cfi_restore esi - pop %ebx - .cfi_adjust_cfa_offset -4 - .cfi_restore ebx - ret -END(syscall) diff --git a/native/src/crt0/bionic/syscall-x86_64.S b/native/src/crt0/bionic/syscall-x86_64.S deleted file mode 100644 index 7a6f4104b..000000000 --- a/native/src/crt0/bionic/syscall-x86_64.S +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (C) 2013 The Android Open Source Project - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, - * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS - * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED - * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT - * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * Generic syscall call. - * Upon entry: - * %rax: system call number - * %rdi: arg0 to system call - * %rsi: arg1 - * %rdx: arg2 - * %rcx: arg3 - syscall expects it at %r10 - * %r8: arg4 - * %r9: arg5 - */ - -#include "private/bionic_asm.h" - -ENTRY(syscall) - # All arguments are passed via registers. - # (Not all will be valid, depending on the syscall.) - mov %edi, %eax - mov %rsi, %rdi - mov %rdx, %rsi - mov %rcx, %rdx - mov %r8, %r10 - mov %r9, %r8 - mov 8(%rsp), %r9 - - # Make the system call. - syscall - cmpq $-MAX_ERRNO, %rax - jb 1f - negl %eax - movl %eax, %edi - call __set_errno_internal -1: - ret -END(syscall) diff --git a/native/src/crt0/dlmalloc/malloc.c b/native/src/crt0/dlmalloc/malloc.c deleted file mode 100644 index b1495f496..000000000 --- a/native/src/crt0/dlmalloc/malloc.c +++ /dev/null @@ -1,6291 +0,0 @@ -/* -Copyright 2023 Doug Lea - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -* Version 2.8.6 Wed Aug 29 06:57:58 2012 Doug Lea - Re-licensed 25 Sep 2023 with MIT-0 replacing obsolete CC0 - See https://opensource.org/license/mit-0/ - -* Quickstart - - This library is all in one file to simplify the most common usage: - ftp it, compile it (-O3), and link it into another program. All of - the compile-time options default to reasonable values for use on - most platforms. You might later want to step through various - compile-time and dynamic tuning options. - - For convenience, an include file for code using this malloc is at: - ftp://gee.cs.oswego.edu/pub/misc/malloc-2.8.6.h - You don't really need this .h file unless you call functions not - defined in your system include files. The .h file contains only the - excerpts from this file needed for using this malloc on ANSI C/C++ - systems, so long as you haven't changed compile-time options about - naming and tuning parameters. If you do, then you can create your - own malloc.h that does include all settings by cutting at the point - indicated below. Note that you may already by default be using a C - library containing a malloc that is based on some version of this - malloc (for example in linux). You might still want to use the one - in this file to customize settings or to avoid overheads associated - with library versions. - -* Vital statistics: - - Supported pointer/size_t representation: 4 or 8 bytes - size_t MUST be an unsigned type of the same width as - pointers. (If you are using an ancient system that declares - size_t as a signed type, or need it to be a different width - than pointers, you can use a previous release of this malloc - (e.g. 2.7.2) supporting these.) - - Alignment: 8 bytes (minimum) - This suffices for nearly all current machines and C compilers. - However, you can define MALLOC_ALIGNMENT to be wider than this - if necessary (up to 128bytes), at the expense of using more space. - - Minimum overhead per allocated chunk: 4 or 8 bytes (if 4byte sizes) - 8 or 16 bytes (if 8byte sizes) - Each malloced chunk has a hidden word of overhead holding size - and status information, and additional cross-check word - if FOOTERS is defined. - - Minimum allocated size: 4-byte ptrs: 16 bytes (including overhead) - 8-byte ptrs: 32 bytes (including overhead) - - Even a request for zero bytes (i.e., malloc(0)) returns a - pointer to something of the minimum allocatable size. - The maximum overhead wastage (i.e., number of extra bytes - allocated than were requested in malloc) is less than or equal - to the minimum size, except for requests >= mmap_threshold that - are serviced via mmap(), where the worst case wastage is about - 32 bytes plus the remainder from a system page (the minimal - mmap unit); typically 4096 or 8192 bytes. - - Security: static-safe; optionally more or less - The "security" of malloc refers to the ability of malicious - code to accentuate the effects of errors (for example, freeing - space that is not currently malloc'ed or overwriting past the - ends of chunks) in code that calls malloc. This malloc - guarantees not to modify any memory locations below the base of - heap, i.e., static variables, even in the presence of usage - errors. The routines additionally detect most improper frees - and reallocs. All this holds as long as the static bookkeeping - for malloc itself is not corrupted by some other means. This - is only one aspect of security -- these checks do not, and - cannot, detect all possible programming errors. - - If FOOTERS is defined nonzero, then each allocated chunk - carries an additional check word to verify that it was malloced - from its space. These check words are the same within each - execution of a program using malloc, but differ across - executions, so externally crafted fake chunks cannot be - freed. This improves security by rejecting frees/reallocs that - could corrupt heap memory, in addition to the checks preventing - writes to statics that are always on. This may further improve - security at the expense of time and space overhead. (Note that - FOOTERS may also be worth using with MSPACES.) - - By default detected errors cause the program to abort (calling - "abort()"). You can override this to instead proceed past - errors by defining PROCEED_ON_ERROR. In this case, a bad free - has no effect, and a malloc that encounters a bad address - caused by user overwrites will ignore the bad address by - dropping pointers and indices to all known memory. This may - be appropriate for programs that should continue if at all - possible in the face of programming errors, although they may - run out of memory because dropped memory is never reclaimed. - - If you don't like either of these options, you can define - CORRUPTION_ERROR_ACTION and USAGE_ERROR_ACTION to do anything - else. And if if you are sure that your program using malloc has - no errors or vulnerabilities, you can define INSECURE to 1, - which might (or might not) provide a small performance improvement. - - It is also possible to limit the maximum total allocatable - space, using malloc_set_footprint_limit. This is not - designed as a security feature in itself (calls to set limits - are not screened or privileged), but may be useful as one - aspect of a secure implementation. - - Thread-safety: NOT thread-safe unless USE_LOCKS defined non-zero - When USE_LOCKS is defined, each public call to malloc, free, - etc is surrounded with a lock. By default, this uses a plain - pthread mutex, win32 critical section, or a spin-lock if if - available for the platform and not disabled by setting - USE_SPIN_LOCKS=0. However, if USE_RECURSIVE_LOCKS is defined, - recursive versions are used instead (which are not required for - base functionality but may be needed in layered extensions). - Using a global lock is not especially fast, and can be a major - bottleneck. It is designed only to provide minimal protection - in concurrent environments, and to provide a basis for - extensions. If you are using malloc in a concurrent program, - consider instead using nedmalloc - (http://www.nedprod.com/programs/portable/nedmalloc/) or - ptmalloc (See http://www.malloc.de), which are derived from - versions of this malloc. - - System requirements: Any combination of MORECORE and/or MMAP/MUNMAP - This malloc can use unix sbrk or any emulation (invoked using - the CALL_MORECORE macro) and/or mmap/munmap or any emulation - (invoked using CALL_MMAP/CALL_MUNMAP) to get and release system - memory. On most unix systems, it tends to work best if both - MORECORE and MMAP are enabled. On Win32, it uses emulations - based on VirtualAlloc. It also uses common C library functions - like memset. - - Compliance: I believe it is compliant with the Single Unix Specification - (See http://www.unix.org). Also SVID/XPG, ANSI C, and probably - others as well. - -* Overview of algorithms - - This is not the fastest, most space-conserving, most portable, or - most tunable malloc ever written. However it is among the fastest - while also being among the most space-conserving, portable and - tunable. Consistent balance across these factors results in a good - general-purpose allocator for malloc-intensive programs. - - In most ways, this malloc is a best-fit allocator. Generally, it - chooses the best-fitting existing chunk for a request, with ties - broken in approximately least-recently-used order. (This strategy - normally maintains low fragmentation.) However, for requests less - than 256bytes, it deviates from best-fit when there is not an - exactly fitting available chunk by preferring to use space adjacent - to that used for the previous small request, as well as by breaking - ties in approximately most-recently-used order. (These enhance - locality of series of small allocations.) And for very large requests - (>= 256Kb by default), it relies on system memory mapping - facilities, if supported. (This helps avoid carrying around and - possibly fragmenting memory used only for large chunks.) - - All operations (except malloc_stats and mallinfo) have execution - times that are bounded by a constant factor of the number of bits in - a size_t, not counting any clearing in calloc or copying in realloc, - or actions surrounding MORECORE and MMAP that have times - proportional to the number of non-contiguous regions returned by - system allocation routines, which is often just 1. In real-time - applications, you can optionally suppress segment traversals using - NO_SEGMENT_TRAVERSAL, which assures bounded execution even when - system allocators return non-contiguous spaces, at the typical - expense of carrying around more memory and increased fragmentation. - - The implementation is not very modular and seriously overuses - macros. Perhaps someday all C compilers will do as good a job - inlining modular code as can now be done by brute-force expansion, - but now, enough of them seem not to. - - Some compilers issue a lot of warnings about code that is - dead/unreachable only on some platforms, and also about intentional - uses of negation on unsigned types. All known cases of each can be - ignored. - - For a longer but out of date high-level description, see - http://gee.cs.oswego.edu/dl/html/malloc.html - -* MSPACES - If MSPACES is defined, then in addition to malloc, free, etc., - this file also defines mspace_malloc, mspace_free, etc. These - are versions of malloc routines that take an "mspace" argument - obtained using create_mspace, to control all internal bookkeeping. - If ONLY_MSPACES is defined, only these versions are compiled. - So if you would like to use this allocator for only some allocations, - and your system malloc for others, you can compile with - ONLY_MSPACES and then do something like... - static mspace mymspace = create_mspace(0,0); // for example - #define mymalloc(bytes) mspace_malloc(mymspace, bytes) - - (Note: If you only need one instance of an mspace, you can instead - use "USE_DL_PREFIX" to relabel the global malloc.) - - You can similarly create thread-local allocators by storing - mspaces as thread-locals. For example: - static __thread mspace tlms = 0; - void* tlmalloc(size_t bytes) { - if (tlms == 0) tlms = create_mspace(0, 0); - return mspace_malloc(tlms, bytes); - } - void tlfree(void* mem) { mspace_free(tlms, mem); } - - Unless FOOTERS is defined, each mspace is completely independent. - You cannot allocate from one and free to another (although - conformance is only weakly checked, so usage errors are not always - caught). If FOOTERS is defined, then each chunk carries around a tag - indicating its originating mspace, and frees are directed to their - originating spaces. Normally, this requires use of locks. - - ------------------------- Compile-time options --------------------------- - -Be careful in setting #define values for numerical constants of type -size_t. On some systems, literal values are not automatically extended -to size_t precision unless they are explicitly casted. You can also -use the symbolic values MAX_SIZE_T, SIZE_T_ONE, etc below. - -WIN32 default: defined if _WIN32 defined - Defining WIN32 sets up defaults for MS environment and compilers. - Otherwise defaults are for unix. Beware that there seem to be some - cases where this malloc might not be a pure drop-in replacement for - Win32 malloc: Random-looking failures from Win32 GDI API's (eg; - SetDIBits()) may be due to bugs in some video driver implementations - when pixel buffers are malloc()ed, and the region spans more than - one VirtualAlloc()ed region. Because dlmalloc uses a small (64Kb) - default granularity, pixel buffers may straddle virtual allocation - regions more often than when using the Microsoft allocator. You can - avoid this by using VirtualAlloc() and VirtualFree() for all pixel - buffers rather than using malloc(). If this is not possible, - recompile this malloc with a larger DEFAULT_GRANULARITY. Note: - in cases where MSC and gcc (cygwin) are known to differ on WIN32, - conditions use _MSC_VER to distinguish them. - -DLMALLOC_EXPORT default: extern - Defines how public APIs are declared. If you want to export via a - Windows DLL, you might define this as - #define DLMALLOC_EXPORT extern __declspec(dllexport) - If you want a POSIX ELF shared object, you might use - #define DLMALLOC_EXPORT extern __attribute__((visibility("default"))) - -MALLOC_ALIGNMENT default: (size_t)(2 * sizeof(void *)) - Controls the minimum alignment for malloc'ed chunks. It must be a - power of two and at least 8, even on machines for which smaller - alignments would suffice. It may be defined as larger than this - though. Note however that code and data structures are optimized for - the case of 8-byte alignment. - -MSPACES default: 0 (false) - If true, compile in support for independent allocation spaces. - This is only supported if HAVE_MMAP is true. - -ONLY_MSPACES default: 0 (false) - If true, only compile in mspace versions, not regular versions. - -USE_LOCKS default: 0 (false) - Causes each call to each public routine to be surrounded with - pthread or WIN32 mutex lock/unlock. (If set true, this can be - overridden on a per-mspace basis for mspace versions.) If set to a - non-zero value other than 1, locks are used, but their - implementation is left out, so lock functions must be supplied manually, - as described below. - -USE_SPIN_LOCKS default: 1 iff USE_LOCKS and spin locks available - If true, uses custom spin locks for locking. This is currently - supported only gcc >= 4.1, older gccs on x86 platforms, and recent - MS compilers. Otherwise, posix locks or win32 critical sections are - used. - -USE_RECURSIVE_LOCKS default: not defined - If defined nonzero, uses recursive (aka reentrant) locks, otherwise - uses plain mutexes. This is not required for malloc proper, but may - be needed for layered allocators such as nedmalloc. - -LOCK_AT_FORK default: not defined - If defined nonzero, performs pthread_atfork upon initialization - to initialize child lock while holding parent lock. The implementation - assumes that pthread locks (not custom locks) are being used. In other - cases, you may need to customize the implementation. - -FOOTERS default: 0 - If true, provide extra checking and dispatching by placing - information in the footers of allocated chunks. This adds - space and time overhead. - -INSECURE default: 0 - If true, omit checks for usage errors and heap space overwrites. - -USE_DL_PREFIX default: NOT defined - Causes compiler to prefix all public routines with the string 'dl'. - This can be useful when you only want to use this malloc in one part - of a program, using your regular system malloc elsewhere. - -MALLOC_INSPECT_ALL default: NOT defined - If defined, compiles malloc_inspect_all and mspace_inspect_all, that - perform traversal of all heap space. Unless access to these - functions is otherwise restricted, you probably do not want to - include them in secure implementations. - -ABORT default: defined as abort() - Defines how to abort on failed checks. On most systems, a failed - check cannot die with an "assert" or even print an informative - message, because the underlying print routines in turn call malloc, - which will fail again. Generally, the best policy is to simply call - abort(). It's not very useful to do more than this because many - errors due to overwriting will show up as address faults (null, odd - addresses etc) rather than malloc-triggered checks, so will also - abort. Also, most compilers know that abort() does not return, so - can better optimize code conditionally calling it. - -PROCEED_ON_ERROR default: defined as 0 (false) - Controls whether detected bad addresses cause them to bypassed - rather than aborting. If set, detected bad arguments to free and - realloc are ignored. And all bookkeeping information is zeroed out - upon a detected overwrite of freed heap space, thus losing the - ability to ever return it from malloc again, but enabling the - application to proceed. If PROCEED_ON_ERROR is defined, the - static variable malloc_corruption_error_count is compiled in - and can be examined to see if errors have occurred. This option - generates slower code than the default abort policy. - -DEBUG default: NOT defined - The DEBUG setting is mainly intended for people trying to modify - this code or diagnose problems when porting to new platforms. - However, it may also be able to better isolate user errors than just - using runtime checks. The assertions in the check routines spell - out in more detail the assumptions and invariants underlying the - algorithms. The checking is fairly extensive, and will slow down - execution noticeably. Calling malloc_stats or mallinfo with DEBUG - set will attempt to check every non-mmapped allocated and free chunk - in the course of computing the summaries. - -ABORT_ON_ASSERT_FAILURE default: defined as 1 (true) - Debugging assertion failures can be nearly impossible if your - version of the assert macro causes malloc to be called, which will - lead to a cascade of further failures, blowing the runtime stack. - ABORT_ON_ASSERT_FAILURE cause assertions failures to call abort(), - which will usually make debugging easier. - -MALLOC_FAILURE_ACTION default: sets errno to ENOMEM, or no-op on win32 - The action to take before "return 0" when malloc fails to be able to - return memory because there is none available. - -HAVE_MORECORE default: 1 (true) unless win32 or ONLY_MSPACES - True if this system supports sbrk or an emulation of it. - -MORECORE default: sbrk - The name of the sbrk-style system routine to call to obtain more - memory. See below for guidance on writing custom MORECORE - functions. The type of the argument to sbrk/MORECORE varies across - systems. It cannot be size_t, because it supports negative - arguments, so it is normally the signed type of the same width as - size_t (sometimes declared as "intptr_t"). It doesn't much matter - though. Internally, we only call it with arguments less than half - the max value of a size_t, which should work across all reasonable - possibilities, although sometimes generating compiler warnings. - -MORECORE_CONTIGUOUS default: 1 (true) if HAVE_MORECORE - If true, take advantage of fact that consecutive calls to MORECORE - with positive arguments always return contiguous increasing - addresses. This is true of unix sbrk. It does not hurt too much to - set it true anyway, since malloc copes with non-contiguities. - Setting it false when definitely non-contiguous saves time - and possibly wasted space it would take to discover this though. - -MORECORE_CANNOT_TRIM default: NOT defined - True if MORECORE cannot release space back to the system when given - negative arguments. This is generally necessary only if you are - using a hand-crafted MORECORE function that cannot handle negative - arguments. - -NO_SEGMENT_TRAVERSAL default: 0 - If non-zero, suppresses traversals of memory segments - returned by either MORECORE or CALL_MMAP. This disables - merging of segments that are contiguous, and selectively - releasing them to the OS if unused, but bounds execution times. - -HAVE_MMAP default: 1 (true) - True if this system supports mmap or an emulation of it. If so, and - HAVE_MORECORE is not true, MMAP is used for all system - allocation. If set and HAVE_MORECORE is true as well, MMAP is - primarily used to directly allocate very large blocks. It is also - used as a backup strategy in cases where MORECORE fails to provide - space from system. Note: A single call to MUNMAP is assumed to be - able to unmap memory that may have be allocated using multiple calls - to MMAP, so long as they are adjacent. - -HAVE_MREMAP default: 1 on linux, else 0 - If true realloc() uses mremap() to re-allocate large blocks and - extend or shrink allocation spaces. - -MMAP_CLEARS default: 1 except on WINCE. - True if mmap clears memory so calloc doesn't need to. This is true - for standard unix mmap using /dev/zero and on WIN32 except for WINCE. - -USE_BUILTIN_FFS default: 0 (i.e., not used) - Causes malloc to use the builtin ffs() function to compute indices. - Some compilers may recognize and intrinsify ffs to be faster than the - supplied C version. Also, the case of x86 using gcc is special-cased - to an asm instruction, so is already as fast as it can be, and so - this setting has no effect. Similarly for Win32 under recent MS compilers. - (On most x86s, the asm version is only slightly faster than the C version.) - -malloc_getpagesize default: derive from system includes, or 4096. - The system page size. To the extent possible, this malloc manages - memory from the system in page-size units. This may be (and - usually is) a function rather than a constant. This is ignored - if WIN32, where page size is determined using getSystemInfo during - initialization. - -USE_DEV_RANDOM default: 0 (i.e., not used) - Causes malloc to use /dev/random to initialize secure magic seed for - stamping footers. Otherwise, the current time is used. - -NO_MALLINFO default: 0 - If defined, don't compile "mallinfo". This can be a simple way - of dealing with mismatches between system declarations and - those in this file. - -MALLINFO_FIELD_TYPE default: size_t - The type of the fields in the mallinfo struct. This was originally - defined as "int" in SVID etc, but is more usefully defined as - size_t. The value is used only if HAVE_USR_INCLUDE_MALLOC_H is not set - -NO_MALLOC_STATS default: 0 - If defined, don't compile "malloc_stats". This avoids calls to - fprintf and bringing in stdio dependencies you might not want. - -REALLOC_ZERO_BYTES_FREES default: not defined - This should be set if a call to realloc with zero bytes should - be the same as a call to free. Some people think it should. Otherwise, - since this malloc returns a unique pointer for malloc(0), so does - realloc(p, 0). - -LACKS_UNISTD_H, LACKS_FCNTL_H, LACKS_SYS_PARAM_H, LACKS_SYS_MMAN_H -LACKS_STRINGS_H, LACKS_STRING_H, LACKS_SYS_TYPES_H, LACKS_ERRNO_H -LACKS_STDLIB_H LACKS_SCHED_H LACKS_TIME_H default: NOT defined unless on WIN32 - Define these if your system does not have these header files. - You might need to manually insert some of the declarations they provide. - -DEFAULT_GRANULARITY default: page size if MORECORE_CONTIGUOUS, - system_info.dwAllocationGranularity in WIN32, - otherwise 64K. - Also settable using mallopt(M_GRANULARITY, x) - The unit for allocating and deallocating memory from the system. On - most systems with contiguous MORECORE, there is no reason to - make this more than a page. However, systems with MMAP tend to - either require or encourage larger granularities. You can increase - this value to prevent system allocation functions to be called so - often, especially if they are slow. The value must be at least one - page and must be a power of two. Setting to 0 causes initialization - to either page size or win32 region size. (Note: In previous - versions of malloc, the equivalent of this option was called - "TOP_PAD") - -DEFAULT_TRIM_THRESHOLD default: 2MB - Also settable using mallopt(M_TRIM_THRESHOLD, x) - The maximum amount of unused top-most memory to keep before - releasing via malloc_trim in free(). Automatic trimming is mainly - useful in long-lived programs using contiguous MORECORE. Because - trimming via sbrk can be slow on some systems, and can sometimes be - wasteful (in cases where programs immediately afterward allocate - more large chunks) the value should be high enough so that your - overall system performance would improve by releasing this much - memory. As a rough guide, you might set to a value close to the - average size of a process (program) running on your system. - Releasing this much memory would allow such a process to run in - memory. Generally, it is worth tuning trim thresholds when a - program undergoes phases where several large chunks are allocated - and released in ways that can reuse each other's storage, perhaps - mixed with phases where there are no such chunks at all. The trim - value must be greater than page size to have any useful effect. To - disable trimming completely, you can set to MAX_SIZE_T. Note that the trick - some people use of mallocing a huge space and then freeing it at - program startup, in an attempt to reserve system memory, doesn't - have the intended effect under automatic trimming, since that memory - will immediately be returned to the system. - -DEFAULT_MMAP_THRESHOLD default: 256K - Also settable using mallopt(M_MMAP_THRESHOLD, x) - The request size threshold for using MMAP to directly service a - request. Requests of at least this size that cannot be allocated - using already-existing space will be serviced via mmap. (If enough - normal freed space already exists it is used instead.) Using mmap - segregates relatively large chunks of memory so that they can be - individually obtained and released from the host system. A request - serviced through mmap is never reused by any other request (at least - not directly; the system may just so happen to remap successive - requests to the same locations). Segregating space in this way has - the benefits that: Mmapped space can always be individually released - back to the system, which helps keep the system level memory demands - of a long-lived program low. Also, mapped memory doesn't become - `locked' between other chunks, as can happen with normally allocated - chunks, which means that even trimming via malloc_trim would not - release them. However, it has the disadvantage that the space - cannot be reclaimed, consolidated, and then used to service later - requests, as happens with normal chunks. The advantages of mmap - nearly always outweigh disadvantages for "large" chunks, but the - value of "large" may vary across systems. The default is an - empirically derived value that works well in most systems. You can - disable mmap by setting to MAX_SIZE_T. - -MAX_RELEASE_CHECK_RATE default: 4095 unless not HAVE_MMAP - The number of consolidated frees between checks to release - unused segments when freeing. When using non-contiguous segments, - especially with multiple mspaces, checking only for topmost space - doesn't always suffice to trigger trimming. To compensate for this, - free() will, with a period of MAX_RELEASE_CHECK_RATE (or the - current number of segments, if greater) try to release unused - segments to the OS when freeing chunks that result in - consolidation. The best value for this parameter is a compromise - between slowing down frees with relatively costly checks that - rarely trigger versus holding on to unused memory. To effectively - disable, set to MAX_SIZE_T. This may lead to a very slight speed - improvement at the expense of carrying around more memory. -*/ - -/* Version identifier to allow people to support multiple versions */ -#ifndef DLMALLOC_VERSION -#define DLMALLOC_VERSION 20806 -#endif /* DLMALLOC_VERSION */ - -#ifndef DLMALLOC_EXPORT -#define DLMALLOC_EXPORT extern -#endif - -#ifndef WIN32 -#ifdef _WIN32 -#define WIN32 1 -#endif /* _WIN32 */ -#ifdef _WIN32_WCE -#define LACKS_FCNTL_H -#define WIN32 1 -#endif /* _WIN32_WCE */ -#endif /* WIN32 */ -#ifdef WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#include -#define HAVE_MMAP 1 -#define HAVE_MORECORE 0 -#define LACKS_UNISTD_H -#define LACKS_SYS_PARAM_H -#define LACKS_SYS_MMAN_H -#define LACKS_STRING_H -#define LACKS_STRINGS_H -#define LACKS_SYS_TYPES_H -#define LACKS_ERRNO_H -#define LACKS_SCHED_H -#ifndef MALLOC_FAILURE_ACTION -#define MALLOC_FAILURE_ACTION -#endif /* MALLOC_FAILURE_ACTION */ -#ifndef MMAP_CLEARS -#ifdef _WIN32_WCE /* WINCE reportedly does not clear */ -#define MMAP_CLEARS 0 -#else -#define MMAP_CLEARS 1 -#endif /* _WIN32_WCE */ -#endif /*MMAP_CLEARS */ -#endif /* WIN32 */ - -#if defined(DARWIN) || defined(_DARWIN) -/* Mac OSX docs advise not to use sbrk; it seems better to use mmap */ -#ifndef HAVE_MORECORE -#define HAVE_MORECORE 0 -#define HAVE_MMAP 1 -/* OSX allocators provide 16 byte alignment */ -#ifndef MALLOC_ALIGNMENT -#define MALLOC_ALIGNMENT ((size_t)16U) -#endif -#endif /* HAVE_MORECORE */ -#endif /* DARWIN */ - -#ifndef LACKS_SYS_TYPES_H -#include /* For size_t */ -#endif /* LACKS_SYS_TYPES_H */ - -/* The maximum possible size_t value has all bits set */ -#define MAX_SIZE_T (~(size_t)0) - -#ifndef USE_LOCKS /* ensure true if spin or recursive locks set */ -#define USE_LOCKS ((defined(USE_SPIN_LOCKS) && USE_SPIN_LOCKS != 0) || \ - (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0)) -#endif /* USE_LOCKS */ - -#if USE_LOCKS /* Spin locks for gcc >= 4.1, older gcc on x86, MSC >= 1310 */ -#if ((defined(__GNUC__) && \ - ((__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) || \ - defined(__i386__) || defined(__x86_64__))) || \ - (defined(_MSC_VER) && _MSC_VER>=1310)) -#ifndef USE_SPIN_LOCKS -#define USE_SPIN_LOCKS 1 -#endif /* USE_SPIN_LOCKS */ -#elif USE_SPIN_LOCKS -#error "USE_SPIN_LOCKS defined without implementation" -#endif /* ... locks available... */ -#elif !defined(USE_SPIN_LOCKS) -#define USE_SPIN_LOCKS 0 -#endif /* USE_LOCKS */ - -#ifndef ONLY_MSPACES -#define ONLY_MSPACES 0 -#endif /* ONLY_MSPACES */ -#ifndef MSPACES -#if ONLY_MSPACES -#define MSPACES 1 -#else /* ONLY_MSPACES */ -#define MSPACES 0 -#endif /* ONLY_MSPACES */ -#endif /* MSPACES */ -#ifndef MALLOC_ALIGNMENT -#define MALLOC_ALIGNMENT ((size_t)(2 * sizeof(void *))) -#endif /* MALLOC_ALIGNMENT */ -#ifndef FOOTERS -#define FOOTERS 0 -#endif /* FOOTERS */ -#ifndef ABORT -#define ABORT abort() -#endif /* ABORT */ -#ifndef ABORT_ON_ASSERT_FAILURE -#define ABORT_ON_ASSERT_FAILURE 1 -#endif /* ABORT_ON_ASSERT_FAILURE */ -#ifndef PROCEED_ON_ERROR -#define PROCEED_ON_ERROR 0 -#endif /* PROCEED_ON_ERROR */ - -#ifndef INSECURE -#define INSECURE 0 -#endif /* INSECURE */ -#ifndef MALLOC_INSPECT_ALL -#define MALLOC_INSPECT_ALL 0 -#endif /* MALLOC_INSPECT_ALL */ -#ifndef HAVE_MMAP -#define HAVE_MMAP 1 -#endif /* HAVE_MMAP */ -#ifndef MMAP_CLEARS -#define MMAP_CLEARS 1 -#endif /* MMAP_CLEARS */ -#ifndef HAVE_MREMAP -#ifdef linux -#define HAVE_MREMAP 1 -#define _GNU_SOURCE /* Turns on mremap() definition */ -#else /* linux */ -#define HAVE_MREMAP 0 -#endif /* linux */ -#endif /* HAVE_MREMAP */ -#ifndef MALLOC_FAILURE_ACTION -#define MALLOC_FAILURE_ACTION errno = ENOMEM; -#endif /* MALLOC_FAILURE_ACTION */ -#ifndef HAVE_MORECORE -#if ONLY_MSPACES -#define HAVE_MORECORE 0 -#else /* ONLY_MSPACES */ -#define HAVE_MORECORE 1 -#endif /* ONLY_MSPACES */ -#endif /* HAVE_MORECORE */ -#if !HAVE_MORECORE -#define MORECORE_CONTIGUOUS 0 -#else /* !HAVE_MORECORE */ -#define MORECORE_DEFAULT sbrk -#ifndef MORECORE_CONTIGUOUS -#define MORECORE_CONTIGUOUS 1 -#endif /* MORECORE_CONTIGUOUS */ -#endif /* HAVE_MORECORE */ -#ifndef DEFAULT_GRANULARITY -#if (MORECORE_CONTIGUOUS || defined(WIN32)) -#define DEFAULT_GRANULARITY (0) /* 0 means to compute in init_mparams */ -#else /* MORECORE_CONTIGUOUS */ -#define DEFAULT_GRANULARITY ((size_t)64U * (size_t)1024U) -#endif /* MORECORE_CONTIGUOUS */ -#endif /* DEFAULT_GRANULARITY */ -#ifndef DEFAULT_TRIM_THRESHOLD -#ifndef MORECORE_CANNOT_TRIM -#define DEFAULT_TRIM_THRESHOLD ((size_t)2U * (size_t)1024U * (size_t)1024U) -#else /* MORECORE_CANNOT_TRIM */ -#define DEFAULT_TRIM_THRESHOLD MAX_SIZE_T -#endif /* MORECORE_CANNOT_TRIM */ -#endif /* DEFAULT_TRIM_THRESHOLD */ -#ifndef DEFAULT_MMAP_THRESHOLD -#if HAVE_MMAP -#define DEFAULT_MMAP_THRESHOLD ((size_t)256U * (size_t)1024U) -#else /* HAVE_MMAP */ -#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T -#endif /* HAVE_MMAP */ -#endif /* DEFAULT_MMAP_THRESHOLD */ -#ifndef MAX_RELEASE_CHECK_RATE -#if HAVE_MMAP -#define MAX_RELEASE_CHECK_RATE 4095 -#else -#define MAX_RELEASE_CHECK_RATE MAX_SIZE_T -#endif /* HAVE_MMAP */ -#endif /* MAX_RELEASE_CHECK_RATE */ -#ifndef USE_BUILTIN_FFS -#define USE_BUILTIN_FFS 0 -#endif /* USE_BUILTIN_FFS */ -#ifndef USE_DEV_RANDOM -#define USE_DEV_RANDOM 0 -#endif /* USE_DEV_RANDOM */ -#ifndef NO_MALLINFO -#define NO_MALLINFO 0 -#endif /* NO_MALLINFO */ -#ifndef MALLINFO_FIELD_TYPE -#define MALLINFO_FIELD_TYPE size_t -#endif /* MALLINFO_FIELD_TYPE */ -#ifndef NO_MALLOC_STATS -#define NO_MALLOC_STATS 0 -#endif /* NO_MALLOC_STATS */ -#ifndef NO_SEGMENT_TRAVERSAL -#define NO_SEGMENT_TRAVERSAL 0 -#endif /* NO_SEGMENT_TRAVERSAL */ - -/* - mallopt tuning options. SVID/XPG defines four standard parameter - numbers for mallopt, normally defined in malloc.h. None of these - are used in this malloc, so setting them has no effect. But this - malloc does support the following options. -*/ - -#define M_TRIM_THRESHOLD (-1) -#define M_GRANULARITY (-2) -#define M_MMAP_THRESHOLD (-3) - -/* ------------------------ Mallinfo declarations ------------------------ */ - -#if !NO_MALLINFO -/* - This version of malloc supports the standard SVID/XPG mallinfo - routine that returns a struct containing usage properties and - statistics. It should work on any system that has a - /usr/include/malloc.h defining struct mallinfo. The main - declaration needed is the mallinfo struct that is returned (by-copy) - by mallinfo(). The malloinfo struct contains a bunch of fields that - are not even meaningful in this version of malloc. These fields are - are instead filled by mallinfo() with other numbers that might be of - interest. - - HAVE_USR_INCLUDE_MALLOC_H should be set if you have a - /usr/include/malloc.h file that includes a declaration of struct - mallinfo. If so, it is included; else a compliant version is - declared below. These must be precisely the same for mallinfo() to - work. The original SVID version of this struct, defined on most - systems with mallinfo, declares all fields as ints. But some others - define as unsigned long. If your system defines the fields using a - type of different width than listed here, you MUST #include your - system version and #define HAVE_USR_INCLUDE_MALLOC_H. -*/ - -/* #define HAVE_USR_INCLUDE_MALLOC_H */ - -#ifdef HAVE_USR_INCLUDE_MALLOC_H -#include "/usr/include/malloc.h" -#else /* HAVE_USR_INCLUDE_MALLOC_H */ -#ifndef STRUCT_MALLINFO_DECLARED -/* HP-UX (and others?) redefines mallinfo unless _STRUCT_MALLINFO is defined */ -#define _STRUCT_MALLINFO -#define STRUCT_MALLINFO_DECLARED 1 -struct mallinfo { - MALLINFO_FIELD_TYPE arena; /* non-mmapped space allocated from system */ - MALLINFO_FIELD_TYPE ordblks; /* number of free chunks */ - MALLINFO_FIELD_TYPE smblks; /* always 0 */ - MALLINFO_FIELD_TYPE hblks; /* always 0 */ - MALLINFO_FIELD_TYPE hblkhd; /* space in mmapped regions */ - MALLINFO_FIELD_TYPE usmblks; /* maximum total allocated space */ - MALLINFO_FIELD_TYPE fsmblks; /* always 0 */ - MALLINFO_FIELD_TYPE uordblks; /* total allocated space */ - MALLINFO_FIELD_TYPE fordblks; /* total free space */ - MALLINFO_FIELD_TYPE keepcost; /* releasable (via malloc_trim) space */ -}; -#endif /* STRUCT_MALLINFO_DECLARED */ -#endif /* HAVE_USR_INCLUDE_MALLOC_H */ -#endif /* NO_MALLINFO */ - -/* - Try to persuade compilers to inline. The most critical functions for - inlining are defined as macros, so these aren't used for them. -*/ - -#ifndef FORCEINLINE - #if defined(__GNUC__) -#define FORCEINLINE __inline __attribute__ ((always_inline)) - #elif defined(_MSC_VER) - #define FORCEINLINE __forceinline - #endif -#endif -#ifndef NOINLINE - #if defined(__GNUC__) - #define NOINLINE __attribute__ ((noinline)) - #elif defined(_MSC_VER) - #define NOINLINE __declspec(noinline) - #else - #define NOINLINE - #endif -#endif - -#ifdef __cplusplus -extern "C" { -#ifndef FORCEINLINE - #define FORCEINLINE inline -#endif -#endif /* __cplusplus */ -#ifndef FORCEINLINE - #define FORCEINLINE -#endif - -#if !ONLY_MSPACES - -/* ------------------- Declarations of public routines ------------------- */ - -#ifndef USE_DL_PREFIX -#define dlcalloc calloc -#define dlfree free -#define dlmalloc malloc -#define dlmemalign memalign -#define dlposix_memalign posix_memalign -#define dlrealloc realloc -#define dlrealloc_in_place realloc_in_place -#define dlvalloc valloc -#define dlpvalloc pvalloc -#define dlmallinfo mallinfo -#define dlmallopt mallopt -#define dlmalloc_trim malloc_trim -#define dlmalloc_stats malloc_stats -// #define dlmalloc_usable_size malloc_usable_size -#define dlmalloc_footprint malloc_footprint -#define dlmalloc_max_footprint malloc_max_footprint -#define dlmalloc_footprint_limit malloc_footprint_limit -#define dlmalloc_set_footprint_limit malloc_set_footprint_limit -#define dlmalloc_inspect_all malloc_inspect_all -#define dlindependent_calloc independent_calloc -#define dlindependent_comalloc independent_comalloc -#define dlbulk_free bulk_free -#endif /* USE_DL_PREFIX */ - -/* - malloc(size_t n) - Returns a pointer to a newly allocated chunk of at least n bytes, or - null if no space is available, in which case errno is set to ENOMEM - on ANSI C systems. - - If n is zero, malloc returns a minimum-sized chunk. (The minimum - size is 16 bytes on most 32bit systems, and 32 bytes on 64bit - systems.) Note that size_t is an unsigned type, so calls with - arguments that would be negative if signed are interpreted as - requests for huge amounts of space, which will often fail. The - maximum supported value of n differs across systems, but is in all - cases less than the maximum representable value of a size_t. -*/ -DLMALLOC_EXPORT void* dlmalloc(size_t); - -/* - free(void* p) - Releases the chunk of memory pointed to by p, that had been previously - allocated using malloc or a related routine such as realloc. - It has no effect if p is null. If p was not malloced or already - freed, free(p) will by default cause the current program to abort. -*/ -DLMALLOC_EXPORT void dlfree(void*); - -/* - calloc(size_t n_elements, size_t element_size); - Returns a pointer to n_elements * element_size bytes, with all locations - set to zero. -*/ -DLMALLOC_EXPORT void* dlcalloc(size_t, size_t); - -/* - realloc(void* p, size_t n) - Returns a pointer to a chunk of size n that contains the same data - as does chunk p up to the minimum of (n, p's size) bytes, or null - if no space is available. - - The returned pointer may or may not be the same as p. The algorithm - prefers extending p in most cases when possible, otherwise it - employs the equivalent of a malloc-copy-free sequence. - - If p is null, realloc is equivalent to malloc. - - If space is not available, realloc returns null, errno is set (if on - ANSI) and p is NOT freed. - - if n is for fewer bytes than already held by p, the newly unused - space is lopped off and freed if possible. realloc with a size - argument of zero (re)allocates a minimum-sized chunk. - - The old unix realloc convention of allowing the last-free'd chunk - to be used as an argument to realloc is not supported. -*/ -DLMALLOC_EXPORT void* dlrealloc(void*, size_t); - -/* - realloc_in_place(void* p, size_t n) - Resizes the space allocated for p to size n, only if this can be - done without moving p (i.e., only if there is adjacent space - available if n is greater than p's current allocated size, or n is - less than or equal to p's size). This may be used instead of plain - realloc if an alternative allocation strategy is needed upon failure - to expand space; for example, reallocation of a buffer that must be - memory-aligned or cleared. You can use realloc_in_place to trigger - these alternatives only when needed. - - Returns p if successful; otherwise null. -*/ -DLMALLOC_EXPORT void* dlrealloc_in_place(void*, size_t); - -/* - memalign(size_t alignment, size_t n); - Returns a pointer to a newly allocated chunk of n bytes, aligned - in accord with the alignment argument. - - The alignment argument should be a power of two. If the argument is - not a power of two, the nearest greater power is used. - 8-byte alignment is guaranteed by normal malloc calls, so don't - bother calling memalign with an argument of 8 or less. - - Overreliance on memalign is a sure way to fragment space. -*/ -DLMALLOC_EXPORT void* dlmemalign(size_t, size_t); - -/* - int posix_memalign(void** pp, size_t alignment, size_t n); - Allocates a chunk of n bytes, aligned in accord with the alignment - argument. Differs from memalign only in that it (1) assigns the - allocated memory to *pp rather than returning it, (2) fails and - returns EINVAL if the alignment is not a power of two (3) fails and - returns ENOMEM if memory cannot be allocated. -*/ -DLMALLOC_EXPORT int dlposix_memalign(void**, size_t, size_t); - -/* - valloc(size_t n); - Equivalent to memalign(pagesize, n), where pagesize is the page - size of the system. If the pagesize is unknown, 4096 is used. -*/ -DLMALLOC_EXPORT void* dlvalloc(size_t); - -/* - mallopt(int parameter_number, int parameter_value) - Sets tunable parameters The format is to provide a - (parameter-number, parameter-value) pair. mallopt then sets the - corresponding parameter to the argument value if it can (i.e., so - long as the value is meaningful), and returns 1 if successful else - 0. To workaround the fact that mallopt is specified to use int, - not size_t parameters, the value -1 is specially treated as the - maximum unsigned size_t value. - - SVID/XPG/ANSI defines four standard param numbers for mallopt, - normally defined in malloc.h. None of these are use in this malloc, - so setting them has no effect. But this malloc also supports other - options in mallopt. See below for details. Briefly, supported - parameters are as follows (listed defaults are for "typical" - configurations). - - Symbol param # default allowed param values - M_TRIM_THRESHOLD -1 2*1024*1024 any (-1 disables) - M_GRANULARITY -2 page size any power of 2 >= page size - M_MMAP_THRESHOLD -3 256*1024 any (or 0 if no MMAP support) -*/ -DLMALLOC_EXPORT int dlmallopt(int, int); - -/* - malloc_footprint(); - Returns the number of bytes obtained from the system. The total - number of bytes allocated by malloc, realloc etc., is less than this - value. Unlike mallinfo, this function returns only a precomputed - result, so can be called frequently to monitor memory consumption. - Even if locks are otherwise defined, this function does not use them, - so results might not be up to date. -*/ -DLMALLOC_EXPORT size_t dlmalloc_footprint(void); - -/* - malloc_max_footprint(); - Returns the maximum number of bytes obtained from the system. This - value will be greater than current footprint if deallocated space - has been reclaimed by the system. The peak number of bytes allocated - by malloc, realloc etc., is less than this value. Unlike mallinfo, - this function returns only a precomputed result, so can be called - frequently to monitor memory consumption. Even if locks are - otherwise defined, this function does not use them, so results might - not be up to date. -*/ -DLMALLOC_EXPORT size_t dlmalloc_max_footprint(void); - -/* - malloc_footprint_limit(); - Returns the number of bytes that the heap is allowed to obtain from - the system, returning the last value returned by - malloc_set_footprint_limit, or the maximum size_t value if - never set. The returned value reflects a permission. There is no - guarantee that this number of bytes can actually be obtained from - the system. -*/ -DLMALLOC_EXPORT size_t dlmalloc_footprint_limit(); - -/* - malloc_set_footprint_limit(); - Sets the maximum number of bytes to obtain from the system, causing - failure returns from malloc and related functions upon attempts to - exceed this value. The argument value may be subject to page - rounding to an enforceable limit; this actual value is returned. - Using an argument of the maximum possible size_t effectively - disables checks. If the argument is less than or equal to the - current malloc_footprint, then all future allocations that require - additional system memory will fail. However, invocation cannot - retroactively deallocate existing used memory. -*/ -DLMALLOC_EXPORT size_t dlmalloc_set_footprint_limit(size_t bytes); - -#if MALLOC_INSPECT_ALL -/* - malloc_inspect_all(void(*handler)(void *start, - void *end, - size_t used_bytes, - void* callback_arg), - void* arg); - Traverses the heap and calls the given handler for each managed - region, skipping all bytes that are (or may be) used for bookkeeping - purposes. Traversal does not include include chunks that have been - directly memory mapped. Each reported region begins at the start - address, and continues up to but not including the end address. The - first used_bytes of the region contain allocated data. If - used_bytes is zero, the region is unallocated. The handler is - invoked with the given callback argument. If locks are defined, they - are held during the entire traversal. It is a bad idea to invoke - other malloc functions from within the handler. - - For example, to count the number of in-use chunks with size greater - than 1000, you could write: - static int count = 0; - void count_chunks(void* start, void* end, size_t used, void* arg) { - if (used >= 1000) ++count; - } - then: - malloc_inspect_all(count_chunks, NULL); - - malloc_inspect_all is compiled only if MALLOC_INSPECT_ALL is defined. -*/ -DLMALLOC_EXPORT void dlmalloc_inspect_all(void(*handler)(void*, void *, size_t, void*), - void* arg); - -#endif /* MALLOC_INSPECT_ALL */ - -#if !NO_MALLINFO -/* - mallinfo() - Returns (by copy) a struct containing various summary statistics: - - arena: current total non-mmapped bytes allocated from system - ordblks: the number of free chunks - smblks: always zero. - hblks: current number of mmapped regions - hblkhd: total bytes held in mmapped regions - usmblks: the maximum total allocated space. This will be greater - than current total if trimming has occurred. - fsmblks: always zero - uordblks: current total allocated space (normal or mmapped) - fordblks: total free space - keepcost: the maximum number of bytes that could ideally be released - back to system via malloc_trim. ("ideally" means that - it ignores page restrictions etc.) - - Because these fields are ints, but internal bookkeeping may - be kept as longs, the reported values may wrap around zero and - thus be inaccurate. -*/ -DLMALLOC_EXPORT struct mallinfo dlmallinfo(void); -#endif /* NO_MALLINFO */ - -/* - independent_calloc(size_t n_elements, size_t element_size, void* chunks[]); - - independent_calloc is similar to calloc, but instead of returning a - single cleared space, it returns an array of pointers to n_elements - independent elements that can hold contents of size elem_size, each - of which starts out cleared, and can be independently freed, - realloc'ed etc. The elements are guaranteed to be adjacently - allocated (this is not guaranteed to occur with multiple callocs or - mallocs), which may also improve cache locality in some - applications. - - The "chunks" argument is optional (i.e., may be null, which is - probably the most typical usage). If it is null, the returned array - is itself dynamically allocated and should also be freed when it is - no longer needed. Otherwise, the chunks array must be of at least - n_elements in length. It is filled in with the pointers to the - chunks. - - In either case, independent_calloc returns this pointer array, or - null if the allocation failed. If n_elements is zero and "chunks" - is null, it returns a chunk representing an array with zero elements - (which should be freed if not wanted). - - Each element must be freed when it is no longer needed. This can be - done all at once using bulk_free. - - independent_calloc simplifies and speeds up implementations of many - kinds of pools. It may also be useful when constructing large data - structures that initially have a fixed number of fixed-sized nodes, - but the number is not known at compile time, and some of the nodes - may later need to be freed. For example: - - struct Node { int item; struct Node* next; }; - - struct Node* build_list() { - struct Node** pool; - int n = read_number_of_nodes_needed(); - if (n <= 0) return 0; - pool = (struct Node**)(independent_calloc(n, sizeof(struct Node), 0); - if (pool == 0) die(); - // organize into a linked list... - struct Node* first = pool[0]; - for (i = 0; i < n-1; ++i) - pool[i]->next = pool[i+1]; - free(pool); // Can now free the array (or not, if it is needed later) - return first; - } -*/ -DLMALLOC_EXPORT void** dlindependent_calloc(size_t, size_t, void**); - -/* - independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]); - - independent_comalloc allocates, all at once, a set of n_elements - chunks with sizes indicated in the "sizes" array. It returns - an array of pointers to these elements, each of which can be - independently freed, realloc'ed etc. The elements are guaranteed to - be adjacently allocated (this is not guaranteed to occur with - multiple callocs or mallocs), which may also improve cache locality - in some applications. - - The "chunks" argument is optional (i.e., may be null). If it is null - the returned array is itself dynamically allocated and should also - be freed when it is no longer needed. Otherwise, the chunks array - must be of at least n_elements in length. It is filled in with the - pointers to the chunks. - - In either case, independent_comalloc returns this pointer array, or - null if the allocation failed. If n_elements is zero and chunks is - null, it returns a chunk representing an array with zero elements - (which should be freed if not wanted). - - Each element must be freed when it is no longer needed. This can be - done all at once using bulk_free. - - independent_comallac differs from independent_calloc in that each - element may have a different size, and also that it does not - automatically clear elements. - - independent_comalloc can be used to speed up allocation in cases - where several structs or objects must always be allocated at the - same time. For example: - - struct Head { ... } - struct Foot { ... } - - void send_message(char* msg) { - int msglen = strlen(msg); - size_t sizes[3] = { sizeof(struct Head), msglen, sizeof(struct Foot) }; - void* chunks[3]; - if (independent_comalloc(3, sizes, chunks) == 0) - die(); - struct Head* head = (struct Head*)(chunks[0]); - char* body = (char*)(chunks[1]); - struct Foot* foot = (struct Foot*)(chunks[2]); - // ... - } - - In general though, independent_comalloc is worth using only for - larger values of n_elements. For small values, you probably won't - detect enough difference from series of malloc calls to bother. - - Overuse of independent_comalloc can increase overall memory usage, - since it cannot reuse existing noncontiguous small chunks that - might be available for some of the elements. -*/ -DLMALLOC_EXPORT void** dlindependent_comalloc(size_t, size_t*, void**); - -/* - bulk_free(void* array[], size_t n_elements) - Frees and clears (sets to null) each non-null pointer in the given - array. This is likely to be faster than freeing them one-by-one. - If footers are used, pointers that have been allocated in different - mspaces are not freed or cleared, and the count of all such pointers - is returned. For large arrays of pointers with poor locality, it - may be worthwhile to sort this array before calling bulk_free. -*/ -DLMALLOC_EXPORT size_t dlbulk_free(void**, size_t n_elements); - -/* - pvalloc(size_t n); - Equivalent to valloc(minimum-page-that-holds(n)), that is, - round up n to nearest pagesize. - */ -DLMALLOC_EXPORT void* dlpvalloc(size_t); - -/* - malloc_trim(size_t pad); - - If possible, gives memory back to the system (via negative arguments - to sbrk) if there is unused memory at the `high' end of the malloc - pool or in unused MMAP segments. You can call this after freeing - large blocks of memory to potentially reduce the system-level memory - requirements of a program. However, it cannot guarantee to reduce - memory. Under some allocation patterns, some large free blocks of - memory will be locked between two used chunks, so they cannot be - given back to the system. - - The `pad' argument to malloc_trim represents the amount of free - trailing space to leave untrimmed. If this argument is zero, only - the minimum amount of memory to maintain internal data structures - will be left. Non-zero arguments can be supplied to maintain enough - trailing space to service future expected allocations without having - to re-obtain memory from the system. - - Malloc_trim returns 1 if it actually released any memory, else 0. -*/ -DLMALLOC_EXPORT int dlmalloc_trim(size_t); - -/* - malloc_stats(); - Prints on stderr the amount of space obtained from the system (both - via sbrk and mmap), the maximum amount (which may be more than - current if malloc_trim and/or munmap got called), and the current - number of bytes allocated via malloc (or realloc, etc) but not yet - freed. Note that this is the number of bytes allocated, not the - number requested. It will be larger than the number requested - because of alignment and bookkeeping overhead. Because it includes - alignment wastage as being in use, this figure may be greater than - zero even when no user-level chunks are allocated. - - The reported current and maximum system memory can be inaccurate if - a program makes other calls to system memory allocation functions - (normally sbrk) outside of malloc. - - malloc_stats prints only the most commonly interesting statistics. - More information can be obtained by calling mallinfo. -*/ -DLMALLOC_EXPORT void dlmalloc_stats(void); - -/* - malloc_usable_size(void* p); - - Returns the number of bytes you can actually use in - an allocated chunk, which may be more than you requested (although - often not) due to alignment and minimum size constraints. - You can use this many bytes without worrying about - overwriting other allocated objects. This is not a particularly great - programming practice. malloc_usable_size can be more useful in - debugging and assertions, for example: - - p = malloc(n); - assert(malloc_usable_size(p) >= 256); -*/ -size_t dlmalloc_usable_size(void*); - -#endif /* ONLY_MSPACES */ - -#if MSPACES - -/* - mspace is an opaque type representing an independent - region of space that supports mspace_malloc, etc. -*/ -typedef void* mspace; - -/* - create_mspace creates and returns a new independent space with the - given initial capacity, or, if 0, the default granularity size. It - returns null if there is no system memory available to create the - space. If argument locked is non-zero, the space uses a separate - lock to control access. The capacity of the space will grow - dynamically as needed to service mspace_malloc requests. You can - control the sizes of incremental increases of this space by - compiling with a different DEFAULT_GRANULARITY or dynamically - setting with mallopt(M_GRANULARITY, value). -*/ -DLMALLOC_EXPORT mspace create_mspace(size_t capacity, int locked); - -/* - destroy_mspace destroys the given space, and attempts to return all - of its memory back to the system, returning the total number of - bytes freed. After destruction, the results of access to all memory - used by the space become undefined. -*/ -DLMALLOC_EXPORT size_t destroy_mspace(mspace msp); - -/* - create_mspace_with_base uses the memory supplied as the initial base - of a new mspace. Part (less than 128*sizeof(size_t) bytes) of this - space is used for bookkeeping, so the capacity must be at least this - large. (Otherwise 0 is returned.) When this initial space is - exhausted, additional memory will be obtained from the system. - Destroying this space will deallocate all additionally allocated - space (if possible) but not the initial base. -*/ -DLMALLOC_EXPORT mspace create_mspace_with_base(void* base, size_t capacity, int locked); - -/* - mspace_track_large_chunks controls whether requests for large chunks - are allocated in their own untracked mmapped regions, separate from - others in this mspace. By default large chunks are not tracked, - which reduces fragmentation. However, such chunks are not - necessarily released to the system upon destroy_mspace. Enabling - tracking by setting to true may increase fragmentation, but avoids - leakage when relying on destroy_mspace to release all memory - allocated using this space. The function returns the previous - setting. -*/ -DLMALLOC_EXPORT int mspace_track_large_chunks(mspace msp, int enable); - - -/* - mspace_malloc behaves as malloc, but operates within - the given space. -*/ -DLMALLOC_EXPORT void* mspace_malloc(mspace msp, size_t bytes); - -/* - mspace_free behaves as free, but operates within - the given space. - - If compiled with FOOTERS==1, mspace_free is not actually needed. - free may be called instead of mspace_free because freed chunks from - any space are handled by their originating spaces. -*/ -DLMALLOC_EXPORT void mspace_free(mspace msp, void* mem); - -/* - mspace_realloc behaves as realloc, but operates within - the given space. - - If compiled with FOOTERS==1, mspace_realloc is not actually - needed. realloc may be called instead of mspace_realloc because - realloced chunks from any space are handled by their originating - spaces. -*/ -DLMALLOC_EXPORT void* mspace_realloc(mspace msp, void* mem, size_t newsize); - -/* - mspace_calloc behaves as calloc, but operates within - the given space. -*/ -DLMALLOC_EXPORT void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size); - -/* - mspace_memalign behaves as memalign, but operates within - the given space. -*/ -DLMALLOC_EXPORT void* mspace_memalign(mspace msp, size_t alignment, size_t bytes); - -/* - mspace_independent_calloc behaves as independent_calloc, but - operates within the given space. -*/ -DLMALLOC_EXPORT void** mspace_independent_calloc(mspace msp, size_t n_elements, - size_t elem_size, void* chunks[]); - -/* - mspace_independent_comalloc behaves as independent_comalloc, but - operates within the given space. -*/ -DLMALLOC_EXPORT void** mspace_independent_comalloc(mspace msp, size_t n_elements, - size_t sizes[], void* chunks[]); - -/* - mspace_footprint() returns the number of bytes obtained from the - system for this space. -*/ -DLMALLOC_EXPORT size_t mspace_footprint(mspace msp); - -/* - mspace_max_footprint() returns the peak number of bytes obtained from the - system for this space. -*/ -DLMALLOC_EXPORT size_t mspace_max_footprint(mspace msp); - - -#if !NO_MALLINFO -/* - mspace_mallinfo behaves as mallinfo, but reports properties of - the given space. -*/ -DLMALLOC_EXPORT struct mallinfo mspace_mallinfo(mspace msp); -#endif /* NO_MALLINFO */ - -/* - malloc_usable_size(void* p) behaves the same as malloc_usable_size; -*/ -DLMALLOC_EXPORT size_t mspace_usable_size(const void* mem); - -/* - mspace_malloc_stats behaves as malloc_stats, but reports - properties of the given space. -*/ -DLMALLOC_EXPORT void mspace_malloc_stats(mspace msp); - -/* - mspace_trim behaves as malloc_trim, but - operates within the given space. -*/ -DLMALLOC_EXPORT int mspace_trim(mspace msp, size_t pad); - -/* - An alias for mallopt. -*/ -DLMALLOC_EXPORT int mspace_mallopt(int, int); - -#endif /* MSPACES */ - -#ifdef __cplusplus -} /* end of extern "C" */ -#endif /* __cplusplus */ - -/* - ======================================================================== - To make a fully customizable malloc.h header file, cut everything - above this line, put into file malloc.h, edit to suit, and #include it - on the next line, as well as in programs that use this malloc. - ======================================================================== -*/ - -/* #include "malloc.h" */ - -/*------------------------------ internal #includes ---------------------- */ - -#ifdef _MSC_VER -#pragma warning( disable : 4146 ) /* no "unsigned" warnings */ -#endif /* _MSC_VER */ -#if !NO_MALLOC_STATS -#include /* for printing in malloc_stats */ -#endif /* NO_MALLOC_STATS */ -#ifndef LACKS_ERRNO_H -#include /* for MALLOC_FAILURE_ACTION */ -#endif /* LACKS_ERRNO_H */ -#ifdef DEBUG -#if ABORT_ON_ASSERT_FAILURE -#undef assert -#define assert(x) if(!(x)) ABORT -#else /* ABORT_ON_ASSERT_FAILURE */ -#include -#endif /* ABORT_ON_ASSERT_FAILURE */ -#else /* DEBUG */ -#ifndef assert -#define assert(x) -#endif -#define DEBUG 0 -#endif /* DEBUG */ -#if !defined(WIN32) && !defined(LACKS_TIME_H) -#include /* for magic initialization */ -#endif /* WIN32 */ -#ifndef LACKS_STDLIB_H -#include /* for abort() */ -#endif /* LACKS_STDLIB_H */ -#ifndef LACKS_STRING_H -#include /* for memset etc */ -#endif /* LACKS_STRING_H */ -#if USE_BUILTIN_FFS -#ifndef LACKS_STRINGS_H -#include /* for ffs */ -#endif /* LACKS_STRINGS_H */ -#endif /* USE_BUILTIN_FFS */ -#if HAVE_MMAP -#ifndef LACKS_SYS_MMAN_H -/* On some versions of linux, mremap decl in mman.h needs __USE_GNU set */ -#if (defined(linux) && !defined(__USE_GNU)) -#define __USE_GNU 1 -#include /* for mmap */ -#undef __USE_GNU -#else -#include /* for mmap */ -#endif /* linux */ -#endif /* LACKS_SYS_MMAN_H */ -#ifndef LACKS_FCNTL_H -#include -#endif /* LACKS_FCNTL_H */ -#endif /* HAVE_MMAP */ -#ifndef LACKS_UNISTD_H -#include /* for sbrk, sysconf */ -#else /* LACKS_UNISTD_H */ -#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) -extern void* sbrk(ptrdiff_t); -#endif /* FreeBSD etc */ -#endif /* LACKS_UNISTD_H */ - -/* Declarations for locking */ -#if USE_LOCKS -#ifndef WIN32 -#if defined (__SVR4) && defined (__sun) /* solaris */ -#include -#elif !defined(LACKS_SCHED_H) -#include -#endif /* solaris or LACKS_SCHED_H */ -#if (defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0) || !USE_SPIN_LOCKS -#include -#endif /* USE_RECURSIVE_LOCKS ... */ -#elif defined(_MSC_VER) -#ifndef _M_AMD64 -/* These are already defined on AMD64 builds */ -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ -LONG __cdecl _InterlockedCompareExchange(LONG volatile *Dest, LONG Exchange, LONG Comp); -LONG __cdecl _InterlockedExchange(LONG volatile *Target, LONG Value); -#ifdef __cplusplus -} -#endif /* __cplusplus */ -#endif /* _M_AMD64 */ -#pragma intrinsic (_InterlockedCompareExchange) -#pragma intrinsic (_InterlockedExchange) -#define interlockedcompareexchange _InterlockedCompareExchange -#define interlockedexchange _InterlockedExchange -#elif defined(WIN32) && defined(__GNUC__) -#define interlockedcompareexchange(a, b, c) __sync_val_compare_and_swap(a, c, b) -#define interlockedexchange __sync_lock_test_and_set -#endif /* Win32 */ -#else /* USE_LOCKS */ -#endif /* USE_LOCKS */ - -#ifndef LOCK_AT_FORK -#define LOCK_AT_FORK 0 -#endif - -/* Declarations for bit scanning on win32 */ -#if defined(_MSC_VER) && _MSC_VER>=1300 -#ifndef BitScanForward /* Try to avoid pulling in WinNT.h */ -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ -unsigned char _BitScanForward(unsigned long *index, unsigned long mask); -unsigned char _BitScanReverse(unsigned long *index, unsigned long mask); -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#define BitScanForward _BitScanForward -#define BitScanReverse _BitScanReverse -#pragma intrinsic(_BitScanForward) -#pragma intrinsic(_BitScanReverse) -#endif /* BitScanForward */ -#endif /* defined(_MSC_VER) && _MSC_VER>=1300 */ - -#ifndef WIN32 -#ifndef malloc_getpagesize -# ifdef _SC_PAGESIZE /* some SVR4 systems omit an underscore */ -# ifndef _SC_PAGE_SIZE -# define _SC_PAGE_SIZE _SC_PAGESIZE -# endif -# endif -# ifdef _SC_PAGE_SIZE -# define malloc_getpagesize sysconf(_SC_PAGE_SIZE) -# else -# if defined(BSD) || defined(DGUX) || defined(HAVE_GETPAGESIZE) - extern size_t getpagesize(); -# define malloc_getpagesize getpagesize() -# else -# ifdef WIN32 /* use supplied emulation of getpagesize */ -# define malloc_getpagesize getpagesize() -# else -# ifndef LACKS_SYS_PARAM_H -# include -# endif -# ifdef EXEC_PAGESIZE -# define malloc_getpagesize EXEC_PAGESIZE -# else -# ifdef NBPG -# ifndef CLSIZE -# define malloc_getpagesize NBPG -# else -# define malloc_getpagesize (NBPG * CLSIZE) -# endif -# else -# ifdef NBPC -# define malloc_getpagesize NBPC -# else -# ifdef PAGESIZE -# define malloc_getpagesize PAGESIZE -# else /* just guess */ -# define malloc_getpagesize ((size_t)4096U) -# endif -# endif -# endif -# endif -# endif -# endif -# endif -#endif -#endif - -/* ------------------- size_t and alignment properties -------------------- */ - -/* The byte and bit size of a size_t */ -#define SIZE_T_SIZE (sizeof(size_t)) -#define SIZE_T_BITSIZE (sizeof(size_t) << 3) - -/* Some constants coerced to size_t */ -/* Annoying but necessary to avoid errors on some platforms */ -#define SIZE_T_ZERO ((size_t)0) -#define SIZE_T_ONE ((size_t)1) -#define SIZE_T_TWO ((size_t)2) -#define SIZE_T_FOUR ((size_t)4) -#define TWO_SIZE_T_SIZES (SIZE_T_SIZE<<1) -#define FOUR_SIZE_T_SIZES (SIZE_T_SIZE<<2) -#define SIX_SIZE_T_SIZES (FOUR_SIZE_T_SIZES+TWO_SIZE_T_SIZES) -#define HALF_MAX_SIZE_T (MAX_SIZE_T / 2U) - -/* The bit mask value corresponding to MALLOC_ALIGNMENT */ -#define CHUNK_ALIGN_MASK (MALLOC_ALIGNMENT - SIZE_T_ONE) - -/* True if address a has acceptable alignment */ -#define is_aligned(A) (((size_t)((A)) & (CHUNK_ALIGN_MASK)) == 0) - -/* the number of bytes to offset an address to align it */ -#define align_offset(A)\ - ((((size_t)(A) & CHUNK_ALIGN_MASK) == 0)? 0 :\ - ((MALLOC_ALIGNMENT - ((size_t)(A) & CHUNK_ALIGN_MASK)) & CHUNK_ALIGN_MASK)) - -/* -------------------------- MMAP preliminaries ------------------------- */ - -/* - If HAVE_MORECORE or HAVE_MMAP are false, we just define calls and - checks to fail so compiler optimizer can delete code rather than - using so many "#if"s. -*/ - - -/* MORECORE and MMAP must return MFAIL on failure */ -#define MFAIL ((void*)(MAX_SIZE_T)) -#define CMFAIL ((char*)(MFAIL)) /* defined for convenience */ - -#if HAVE_MMAP - -#ifndef WIN32 -#define MUNMAP_DEFAULT(a, s) munmap((a), (s)) -#define MMAP_PROT (PROT_READ|PROT_WRITE) -#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) -#define MAP_ANONYMOUS MAP_ANON -#endif /* MAP_ANON */ -#ifdef MAP_ANONYMOUS -#define MMAP_FLAGS (MAP_PRIVATE|MAP_ANONYMOUS) -#define MMAP_DEFAULT(s) mmap(0, (s), MMAP_PROT, MMAP_FLAGS, -1, 0) -#else /* MAP_ANONYMOUS */ -/* - Nearly all versions of mmap support MAP_ANONYMOUS, so the following - is unlikely to be needed, but is supplied just in case. -*/ -#define MMAP_FLAGS (MAP_PRIVATE) -static int dev_zero_fd = -1; /* Cached file descriptor for /dev/zero. */ -#define MMAP_DEFAULT(s) ((dev_zero_fd < 0) ? \ - (dev_zero_fd = open("/dev/zero", O_RDWR), \ - mmap(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0)) : \ - mmap(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0)) -#endif /* MAP_ANONYMOUS */ - -#define DIRECT_MMAP_DEFAULT(s) MMAP_DEFAULT(s) - -#else /* WIN32 */ - -/* Win32 MMAP via VirtualAlloc */ -static FORCEINLINE void* win32mmap(size_t size) { - void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); - return (ptr != 0)? ptr: MFAIL; -} - -/* For direct MMAP, use MEM_TOP_DOWN to minimize interference */ -static FORCEINLINE void* win32direct_mmap(size_t size) { - void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN, - PAGE_READWRITE); - return (ptr != 0)? ptr: MFAIL; -} - -/* This function supports releasing coalesed segments */ -static FORCEINLINE int win32munmap(void* ptr, size_t size) { - MEMORY_BASIC_INFORMATION minfo; - char* cptr = (char*)ptr; - while (size) { - if (VirtualQuery(cptr, &minfo, sizeof(minfo)) == 0) - return -1; - if (minfo.BaseAddress != cptr || minfo.AllocationBase != cptr || - minfo.State != MEM_COMMIT || minfo.RegionSize > size) - return -1; - if (VirtualFree(cptr, 0, MEM_RELEASE) == 0) - return -1; - cptr += minfo.RegionSize; - size -= minfo.RegionSize; - } - return 0; -} - -#define MMAP_DEFAULT(s) win32mmap(s) -#define MUNMAP_DEFAULT(a, s) win32munmap((a), (s)) -#define DIRECT_MMAP_DEFAULT(s) win32direct_mmap(s) -#endif /* WIN32 */ -#endif /* HAVE_MMAP */ - -#if HAVE_MREMAP -#ifndef WIN32 -#define MREMAP_DEFAULT(addr, osz, nsz, mv) mremap((addr), (osz), (nsz), (mv)) -#endif /* WIN32 */ -#endif /* HAVE_MREMAP */ - -/** - * Define CALL_MORECORE - */ -#if HAVE_MORECORE - #ifdef MORECORE - #define CALL_MORECORE(S) MORECORE(S) - #else /* MORECORE */ - #define CALL_MORECORE(S) MORECORE_DEFAULT(S) - #endif /* MORECORE */ -#else /* HAVE_MORECORE */ - #define CALL_MORECORE(S) MFAIL -#endif /* HAVE_MORECORE */ - -/** - * Define CALL_MMAP/CALL_MUNMAP/CALL_DIRECT_MMAP - */ -#if HAVE_MMAP - #define USE_MMAP_BIT (SIZE_T_ONE) - - #ifdef MMAP - #define CALL_MMAP(s) MMAP(s) - #else /* MMAP */ - #define CALL_MMAP(s) MMAP_DEFAULT(s) - #endif /* MMAP */ - #ifdef MUNMAP - #define CALL_MUNMAP(a, s) MUNMAP((a), (s)) - #else /* MUNMAP */ - #define CALL_MUNMAP(a, s) MUNMAP_DEFAULT((a), (s)) - #endif /* MUNMAP */ - #ifdef DIRECT_MMAP - #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s) - #else /* DIRECT_MMAP */ - #define CALL_DIRECT_MMAP(s) DIRECT_MMAP_DEFAULT(s) - #endif /* DIRECT_MMAP */ -#else /* HAVE_MMAP */ - #define USE_MMAP_BIT (SIZE_T_ZERO) - - #define MMAP(s) MFAIL - #define MUNMAP(a, s) (-1) - #define DIRECT_MMAP(s) MFAIL - #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s) - #define CALL_MMAP(s) MMAP(s) - #define CALL_MUNMAP(a, s) MUNMAP((a), (s)) -#endif /* HAVE_MMAP */ - -/** - * Define CALL_MREMAP - */ -#if HAVE_MMAP && HAVE_MREMAP - #ifdef MREMAP - #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP((addr), (osz), (nsz), (mv)) - #else /* MREMAP */ - #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP_DEFAULT((addr), (osz), (nsz), (mv)) - #endif /* MREMAP */ -#else /* HAVE_MMAP && HAVE_MREMAP */ - #define CALL_MREMAP(addr, osz, nsz, mv) MFAIL -#endif /* HAVE_MMAP && HAVE_MREMAP */ - -/* mstate bit set if continguous morecore disabled or failed */ -#define USE_NONCONTIGUOUS_BIT (4U) - -/* segment bit set in create_mspace_with_base */ -#define EXTERN_BIT (8U) - - -/* --------------------------- Lock preliminaries ------------------------ */ - -/* - When locks are defined, there is one global lock, plus - one per-mspace lock. - - The global lock_ensures that mparams.magic and other unique - mparams values are initialized only once. It also protects - sequences of calls to MORECORE. In many cases sys_alloc requires - two calls, that should not be interleaved with calls by other - threads. This does not protect against direct calls to MORECORE - by other threads not using this lock, so there is still code to - cope the best we can on interference. - - Per-mspace locks surround calls to malloc, free, etc. - By default, locks are simple non-reentrant mutexes. - - Because lock-protected regions generally have bounded times, it is - OK to use the supplied simple spinlocks. Spinlocks are likely to - improve performance for lightly contended applications, but worsen - performance under heavy contention. - - If USE_LOCKS is > 1, the definitions of lock routines here are - bypassed, in which case you will need to define the type MLOCK_T, - and at least INITIAL_LOCK, DESTROY_LOCK, ACQUIRE_LOCK, RELEASE_LOCK - and TRY_LOCK. You must also declare a - static MLOCK_T malloc_global_mutex = { initialization values };. - -*/ - -#if !USE_LOCKS -#define USE_LOCK_BIT (0U) -#define INITIAL_LOCK(l) (0) -#define DESTROY_LOCK(l) (0) -#define ACQUIRE_MALLOC_GLOBAL_LOCK() -#define RELEASE_MALLOC_GLOBAL_LOCK() - -#else -#if USE_LOCKS > 1 -/* ----------------------- User-defined locks ------------------------ */ -/* Define your own lock implementation here */ -/* #define INITIAL_LOCK(lk) ... */ -/* #define DESTROY_LOCK(lk) ... */ -/* #define ACQUIRE_LOCK(lk) ... */ -/* #define RELEASE_LOCK(lk) ... */ -/* #define TRY_LOCK(lk) ... */ -/* static MLOCK_T malloc_global_mutex = ... */ - -#elif USE_SPIN_LOCKS - -/* First, define CAS_LOCK and CLEAR_LOCK on ints */ -/* Note CAS_LOCK defined to return 0 on success */ - -#if defined(__GNUC__)&& (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)) -#define CAS_LOCK(sl) __sync_lock_test_and_set(sl, 1) -#define CLEAR_LOCK(sl) __sync_lock_release(sl) - -#elif (defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))) -/* Custom spin locks for older gcc on x86 */ -static FORCEINLINE int x86_cas_lock(int *sl) { - int ret; - int val = 1; - int cmp = 0; - __asm__ __volatile__ ("lock; cmpxchgl %1, %2" - : "=a" (ret) - : "r" (val), "m" (*(sl)), "0"(cmp) - : "memory", "cc"); - return ret; -} - -static FORCEINLINE void x86_clear_lock(int* sl) { - assert(*sl != 0); - int prev = 0; - int ret; - __asm__ __volatile__ ("lock; xchgl %0, %1" - : "=r" (ret) - : "m" (*(sl)), "0"(prev) - : "memory"); -} - -#define CAS_LOCK(sl) x86_cas_lock(sl) -#define CLEAR_LOCK(sl) x86_clear_lock(sl) - -#else /* Win32 MSC */ -#define CAS_LOCK(sl) interlockedexchange(sl, (LONG)1) -#define CLEAR_LOCK(sl) interlockedexchange (sl, (LONG)0) - -#endif /* ... gcc spins locks ... */ - -/* How to yield for a spin lock */ -#define SPINS_PER_YIELD 63 -#if defined(_MSC_VER) -#define SLEEP_EX_DURATION 50 /* delay for yield/sleep */ -#define SPIN_LOCK_YIELD SleepEx(SLEEP_EX_DURATION, FALSE) -#elif defined (__SVR4) && defined (__sun) /* solaris */ -#define SPIN_LOCK_YIELD thr_yield(); -#elif !defined(LACKS_SCHED_H) -#define SPIN_LOCK_YIELD sched_yield(); -#else -#define SPIN_LOCK_YIELD -#endif /* ... yield ... */ - -#if !defined(USE_RECURSIVE_LOCKS) || USE_RECURSIVE_LOCKS == 0 -/* Plain spin locks use single word (embedded in malloc_states) */ -static int spin_acquire_lock(int *sl) { - int spins = 0; - while (*(volatile int *)sl != 0 || CAS_LOCK(sl)) { - if ((++spins & SPINS_PER_YIELD) == 0) { - SPIN_LOCK_YIELD; - } - } - return 0; -} - -#define MLOCK_T int -#define TRY_LOCK(sl) !CAS_LOCK(sl) -#define RELEASE_LOCK(sl) CLEAR_LOCK(sl) -#define ACQUIRE_LOCK(sl) (CAS_LOCK(sl)? spin_acquire_lock(sl) : 0) -#define INITIAL_LOCK(sl) (*sl = 0) -#define DESTROY_LOCK(sl) (0) -static MLOCK_T malloc_global_mutex = 0; - -#else /* USE_RECURSIVE_LOCKS */ -/* types for lock owners */ -#ifdef WIN32 -#define THREAD_ID_T DWORD -#define CURRENT_THREAD GetCurrentThreadId() -#define EQ_OWNER(X,Y) ((X) == (Y)) -#else -/* - Note: the following assume that pthread_t is a type that can be - initialized to (casted) zero. If this is not the case, you will need to - somehow redefine these or not use spin locks. -*/ -#define THREAD_ID_T pthread_t -#define CURRENT_THREAD pthread_self() -#define EQ_OWNER(X,Y) pthread_equal(X, Y) -#endif - -struct malloc_recursive_lock { - int sl; - unsigned int c; - THREAD_ID_T threadid; -}; - -#define MLOCK_T struct malloc_recursive_lock -static MLOCK_T malloc_global_mutex = { 0, 0, (THREAD_ID_T)0}; - -static FORCEINLINE void recursive_release_lock(MLOCK_T *lk) { - assert(lk->sl != 0); - if (--lk->c == 0) { - CLEAR_LOCK(&lk->sl); - } -} - -static FORCEINLINE int recursive_acquire_lock(MLOCK_T *lk) { - THREAD_ID_T mythreadid = CURRENT_THREAD; - int spins = 0; - for (;;) { - if (*((volatile int *)(&lk->sl)) == 0) { - if (!CAS_LOCK(&lk->sl)) { - lk->threadid = mythreadid; - lk->c = 1; - return 0; - } - } - else if (EQ_OWNER(lk->threadid, mythreadid)) { - ++lk->c; - return 0; - } - if ((++spins & SPINS_PER_YIELD) == 0) { - SPIN_LOCK_YIELD; - } - } -} - -static FORCEINLINE int recursive_try_lock(MLOCK_T *lk) { - THREAD_ID_T mythreadid = CURRENT_THREAD; - if (*((volatile int *)(&lk->sl)) == 0) { - if (!CAS_LOCK(&lk->sl)) { - lk->threadid = mythreadid; - lk->c = 1; - return 1; - } - } - else if (EQ_OWNER(lk->threadid, mythreadid)) { - ++lk->c; - return 1; - } - return 0; -} - -#define RELEASE_LOCK(lk) recursive_release_lock(lk) -#define TRY_LOCK(lk) recursive_try_lock(lk) -#define ACQUIRE_LOCK(lk) recursive_acquire_lock(lk) -#define INITIAL_LOCK(lk) ((lk)->threadid = (THREAD_ID_T)0, (lk)->sl = 0, (lk)->c = 0) -#define DESTROY_LOCK(lk) (0) -#endif /* USE_RECURSIVE_LOCKS */ - -#elif defined(WIN32) /* Win32 critical sections */ -#define MLOCK_T CRITICAL_SECTION -#define ACQUIRE_LOCK(lk) (EnterCriticalSection(lk), 0) -#define RELEASE_LOCK(lk) LeaveCriticalSection(lk) -#define TRY_LOCK(lk) TryEnterCriticalSection(lk) -#define INITIAL_LOCK(lk) (!InitializeCriticalSectionAndSpinCount((lk), 0x80000000|4000)) -#define DESTROY_LOCK(lk) (DeleteCriticalSection(lk), 0) -#define NEED_GLOBAL_LOCK_INIT - -static MLOCK_T malloc_global_mutex; -static volatile LONG malloc_global_mutex_status; - -/* Use spin loop to initialize global lock */ -static void init_malloc_global_mutex() { - for (;;) { - long stat = malloc_global_mutex_status; - if (stat > 0) - return; - /* transition to < 0 while initializing, then to > 0) */ - if (stat == 0 && - interlockedcompareexchange(&malloc_global_mutex_status, (LONG)-1, (LONG)0) == 0) { - InitializeCriticalSection(&malloc_global_mutex); - interlockedexchange(&malloc_global_mutex_status, (LONG)1); - return; - } - SleepEx(0, FALSE); - } -} - -#else /* pthreads-based locks */ -#define MLOCK_T pthread_mutex_t -#define ACQUIRE_LOCK(lk) pthread_mutex_lock(lk) -#define RELEASE_LOCK(lk) pthread_mutex_unlock(lk) -#define TRY_LOCK(lk) (!pthread_mutex_trylock(lk)) -#define INITIAL_LOCK(lk) pthread_init_lock(lk) -#define DESTROY_LOCK(lk) pthread_mutex_destroy(lk) - -#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0 && defined(linux) && !defined(PTHREAD_MUTEX_RECURSIVE) -/* Cope with old-style linux recursive lock initialization by adding */ -/* skipped internal declaration from pthread.h */ -extern int pthread_mutexattr_setkind_np __P ((pthread_mutexattr_t *__attr, - int __kind)); -#define PTHREAD_MUTEX_RECURSIVE PTHREAD_MUTEX_RECURSIVE_NP -#define pthread_mutexattr_settype(x,y) pthread_mutexattr_setkind_np(x,y) -#endif /* USE_RECURSIVE_LOCKS ... */ - -static MLOCK_T malloc_global_mutex = PTHREAD_MUTEX_INITIALIZER; - -static int pthread_init_lock (MLOCK_T *lk) { - pthread_mutexattr_t attr; - if (pthread_mutexattr_init(&attr)) return 1; -#if defined(USE_RECURSIVE_LOCKS) && USE_RECURSIVE_LOCKS != 0 - if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) return 1; -#endif - if (pthread_mutex_init(lk, &attr)) return 1; - if (pthread_mutexattr_destroy(&attr)) return 1; - return 0; -} - -#endif /* ... lock types ... */ - -/* Common code for all lock types */ -#define USE_LOCK_BIT (2U) - -#ifndef ACQUIRE_MALLOC_GLOBAL_LOCK -#define ACQUIRE_MALLOC_GLOBAL_LOCK() ACQUIRE_LOCK(&malloc_global_mutex); -#endif - -#ifndef RELEASE_MALLOC_GLOBAL_LOCK -#define RELEASE_MALLOC_GLOBAL_LOCK() RELEASE_LOCK(&malloc_global_mutex); -#endif - -#endif /* USE_LOCKS */ - -/* ----------------------- Chunk representations ------------------------ */ - -/* - (The following includes lightly edited explanations by Colin Plumb.) - - The malloc_chunk declaration below is misleading (but accurate and - necessary). It declares a "view" into memory allowing access to - necessary fields at known offsets from a given base. - - Chunks of memory are maintained using a `boundary tag' method as - originally described by Knuth. (See the paper by Paul Wilson - ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a survey of such - techniques.) Sizes of free chunks are stored both in the front of - each chunk and at the end. This makes consolidating fragmented - chunks into bigger chunks fast. The head fields also hold bits - representing whether chunks are free or in use. - - Here are some pictures to make it clearer. They are "exploded" to - show that the state of a chunk can be thought of as extending from - the high 31 bits of the head field of its header through the - prev_foot and PINUSE_BIT bit of the following chunk header. - - A chunk that's in use looks like: - - chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Size of previous chunk (if P = 0) | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P| - | Size of this chunk 1| +-+ - mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | | - +- -+ - | | - +- -+ - | : - +- size - sizeof(size_t) available payload bytes -+ - : | - chunk-> +- -+ - | | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |1| - | Size of next chunk (may or may not be in use) | +-+ - mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - And if it's free, it looks like this: - - chunk-> +- -+ - | User payload (must be in use, or we would have merged!) | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P| - | Size of this chunk 0| +-+ - mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Next pointer | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Prev pointer | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | : - +- size - sizeof(struct chunk) unused bytes -+ - : | - chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Size of this chunk | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |0| - | Size of next chunk (must be in use, or we would have merged)| +-+ - mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | : - +- User payload -+ - : | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - |0| - +-+ - Note that since we always merge adjacent free chunks, the chunks - adjacent to a free chunk must be in use. - - Given a pointer to a chunk (which can be derived trivially from the - payload pointer) we can, in O(1) time, find out whether the adjacent - chunks are free, and if so, unlink them from the lists that they - are on and merge them with the current chunk. - - Chunks always begin on even word boundaries, so the mem portion - (which is returned to the user) is also on an even word boundary, and - thus at least double-word aligned. - - The P (PINUSE_BIT) bit, stored in the unused low-order bit of the - chunk size (which is always a multiple of two words), is an in-use - bit for the *previous* chunk. If that bit is *clear*, then the - word before the current chunk size contains the previous chunk - size, and can be used to find the front of the previous chunk. - The very first chunk allocated always has this bit set, preventing - access to non-existent (or non-owned) memory. If pinuse is set for - any given chunk, then you CANNOT determine the size of the - previous chunk, and might even get a memory addressing fault when - trying to do so. - - The C (CINUSE_BIT) bit, stored in the unused second-lowest bit of - the chunk size redundantly records whether the current chunk is - inuse (unless the chunk is mmapped). This redundancy enables usage - checks within free and realloc, and reduces indirection when freeing - and consolidating chunks. - - Each freshly allocated chunk must have both cinuse and pinuse set. - That is, each allocated chunk borders either a previously allocated - and still in-use chunk, or the base of its memory arena. This is - ensured by making all allocations from the `lowest' part of any - found chunk. Further, no free chunk physically borders another one, - so each free chunk is known to be preceded and followed by either - inuse chunks or the ends of memory. - - Note that the `foot' of the current chunk is actually represented - as the prev_foot of the NEXT chunk. This makes it easier to - deal with alignments etc but can be very confusing when trying - to extend or adapt this code. - - The exceptions to all this are - - 1. The special chunk `top' is the top-most available chunk (i.e., - the one bordering the end of available memory). It is treated - specially. Top is never included in any bin, is used only if - no other chunk is available, and is released back to the - system if it is very large (see M_TRIM_THRESHOLD). In effect, - the top chunk is treated as larger (and thus less well - fitting) than any other available chunk. The top chunk - doesn't update its trailing size field since there is no next - contiguous chunk that would have to index off it. However, - space is still allocated for it (TOP_FOOT_SIZE) to enable - separation or merging when space is extended. - - 3. Chunks allocated via mmap, have both cinuse and pinuse bits - cleared in their head fields. Because they are allocated - one-by-one, each must carry its own prev_foot field, which is - also used to hold the offset this chunk has within its mmapped - region, which is needed to preserve alignment. Each mmapped - chunk is trailed by the first two fields of a fake next-chunk - for sake of usage checks. - -*/ - -struct malloc_chunk { - size_t prev_foot; /* Size of previous chunk (if free). */ - size_t head; /* Size and inuse bits. */ - struct malloc_chunk* fd; /* double links -- used only if free. */ - struct malloc_chunk* bk; -}; - -typedef struct malloc_chunk mchunk; -typedef struct malloc_chunk* mchunkptr; -typedef struct malloc_chunk* sbinptr; /* The type of bins of chunks */ -typedef unsigned int bindex_t; /* Described below */ -typedef unsigned int binmap_t; /* Described below */ -typedef unsigned int flag_t; /* The type of various bit flag sets */ - -/* ------------------- Chunks sizes and alignments ----------------------- */ - -#define MCHUNK_SIZE (sizeof(mchunk)) - -#if FOOTERS -#define CHUNK_OVERHEAD (TWO_SIZE_T_SIZES) -#else /* FOOTERS */ -#define CHUNK_OVERHEAD (SIZE_T_SIZE) -#endif /* FOOTERS */ - -/* MMapped chunks need a second word of overhead ... */ -#define MMAP_CHUNK_OVERHEAD (TWO_SIZE_T_SIZES) -/* ... and additional padding for fake next-chunk at foot */ -#define MMAP_FOOT_PAD (FOUR_SIZE_T_SIZES) - -/* The smallest size we can malloc is an aligned minimal chunk */ -#define MIN_CHUNK_SIZE\ - ((MCHUNK_SIZE + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK) - -/* conversion from malloc headers to user pointers, and back */ -#define chunk2mem(p) ((void*)((char*)(p) + TWO_SIZE_T_SIZES)) -#define mem2chunk(mem) ((mchunkptr)((char*)(mem) - TWO_SIZE_T_SIZES)) -/* chunk associated with aligned address A */ -#define align_as_chunk(A) (mchunkptr)((A) + align_offset(chunk2mem(A))) - -/* Bounds on request (not chunk) sizes. */ -#define MAX_REQUEST ((-MIN_CHUNK_SIZE) << 2) -#define MIN_REQUEST (MIN_CHUNK_SIZE - CHUNK_OVERHEAD - SIZE_T_ONE) - -/* pad request bytes into a usable size */ -#define pad_request(req) \ - (((req) + CHUNK_OVERHEAD + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK) - -/* pad request, checking for minimum (but not maximum) */ -#define request2size(req) \ - (((req) < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(req)) - - -/* ------------------ Operations on head and foot fields ----------------- */ - -/* - The head field of a chunk is or'ed with PINUSE_BIT when previous - adjacent chunk in use, and or'ed with CINUSE_BIT if this chunk is in - use, unless mmapped, in which case both bits are cleared. - - FLAG4_BIT is not used by this malloc, but might be useful in extensions. -*/ - -#define PINUSE_BIT (SIZE_T_ONE) -#define CINUSE_BIT (SIZE_T_TWO) -#define FLAG4_BIT (SIZE_T_FOUR) -#define INUSE_BITS (PINUSE_BIT|CINUSE_BIT) -#define FLAG_BITS (PINUSE_BIT|CINUSE_BIT|FLAG4_BIT) - -/* Head value for fenceposts */ -#define FENCEPOST_HEAD (INUSE_BITS|SIZE_T_SIZE) - -/* extraction of fields from head words */ -#define cinuse(p) ((p)->head & CINUSE_BIT) -#define pinuse(p) ((p)->head & PINUSE_BIT) -#define flag4inuse(p) ((p)->head & FLAG4_BIT) -#define is_inuse(p) (((p)->head & INUSE_BITS) != PINUSE_BIT) -#define is_mmapped(p) (((p)->head & INUSE_BITS) == 0) - -#define chunksize(p) ((p)->head & ~(FLAG_BITS)) - -#define clear_pinuse(p) ((p)->head &= ~PINUSE_BIT) -#define set_flag4(p) ((p)->head |= FLAG4_BIT) -#define clear_flag4(p) ((p)->head &= ~FLAG4_BIT) - -/* Treat space at ptr +/- offset as a chunk */ -#define chunk_plus_offset(p, s) ((mchunkptr)(((char*)(p)) + (s))) -#define chunk_minus_offset(p, s) ((mchunkptr)(((char*)(p)) - (s))) - -/* Ptr to next or previous physical malloc_chunk. */ -#define next_chunk(p) ((mchunkptr)( ((char*)(p)) + ((p)->head & ~FLAG_BITS))) -#define prev_chunk(p) ((mchunkptr)( ((char*)(p)) - ((p)->prev_foot) )) - -/* extract next chunk's pinuse bit */ -#define next_pinuse(p) ((next_chunk(p)->head) & PINUSE_BIT) - -/* Get/set size at footer */ -#define get_foot(p, s) (((mchunkptr)((char*)(p) + (s)))->prev_foot) -#define set_foot(p, s) (((mchunkptr)((char*)(p) + (s)))->prev_foot = (s)) - -/* Set size, pinuse bit, and foot */ -#define set_size_and_pinuse_of_free_chunk(p, s)\ - ((p)->head = (s|PINUSE_BIT), set_foot(p, s)) - -/* Set size, pinuse bit, foot, and clear next pinuse */ -#define set_free_with_pinuse(p, s, n)\ - (clear_pinuse(n), set_size_and_pinuse_of_free_chunk(p, s)) - -/* Get the internal overhead associated with chunk p */ -#define overhead_for(p)\ - (is_mmapped(p)? MMAP_CHUNK_OVERHEAD : CHUNK_OVERHEAD) - -/* Return true if malloced space is not necessarily cleared */ -#if MMAP_CLEARS -#define calloc_must_clear(p) (!is_mmapped(p)) -#else /* MMAP_CLEARS */ -#define calloc_must_clear(p) (1) -#endif /* MMAP_CLEARS */ - -/* ---------------------- Overlaid data structures ----------------------- */ - -/* - When chunks are not in use, they are treated as nodes of either - lists or trees. - - "Small" chunks are stored in circular doubly-linked lists, and look - like this: - - chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Size of previous chunk | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - `head:' | Size of chunk, in bytes |P| - mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Forward pointer to next chunk in list | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Back pointer to previous chunk in list | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Unused space (may be 0 bytes long) . - . . - . | -nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - `foot:' | Size of chunk, in bytes | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - Larger chunks are kept in a form of bitwise digital trees (aka - tries) keyed on chunksizes. Because malloc_tree_chunks are only for - free chunks greater than 256 bytes, their size doesn't impose any - constraints on user chunk sizes. Each node looks like: - - chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Size of previous chunk | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - `head:' | Size of chunk, in bytes |P| - mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Forward pointer to next chunk of same size | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Back pointer to previous chunk of same size | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Pointer to left child (child[0]) | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Pointer to right child (child[1]) | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Pointer to parent | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | bin index of this chunk | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - | Unused space . - . | -nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - `foot:' | Size of chunk, in bytes | - +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - - Each tree holding treenodes is a tree of unique chunk sizes. Chunks - of the same size are arranged in a circularly-linked list, with only - the oldest chunk (the next to be used, in our FIFO ordering) - actually in the tree. (Tree members are distinguished by a non-null - parent pointer.) If a chunk with the same size an an existing node - is inserted, it is linked off the existing node using pointers that - work in the same way as fd/bk pointers of small chunks. - - Each tree contains a power of 2 sized range of chunk sizes (the - smallest is 0x100 <= x < 0x180), which is is divided in half at each - tree level, with the chunks in the smaller half of the range (0x100 - <= x < 0x140 for the top nose) in the left subtree and the larger - half (0x140 <= x < 0x180) in the right subtree. This is, of course, - done by inspecting individual bits. - - Using these rules, each node's left subtree contains all smaller - sizes than its right subtree. However, the node at the root of each - subtree has no particular ordering relationship to either. (The - dividing line between the subtree sizes is based on trie relation.) - If we remove the last chunk of a given size from the interior of the - tree, we need to replace it with a leaf node. The tree ordering - rules permit a node to be replaced by any leaf below it. - - The smallest chunk in a tree (a common operation in a best-fit - allocator) can be found by walking a path to the leftmost leaf in - the tree. Unlike a usual binary tree, where we follow left child - pointers until we reach a null, here we follow the right child - pointer any time the left one is null, until we reach a leaf with - both child pointers null. The smallest chunk in the tree will be - somewhere along that path. - - The worst case number of steps to add, find, or remove a node is - bounded by the number of bits differentiating chunks within - bins. Under current bin calculations, this ranges from 6 up to 21 - (for 32 bit sizes) or up to 53 (for 64 bit sizes). The typical case - is of course much better. -*/ - -struct malloc_tree_chunk { - /* The first four fields must be compatible with malloc_chunk */ - size_t prev_foot; - size_t head; - struct malloc_tree_chunk* fd; - struct malloc_tree_chunk* bk; - - struct malloc_tree_chunk* child[2]; - struct malloc_tree_chunk* parent; - bindex_t index; -}; - -typedef struct malloc_tree_chunk tchunk; -typedef struct malloc_tree_chunk* tchunkptr; -typedef struct malloc_tree_chunk* tbinptr; /* The type of bins of trees */ - -/* A little helper macro for trees */ -#define leftmost_child(t) ((t)->child[0] != 0? (t)->child[0] : (t)->child[1]) - -/* ----------------------------- Segments -------------------------------- */ - -/* - Each malloc space may include non-contiguous segments, held in a - list headed by an embedded malloc_segment record representing the - top-most space. Segments also include flags holding properties of - the space. Large chunks that are directly allocated by mmap are not - included in this list. They are instead independently created and - destroyed without otherwise keeping track of them. - - Segment management mainly comes into play for spaces allocated by - MMAP. Any call to MMAP might or might not return memory that is - adjacent to an existing segment. MORECORE normally contiguously - extends the current space, so this space is almost always adjacent, - which is simpler and faster to deal with. (This is why MORECORE is - used preferentially to MMAP when both are available -- see - sys_alloc.) When allocating using MMAP, we don't use any of the - hinting mechanisms (inconsistently) supported in various - implementations of unix mmap, or distinguish reserving from - committing memory. Instead, we just ask for space, and exploit - contiguity when we get it. It is probably possible to do - better than this on some systems, but no general scheme seems - to be significantly better. - - Management entails a simpler variant of the consolidation scheme - used for chunks to reduce fragmentation -- new adjacent memory is - normally prepended or appended to an existing segment. However, - there are limitations compared to chunk consolidation that mostly - reflect the fact that segment processing is relatively infrequent - (occurring only when getting memory from system) and that we - don't expect to have huge numbers of segments: - - * Segments are not indexed, so traversal requires linear scans. (It - would be possible to index these, but is not worth the extra - overhead and complexity for most programs on most platforms.) - * New segments are only appended to old ones when holding top-most - memory; if they cannot be prepended to others, they are held in - different segments. - - Except for the top-most segment of an mstate, each segment record - is kept at the tail of its segment. Segments are added by pushing - segment records onto the list headed by &mstate.seg for the - containing mstate. - - Segment flags control allocation/merge/deallocation policies: - * If EXTERN_BIT set, then we did not allocate this segment, - and so should not try to deallocate or merge with others. - (This currently holds only for the initial segment passed - into create_mspace_with_base.) - * If USE_MMAP_BIT set, the segment may be merged with - other surrounding mmapped segments and trimmed/de-allocated - using munmap. - * If neither bit is set, then the segment was obtained using - MORECORE so can be merged with surrounding MORECORE'd segments - and deallocated/trimmed using MORECORE with negative arguments. -*/ - -struct malloc_segment { - char* base; /* base address */ - size_t size; /* allocated size */ - struct malloc_segment* next; /* ptr to next segment */ - flag_t sflags; /* mmap and extern flag */ -}; - -#define is_mmapped_segment(S) ((S)->sflags & USE_MMAP_BIT) -#define is_extern_segment(S) ((S)->sflags & EXTERN_BIT) - -typedef struct malloc_segment msegment; -typedef struct malloc_segment* msegmentptr; - -/* ---------------------------- malloc_state ----------------------------- */ - -/* - A malloc_state holds all of the bookkeeping for a space. - The main fields are: - - Top - The topmost chunk of the currently active segment. Its size is - cached in topsize. The actual size of topmost space is - topsize+TOP_FOOT_SIZE, which includes space reserved for adding - fenceposts and segment records if necessary when getting more - space from the system. The size at which to autotrim top is - cached from mparams in trim_check, except that it is disabled if - an autotrim fails. - - Designated victim (dv) - This is the preferred chunk for servicing small requests that - don't have exact fits. It is normally the chunk split off most - recently to service another small request. Its size is cached in - dvsize. The link fields of this chunk are not maintained since it - is not kept in a bin. - - SmallBins - An array of bin headers for free chunks. These bins hold chunks - with sizes less than MIN_LARGE_SIZE bytes. Each bin contains - chunks of all the same size, spaced 8 bytes apart. To simplify - use in double-linked lists, each bin header acts as a malloc_chunk - pointing to the real first node, if it exists (else pointing to - itself). This avoids special-casing for headers. But to avoid - waste, we allocate only the fd/bk pointers of bins, and then use - repositioning tricks to treat these as the fields of a chunk. - - TreeBins - Treebins are pointers to the roots of trees holding a range of - sizes. There are 2 equally spaced treebins for each power of two - from TREE_SHIFT to TREE_SHIFT+16. The last bin holds anything - larger. - - Bin maps - There is one bit map for small bins ("smallmap") and one for - treebins ("treemap). Each bin sets its bit when non-empty, and - clears the bit when empty. Bit operations are then used to avoid - bin-by-bin searching -- nearly all "search" is done without ever - looking at bins that won't be selected. The bit maps - conservatively use 32 bits per map word, even if on 64bit system. - For a good description of some of the bit-based techniques used - here, see Henry S. Warren Jr's book "Hacker's Delight" (and - supplement at http://hackersdelight.org/). Many of these are - intended to reduce the branchiness of paths through malloc etc, as - well as to reduce the number of memory locations read or written. - - Segments - A list of segments headed by an embedded malloc_segment record - representing the initial space. - - Address check support - The least_addr field is the least address ever obtained from - MORECORE or MMAP. Attempted frees and reallocs of any address less - than this are trapped (unless INSECURE is defined). - - Magic tag - A cross-check field that should always hold same value as mparams.magic. - - Max allowed footprint - The maximum allowed bytes to allocate from system (zero means no limit) - - Flags - Bits recording whether to use MMAP, locks, or contiguous MORECORE - - Statistics - Each space keeps track of current and maximum system memory - obtained via MORECORE or MMAP. - - Trim support - Fields holding the amount of unused topmost memory that should trigger - trimming, and a counter to force periodic scanning to release unused - non-topmost segments. - - Locking - If USE_LOCKS is defined, the "mutex" lock is acquired and released - around every public call using this mspace. - - Extension support - A void* pointer and a size_t field that can be used to help implement - extensions to this malloc. -*/ - -/* Bin types, widths and sizes */ -#define NSMALLBINS (32U) -#define NTREEBINS (32U) -#define SMALLBIN_SHIFT (3U) -#define SMALLBIN_WIDTH (SIZE_T_ONE << SMALLBIN_SHIFT) -#define TREEBIN_SHIFT (8U) -#define MIN_LARGE_SIZE (SIZE_T_ONE << TREEBIN_SHIFT) -#define MAX_SMALL_SIZE (MIN_LARGE_SIZE - SIZE_T_ONE) -#define MAX_SMALL_REQUEST (MAX_SMALL_SIZE - CHUNK_ALIGN_MASK - CHUNK_OVERHEAD) - -struct malloc_state { - binmap_t smallmap; - binmap_t treemap; - size_t dvsize; - size_t topsize; - char* least_addr; - mchunkptr dv; - mchunkptr top; - size_t trim_check; - size_t release_checks; - size_t magic; - mchunkptr smallbins[(NSMALLBINS+1)*2]; - tbinptr treebins[NTREEBINS]; - size_t footprint; - size_t max_footprint; - size_t footprint_limit; /* zero means no limit */ - flag_t mflags; -#if USE_LOCKS - MLOCK_T mutex; /* locate lock among fields that rarely change */ -#endif /* USE_LOCKS */ - msegment seg; - void* extp; /* Unused but available for extensions */ - size_t exts; -}; - -typedef struct malloc_state* mstate; - -/* ------------- Global malloc_state and malloc_params ------------------- */ - -/* - malloc_params holds global properties, including those that can be - dynamically set using mallopt. There is a single instance, mparams, - initialized in init_mparams. Note that the non-zeroness of "magic" - also serves as an initialization flag. -*/ - -struct malloc_params { - size_t magic; - size_t page_size; - size_t granularity; - size_t mmap_threshold; - size_t trim_threshold; - flag_t default_mflags; -}; - -static struct malloc_params mparams; - -/* Ensure mparams initialized */ -#define ensure_initialization() (void)(mparams.magic != 0 || init_mparams()) - -#if !ONLY_MSPACES - -/* The global malloc_state used for all non-"mspace" calls */ -static struct malloc_state _gm_; -#define gm (&_gm_) -#define is_global(M) ((M) == &_gm_) - -#endif /* !ONLY_MSPACES */ - -#define is_initialized(M) ((M)->top != 0) - -/* -------------------------- system alloc setup ------------------------- */ - -/* Operations on mflags */ - -#define use_lock(M) ((M)->mflags & USE_LOCK_BIT) -#define enable_lock(M) ((M)->mflags |= USE_LOCK_BIT) -#if USE_LOCKS -#define disable_lock(M) ((M)->mflags &= ~USE_LOCK_BIT) -#else -#define disable_lock(M) -#endif - -#define use_mmap(M) ((M)->mflags & USE_MMAP_BIT) -#define enable_mmap(M) ((M)->mflags |= USE_MMAP_BIT) -#if HAVE_MMAP -#define disable_mmap(M) ((M)->mflags &= ~USE_MMAP_BIT) -#else -#define disable_mmap(M) -#endif - -#define use_noncontiguous(M) ((M)->mflags & USE_NONCONTIGUOUS_BIT) -#define disable_contiguous(M) ((M)->mflags |= USE_NONCONTIGUOUS_BIT) - -#define set_lock(M,L)\ - ((M)->mflags = (L)?\ - ((M)->mflags | USE_LOCK_BIT) :\ - ((M)->mflags & ~USE_LOCK_BIT)) - -/* page-align a size */ -#define page_align(S)\ - (((S) + (mparams.page_size - SIZE_T_ONE)) & ~(mparams.page_size - SIZE_T_ONE)) - -/* granularity-align a size */ -#define granularity_align(S)\ - (((S) + (mparams.granularity - SIZE_T_ONE))\ - & ~(mparams.granularity - SIZE_T_ONE)) - - -/* For mmap, use granularity alignment on windows, else page-align */ -#ifdef WIN32 -#define mmap_align(S) granularity_align(S) -#else -#define mmap_align(S) page_align(S) -#endif - -/* For sys_alloc, enough padding to ensure can malloc request on success */ -#define SYS_ALLOC_PADDING (TOP_FOOT_SIZE + MALLOC_ALIGNMENT) - -#define is_page_aligned(S)\ - (((size_t)(S) & (mparams.page_size - SIZE_T_ONE)) == 0) -#define is_granularity_aligned(S)\ - (((size_t)(S) & (mparams.granularity - SIZE_T_ONE)) == 0) - -/* True if segment S holds address A */ -#define segment_holds(S, A)\ - ((char*)(A) >= S->base && (char*)(A) < S->base + S->size) - -/* Return segment holding given address */ -static msegmentptr segment_holding(mstate m, char* addr) { - msegmentptr sp = &m->seg; - for (;;) { - if (addr >= sp->base && addr < sp->base + sp->size) - return sp; - if ((sp = sp->next) == 0) - return 0; - } -} - -/* Return true if segment contains a segment link */ -static int has_segment_link(mstate m, msegmentptr ss) { - msegmentptr sp = &m->seg; - for (;;) { - if ((char*)sp >= ss->base && (char*)sp < ss->base + ss->size) - return 1; - if ((sp = sp->next) == 0) - return 0; - } -} - -#ifndef MORECORE_CANNOT_TRIM -#define should_trim(M,s) ((s) > (M)->trim_check) -#else /* MORECORE_CANNOT_TRIM */ -#define should_trim(M,s) (0) -#endif /* MORECORE_CANNOT_TRIM */ - -/* - TOP_FOOT_SIZE is padding at the end of a segment, including space - that may be needed to place segment records and fenceposts when new - noncontiguous segments are added. -*/ -#define TOP_FOOT_SIZE\ - (align_offset(chunk2mem(0))+pad_request(sizeof(struct malloc_segment))+MIN_CHUNK_SIZE) - - -/* ------------------------------- Hooks -------------------------------- */ - -/* - PREACTION should be defined to return 0 on success, and nonzero on - failure. If you are not using locking, you can redefine these to do - anything you like. -*/ - -#if USE_LOCKS -#define PREACTION(M) ((use_lock(M))? ACQUIRE_LOCK(&(M)->mutex) : 0) -#define POSTACTION(M) { if (use_lock(M)) RELEASE_LOCK(&(M)->mutex); } -#else /* USE_LOCKS */ - -#ifndef PREACTION -#define PREACTION(M) (0) -#endif /* PREACTION */ - -#ifndef POSTACTION -#define POSTACTION(M) -#endif /* POSTACTION */ - -#endif /* USE_LOCKS */ - -/* - CORRUPTION_ERROR_ACTION is triggered upon detected bad addresses. - USAGE_ERROR_ACTION is triggered on detected bad frees and - reallocs. The argument p is an address that might have triggered the - fault. It is ignored by the two predefined actions, but might be - useful in custom actions that try to help diagnose errors. -*/ - -#if PROCEED_ON_ERROR - -/* A count of the number of corruption errors causing resets */ -int malloc_corruption_error_count; - -/* default corruption action */ -static void reset_on_error(mstate m); - -#define CORRUPTION_ERROR_ACTION(m) reset_on_error(m) -#define USAGE_ERROR_ACTION(m, p) - -#else /* PROCEED_ON_ERROR */ - -#ifndef CORRUPTION_ERROR_ACTION -#define CORRUPTION_ERROR_ACTION(m) ABORT -#endif /* CORRUPTION_ERROR_ACTION */ - -#ifndef USAGE_ERROR_ACTION -#define USAGE_ERROR_ACTION(m,p) ABORT -#endif /* USAGE_ERROR_ACTION */ - -#endif /* PROCEED_ON_ERROR */ - - -/* -------------------------- Debugging setup ---------------------------- */ - -#if ! DEBUG - -#define check_free_chunk(M,P) -#define check_inuse_chunk(M,P) -#define check_malloced_chunk(M,P,N) -#define check_mmapped_chunk(M,P) -#define check_malloc_state(M) -#define check_top_chunk(M,P) - -#else /* DEBUG */ -#define check_free_chunk(M,P) do_check_free_chunk(M,P) -#define check_inuse_chunk(M,P) do_check_inuse_chunk(M,P) -#define check_top_chunk(M,P) do_check_top_chunk(M,P) -#define check_malloced_chunk(M,P,N) do_check_malloced_chunk(M,P,N) -#define check_mmapped_chunk(M,P) do_check_mmapped_chunk(M,P) -#define check_malloc_state(M) do_check_malloc_state(M) - -static void do_check_any_chunk(mstate m, mchunkptr p); -static void do_check_top_chunk(mstate m, mchunkptr p); -static void do_check_mmapped_chunk(mstate m, mchunkptr p); -static void do_check_inuse_chunk(mstate m, mchunkptr p); -static void do_check_free_chunk(mstate m, mchunkptr p); -static void do_check_malloced_chunk(mstate m, void* mem, size_t s); -static void do_check_tree(mstate m, tchunkptr t); -static void do_check_treebin(mstate m, bindex_t i); -static void do_check_smallbin(mstate m, bindex_t i); -static void do_check_malloc_state(mstate m); -static int bin_find(mstate m, mchunkptr x); -static size_t traverse_and_check(mstate m); -#endif /* DEBUG */ - -/* ---------------------------- Indexing Bins ---------------------------- */ - -#define is_small(s) (((s) >> SMALLBIN_SHIFT) < NSMALLBINS) -#define small_index(s) (bindex_t)((s) >> SMALLBIN_SHIFT) -#define small_index2size(i) ((i) << SMALLBIN_SHIFT) -#define MIN_SMALL_INDEX (small_index(MIN_CHUNK_SIZE)) - -/* addressing by index. See above about smallbin repositioning */ -#define smallbin_at(M, i) ((sbinptr)((char*)&((M)->smallbins[(i)<<1]))) -#define treebin_at(M,i) (&((M)->treebins[i])) - -/* assign tree index for size S to variable I. Use x86 asm if possible */ -#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) -#define compute_tree_index(S, I)\ -{\ - unsigned int X = S >> TREEBIN_SHIFT;\ - if (X == 0)\ - I = 0;\ - else if (X > 0xFFFF)\ - I = NTREEBINS-1;\ - else {\ - unsigned int K = (unsigned) sizeof(X)*__CHAR_BIT__ - 1 - (unsigned) __builtin_clz(X); \ - I = (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\ - }\ -} - -#elif defined (__INTEL_COMPILER) -#define compute_tree_index(S, I)\ -{\ - size_t X = S >> TREEBIN_SHIFT;\ - if (X == 0)\ - I = 0;\ - else if (X > 0xFFFF)\ - I = NTREEBINS-1;\ - else {\ - unsigned int K = _bit_scan_reverse (X); \ - I = (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\ - }\ -} - -#elif defined(_MSC_VER) && _MSC_VER>=1300 -#define compute_tree_index(S, I)\ -{\ - size_t X = S >> TREEBIN_SHIFT;\ - if (X == 0)\ - I = 0;\ - else if (X > 0xFFFF)\ - I = NTREEBINS-1;\ - else {\ - unsigned int K;\ - _BitScanReverse((DWORD *) &K, (DWORD) X);\ - I = (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\ - }\ -} - -#else /* GNUC */ -#define compute_tree_index(S, I)\ -{\ - size_t X = S >> TREEBIN_SHIFT;\ - if (X == 0)\ - I = 0;\ - else if (X > 0xFFFF)\ - I = NTREEBINS-1;\ - else {\ - unsigned int Y = (unsigned int)X;\ - unsigned int N = ((Y - 0x100) >> 16) & 8;\ - unsigned int K = (((Y <<= N) - 0x1000) >> 16) & 4;\ - N += K;\ - N += K = (((Y <<= K) - 0x4000) >> 16) & 2;\ - K = 14 - N + ((Y <<= K) >> 15);\ - I = (K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1));\ - }\ -} -#endif /* GNUC */ - -/* Bit representing maximum resolved size in a treebin at i */ -#define bit_for_tree_index(i) \ - (i == NTREEBINS-1)? (SIZE_T_BITSIZE-1) : (((i) >> 1) + TREEBIN_SHIFT - 2) - -/* Shift placing maximum resolved bit in a treebin at i as sign bit */ -#define leftshift_for_tree_index(i) \ - ((i == NTREEBINS-1)? 0 : \ - ((SIZE_T_BITSIZE-SIZE_T_ONE) - (((i) >> 1) + TREEBIN_SHIFT - 2))) - -/* The size of the smallest chunk held in bin with index i */ -#define minsize_for_tree_index(i) \ - ((SIZE_T_ONE << (((i) >> 1) + TREEBIN_SHIFT)) | \ - (((size_t)((i) & SIZE_T_ONE)) << (((i) >> 1) + TREEBIN_SHIFT - 1))) - - -/* ------------------------ Operations on bin maps ----------------------- */ - -/* bit corresponding to given index */ -#define idx2bit(i) ((binmap_t)(1) << (i)) - -/* Mark/Clear bits with given index */ -#define mark_smallmap(M,i) ((M)->smallmap |= idx2bit(i)) -#define clear_smallmap(M,i) ((M)->smallmap &= ~idx2bit(i)) -#define smallmap_is_marked(M,i) ((M)->smallmap & idx2bit(i)) - -#define mark_treemap(M,i) ((M)->treemap |= idx2bit(i)) -#define clear_treemap(M,i) ((M)->treemap &= ~idx2bit(i)) -#define treemap_is_marked(M,i) ((M)->treemap & idx2bit(i)) - -/* isolate the least set bit of a bitmap */ -#define least_bit(x) ((x) & -(x)) - -/* mask with all bits to left of least bit of x on */ -#define left_bits(x) ((x<<1) | -(x<<1)) - -/* mask with all bits to left of or equal to least bit of x on */ -#define same_or_left_bits(x) ((x) | -(x)) - -/* index corresponding to given bit. Use x86 asm if possible */ - -#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) -#define compute_bit2idx(X, I)\ -{\ - unsigned int J;\ - J = __builtin_ctz(X); \ - I = (bindex_t)J;\ -} - -#elif defined (__INTEL_COMPILER) -#define compute_bit2idx(X, I)\ -{\ - unsigned int J;\ - J = _bit_scan_forward (X); \ - I = (bindex_t)J;\ -} - -#elif defined(_MSC_VER) && _MSC_VER>=1300 -#define compute_bit2idx(X, I)\ -{\ - unsigned int J;\ - _BitScanForward((DWORD *) &J, X);\ - I = (bindex_t)J;\ -} - -#elif USE_BUILTIN_FFS -#define compute_bit2idx(X, I) I = ffs(X)-1 - -#else -#define compute_bit2idx(X, I)\ -{\ - unsigned int Y = X - 1;\ - unsigned int K = Y >> (16-4) & 16;\ - unsigned int N = K; Y >>= K;\ - N += K = Y >> (8-3) & 8; Y >>= K;\ - N += K = Y >> (4-2) & 4; Y >>= K;\ - N += K = Y >> (2-1) & 2; Y >>= K;\ - N += K = Y >> (1-0) & 1; Y >>= K;\ - I = (bindex_t)(N + Y);\ -} -#endif /* GNUC */ - - -/* ----------------------- Runtime Check Support ------------------------- */ - -/* - For security, the main invariant is that malloc/free/etc never - writes to a static address other than malloc_state, unless static - malloc_state itself has been corrupted, which cannot occur via - malloc (because of these checks). In essence this means that we - believe all pointers, sizes, maps etc held in malloc_state, but - check all of those linked or offsetted from other embedded data - structures. These checks are interspersed with main code in a way - that tends to minimize their run-time cost. - - When FOOTERS is defined, in addition to range checking, we also - verify footer fields of inuse chunks, which can be used guarantee - that the mstate controlling malloc/free is intact. This is a - streamlined version of the approach described by William Robertson - et al in "Run-time Detection of Heap-based Overflows" LISA'03 - http://www.usenix.org/events/lisa03/tech/robertson.html The footer - of an inuse chunk holds the xor of its mstate and a random seed, - that is checked upon calls to free() and realloc(). This is - (probabalistically) unguessable from outside the program, but can be - computed by any code successfully malloc'ing any chunk, so does not - itself provide protection against code that has already broken - security through some other means. Unlike Robertson et al, we - always dynamically check addresses of all offset chunks (previous, - next, etc). This turns out to be cheaper than relying on hashes. -*/ - -#if !INSECURE -/* Check if address a is at least as high as any from MORECORE or MMAP */ -#define ok_address(M, a) ((char*)(a) >= (M)->least_addr) -/* Check if address of next chunk n is higher than base chunk p */ -#define ok_next(p, n) ((char*)(p) < (char*)(n)) -/* Check if p has inuse status */ -#define ok_inuse(p) is_inuse(p) -/* Check if p has its pinuse bit on */ -#define ok_pinuse(p) pinuse(p) - -#else /* !INSECURE */ -#define ok_address(M, a) (1) -#define ok_next(b, n) (1) -#define ok_inuse(p) (1) -#define ok_pinuse(p) (1) -#endif /* !INSECURE */ - -#if (FOOTERS && !INSECURE) -/* Check if (alleged) mstate m has expected magic field */ -#define ok_magic(M) ((M)->magic == mparams.magic) -#else /* (FOOTERS && !INSECURE) */ -#define ok_magic(M) (1) -#endif /* (FOOTERS && !INSECURE) */ - -/* In gcc, use __builtin_expect to minimize impact of checks */ -#if !INSECURE -#if defined(__GNUC__) && __GNUC__ >= 3 -#define RTCHECK(e) __builtin_expect(e, 1) -#else /* GNUC */ -#define RTCHECK(e) (e) -#endif /* GNUC */ -#else /* !INSECURE */ -#define RTCHECK(e) (1) -#endif /* !INSECURE */ - -/* macros to set up inuse chunks with or without footers */ - -#if !FOOTERS - -#define mark_inuse_foot(M,p,s) - -/* Macros for setting head/foot of non-mmapped chunks */ - -/* Set cinuse bit and pinuse bit of next chunk */ -#define set_inuse(M,p,s)\ - ((p)->head = (((p)->head & PINUSE_BIT)|s|CINUSE_BIT),\ - ((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT) - -/* Set cinuse and pinuse of this chunk and pinuse of next chunk */ -#define set_inuse_and_pinuse(M,p,s)\ - ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\ - ((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT) - -/* Set size, cinuse and pinuse bit of this chunk */ -#define set_size_and_pinuse_of_inuse_chunk(M, p, s)\ - ((p)->head = (s|PINUSE_BIT|CINUSE_BIT)) - -#else /* FOOTERS */ - -/* Set foot of inuse chunk to be xor of mstate and seed */ -#define mark_inuse_foot(M,p,s)\ - (((mchunkptr)((char*)(p) + (s)))->prev_foot = ((size_t)(M) ^ mparams.magic)) - -#define get_mstate_for(p)\ - ((mstate)(((mchunkptr)((char*)(p) +\ - (chunksize(p))))->prev_foot ^ mparams.magic)) - -#define set_inuse(M,p,s)\ - ((p)->head = (((p)->head & PINUSE_BIT)|s|CINUSE_BIT),\ - (((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT), \ - mark_inuse_foot(M,p,s)) - -#define set_inuse_and_pinuse(M,p,s)\ - ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\ - (((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT),\ - mark_inuse_foot(M,p,s)) - -#define set_size_and_pinuse_of_inuse_chunk(M, p, s)\ - ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\ - mark_inuse_foot(M, p, s)) - -#endif /* !FOOTERS */ - -/* ---------------------------- setting mparams -------------------------- */ - -#if LOCK_AT_FORK -static void pre_fork(void) { ACQUIRE_LOCK(&(gm)->mutex); } -static void post_fork_parent(void) { RELEASE_LOCK(&(gm)->mutex); } -static void post_fork_child(void) { INITIAL_LOCK(&(gm)->mutex); } -#endif /* LOCK_AT_FORK */ - -/* Initialize mparams */ -static int init_mparams(void) { -#ifdef NEED_GLOBAL_LOCK_INIT - if (malloc_global_mutex_status <= 0) - init_malloc_global_mutex(); -#endif - - ACQUIRE_MALLOC_GLOBAL_LOCK(); - if (mparams.magic == 0) { - size_t magic; - size_t psize; - size_t gsize; - -#ifndef WIN32 - psize = malloc_getpagesize; - gsize = ((DEFAULT_GRANULARITY != 0)? DEFAULT_GRANULARITY : psize); -#else /* WIN32 */ - { - SYSTEM_INFO system_info; - GetSystemInfo(&system_info); - psize = system_info.dwPageSize; - gsize = ((DEFAULT_GRANULARITY != 0)? - DEFAULT_GRANULARITY : system_info.dwAllocationGranularity); - } -#endif /* WIN32 */ - - /* Sanity-check configuration: - size_t must be unsigned and as wide as pointer type. - ints must be at least 4 bytes. - alignment must be at least 8. - Alignment, min chunk size, and page size must all be powers of 2. - */ - if ((sizeof(size_t) != sizeof(char*)) || - (MAX_SIZE_T < MIN_CHUNK_SIZE) || - (sizeof(int) < 4) || - (MALLOC_ALIGNMENT < (size_t)8U) || - ((MALLOC_ALIGNMENT & (MALLOC_ALIGNMENT-SIZE_T_ONE)) != 0) || - ((MCHUNK_SIZE & (MCHUNK_SIZE-SIZE_T_ONE)) != 0) || - ((gsize & (gsize-SIZE_T_ONE)) != 0) || - ((psize & (psize-SIZE_T_ONE)) != 0)) - ABORT; - mparams.granularity = gsize; - mparams.page_size = psize; - mparams.mmap_threshold = DEFAULT_MMAP_THRESHOLD; - mparams.trim_threshold = DEFAULT_TRIM_THRESHOLD; -#if MORECORE_CONTIGUOUS - mparams.default_mflags = USE_LOCK_BIT|USE_MMAP_BIT; -#else /* MORECORE_CONTIGUOUS */ - mparams.default_mflags = USE_LOCK_BIT|USE_MMAP_BIT|USE_NONCONTIGUOUS_BIT; -#endif /* MORECORE_CONTIGUOUS */ - -#if !ONLY_MSPACES - /* Set up lock for main malloc area */ - gm->mflags = mparams.default_mflags; - (void)INITIAL_LOCK(&gm->mutex); -#endif -#if LOCK_AT_FORK - pthread_atfork(&pre_fork, &post_fork_parent, &post_fork_child); -#endif - - { -#if USE_DEV_RANDOM - int fd; - unsigned char buf[sizeof(size_t)]; - /* Try to use /dev/urandom, else fall back on using time */ - if ((fd = open("/dev/urandom", O_RDONLY)) >= 0 && - read(fd, buf, sizeof(buf)) == sizeof(buf)) { - magic = *((size_t *) buf); - close(fd); - } - else -#endif /* USE_DEV_RANDOM */ -#ifdef WIN32 - magic = (size_t)(GetTickCount() ^ (size_t)0x55555555U); -#elif defined(LACKS_TIME_H) - magic = (size_t)&magic ^ (size_t)0x55555555U; -#else - magic = (size_t)(time(0) ^ (size_t)0x55555555U); -#endif - magic |= (size_t)8U; /* ensure nonzero */ - magic &= ~(size_t)7U; /* improve chances of fault for bad values */ - /* Until memory modes commonly available, use volatile-write */ - (*(volatile size_t *)(&(mparams.magic))) = magic; - } - } - - RELEASE_MALLOC_GLOBAL_LOCK(); - return 1; -} - -/* support for mallopt */ -static int change_mparam(int param_number, int value) { - size_t val; - ensure_initialization(); - val = (value == -1)? MAX_SIZE_T : (size_t)value; - switch(param_number) { - case M_TRIM_THRESHOLD: - mparams.trim_threshold = val; - return 1; - case M_GRANULARITY: - if (val >= mparams.page_size && ((val & (val-1)) == 0)) { - mparams.granularity = val; - return 1; - } - else - return 0; - case M_MMAP_THRESHOLD: - mparams.mmap_threshold = val; - return 1; - default: - return 0; - } -} - -#if DEBUG -/* ------------------------- Debugging Support --------------------------- */ - -/* Check properties of any chunk, whether free, inuse, mmapped etc */ -static void do_check_any_chunk(mstate m, mchunkptr p) { - assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD)); - assert(ok_address(m, p)); -} - -/* Check properties of top chunk */ -static void do_check_top_chunk(mstate m, mchunkptr p) { - msegmentptr sp = segment_holding(m, (char*)p); - size_t sz = p->head & ~INUSE_BITS; /* third-lowest bit can be set! */ - assert(sp != 0); - assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD)); - assert(ok_address(m, p)); - assert(sz == m->topsize); - assert(sz > 0); - assert(sz == ((sp->base + sp->size) - (char*)p) - TOP_FOOT_SIZE); - assert(pinuse(p)); - assert(!pinuse(chunk_plus_offset(p, sz))); -} - -/* Check properties of (inuse) mmapped chunks */ -static void do_check_mmapped_chunk(mstate m, mchunkptr p) { - size_t sz = chunksize(p); - size_t len = (sz + (p->prev_foot) + MMAP_FOOT_PAD); - assert(is_mmapped(p)); - assert(use_mmap(m)); - assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD)); - assert(ok_address(m, p)); - assert(!is_small(sz)); - assert((len & (mparams.page_size-SIZE_T_ONE)) == 0); - assert(chunk_plus_offset(p, sz)->head == FENCEPOST_HEAD); - assert(chunk_plus_offset(p, sz+SIZE_T_SIZE)->head == 0); -} - -/* Check properties of inuse chunks */ -static void do_check_inuse_chunk(mstate m, mchunkptr p) { - do_check_any_chunk(m, p); - assert(is_inuse(p)); - assert(next_pinuse(p)); - /* If not pinuse and not mmapped, previous chunk has OK offset */ - assert(is_mmapped(p) || pinuse(p) || next_chunk(prev_chunk(p)) == p); - if (is_mmapped(p)) - do_check_mmapped_chunk(m, p); -} - -/* Check properties of free chunks */ -static void do_check_free_chunk(mstate m, mchunkptr p) { - size_t sz = chunksize(p); - mchunkptr next = chunk_plus_offset(p, sz); - do_check_any_chunk(m, p); - assert(!is_inuse(p)); - assert(!next_pinuse(p)); - assert (!is_mmapped(p)); - if (p != m->dv && p != m->top) { - if (sz >= MIN_CHUNK_SIZE) { - assert((sz & CHUNK_ALIGN_MASK) == 0); - assert(is_aligned(chunk2mem(p))); - assert(next->prev_foot == sz); - assert(pinuse(p)); - assert (next == m->top || is_inuse(next)); - assert(p->fd->bk == p); - assert(p->bk->fd == p); - } - else /* markers are always of size SIZE_T_SIZE */ - assert(sz == SIZE_T_SIZE); - } -} - -/* Check properties of malloced chunks at the point they are malloced */ -static void do_check_malloced_chunk(mstate m, void* mem, size_t s) { - if (mem != 0) { - mchunkptr p = mem2chunk(mem); - size_t sz = p->head & ~INUSE_BITS; - do_check_inuse_chunk(m, p); - assert((sz & CHUNK_ALIGN_MASK) == 0); - assert(sz >= MIN_CHUNK_SIZE); - assert(sz >= s); - /* unless mmapped, size is less than MIN_CHUNK_SIZE more than request */ - assert(is_mmapped(p) || sz < (s + MIN_CHUNK_SIZE)); - } -} - -/* Check a tree and its subtrees. */ -static void do_check_tree(mstate m, tchunkptr t) { - tchunkptr head = 0; - tchunkptr u = t; - bindex_t tindex = t->index; - size_t tsize = chunksize(t); - bindex_t idx; - compute_tree_index(tsize, idx); - assert(tindex == idx); - assert(tsize >= MIN_LARGE_SIZE); - assert(tsize >= minsize_for_tree_index(idx)); - assert((idx == NTREEBINS-1) || (tsize < minsize_for_tree_index((idx+1)))); - - do { /* traverse through chain of same-sized nodes */ - do_check_any_chunk(m, ((mchunkptr)u)); - assert(u->index == tindex); - assert(chunksize(u) == tsize); - assert(!is_inuse(u)); - assert(!next_pinuse(u)); - assert(u->fd->bk == u); - assert(u->bk->fd == u); - if (u->parent == 0) { - assert(u->child[0] == 0); - assert(u->child[1] == 0); - } - else { - assert(head == 0); /* only one node on chain has parent */ - head = u; - assert(u->parent != u); - assert (u->parent->child[0] == u || - u->parent->child[1] == u || - *((tbinptr*)(u->parent)) == u); - if (u->child[0] != 0) { - assert(u->child[0]->parent == u); - assert(u->child[0] != u); - do_check_tree(m, u->child[0]); - } - if (u->child[1] != 0) { - assert(u->child[1]->parent == u); - assert(u->child[1] != u); - do_check_tree(m, u->child[1]); - } - if (u->child[0] != 0 && u->child[1] != 0) { - assert(chunksize(u->child[0]) < chunksize(u->child[1])); - } - } - u = u->fd; - } while (u != t); - assert(head != 0); -} - -/* Check all the chunks in a treebin. */ -static void do_check_treebin(mstate m, bindex_t i) { - tbinptr* tb = treebin_at(m, i); - tchunkptr t = *tb; - int empty = (m->treemap & (1U << i)) == 0; - if (t == 0) - assert(empty); - if (!empty) - do_check_tree(m, t); -} - -/* Check all the chunks in a smallbin. */ -static void do_check_smallbin(mstate m, bindex_t i) { - sbinptr b = smallbin_at(m, i); - mchunkptr p = b->bk; - unsigned int empty = (m->smallmap & (1U << i)) == 0; - if (p == b) - assert(empty); - if (!empty) { - for (; p != b; p = p->bk) { - size_t size = chunksize(p); - mchunkptr q; - /* each chunk claims to be free */ - do_check_free_chunk(m, p); - /* chunk belongs in bin */ - assert(small_index(size) == i); - assert(p->bk == b || chunksize(p->bk) == chunksize(p)); - /* chunk is followed by an inuse chunk */ - q = next_chunk(p); - if (q->head != FENCEPOST_HEAD) - do_check_inuse_chunk(m, q); - } - } -} - -/* Find x in a bin. Used in other check functions. */ -static int bin_find(mstate m, mchunkptr x) { - size_t size = chunksize(x); - if (is_small(size)) { - bindex_t sidx = small_index(size); - sbinptr b = smallbin_at(m, sidx); - if (smallmap_is_marked(m, sidx)) { - mchunkptr p = b; - do { - if (p == x) - return 1; - } while ((p = p->fd) != b); - } - } - else { - bindex_t tidx; - compute_tree_index(size, tidx); - if (treemap_is_marked(m, tidx)) { - tchunkptr t = *treebin_at(m, tidx); - size_t sizebits = size << leftshift_for_tree_index(tidx); - while (t != 0 && chunksize(t) != size) { - t = t->child[(sizebits >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1]; - sizebits <<= 1; - } - if (t != 0) { - tchunkptr u = t; - do { - if (u == (tchunkptr)x) - return 1; - } while ((u = u->fd) != t); - } - } - } - return 0; -} - -/* Traverse each chunk and check it; return total */ -static size_t traverse_and_check(mstate m) { - size_t sum = 0; - if (is_initialized(m)) { - msegmentptr s = &m->seg; - sum += m->topsize + TOP_FOOT_SIZE; - while (s != 0) { - mchunkptr q = align_as_chunk(s->base); - mchunkptr lastq = 0; - assert(pinuse(q)); - while (segment_holds(s, q) && - q != m->top && q->head != FENCEPOST_HEAD) { - sum += chunksize(q); - if (is_inuse(q)) { - assert(!bin_find(m, q)); - do_check_inuse_chunk(m, q); - } - else { - assert(q == m->dv || bin_find(m, q)); - assert(lastq == 0 || is_inuse(lastq)); /* Not 2 consecutive free */ - do_check_free_chunk(m, q); - } - lastq = q; - q = next_chunk(q); - } - s = s->next; - } - } - return sum; -} - - -/* Check all properties of malloc_state. */ -static void do_check_malloc_state(mstate m) { - bindex_t i; - size_t total; - /* check bins */ - for (i = 0; i < NSMALLBINS; ++i) - do_check_smallbin(m, i); - for (i = 0; i < NTREEBINS; ++i) - do_check_treebin(m, i); - - if (m->dvsize != 0) { /* check dv chunk */ - do_check_any_chunk(m, m->dv); - assert(m->dvsize == chunksize(m->dv)); - assert(m->dvsize >= MIN_CHUNK_SIZE); - assert(bin_find(m, m->dv) == 0); - } - - if (m->top != 0) { /* check top chunk */ - do_check_top_chunk(m, m->top); - /*assert(m->topsize == chunksize(m->top)); redundant */ - assert(m->topsize > 0); - assert(bin_find(m, m->top) == 0); - } - - total = traverse_and_check(m); - assert(total <= m->footprint); - assert(m->footprint <= m->max_footprint); -} -#endif /* DEBUG */ - -/* ----------------------------- statistics ------------------------------ */ - -#if !NO_MALLINFO -static struct mallinfo internal_mallinfo(mstate m) { - struct mallinfo nm = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - ensure_initialization(); - if (!PREACTION(m)) { - check_malloc_state(m); - if (is_initialized(m)) { - size_t nfree = SIZE_T_ONE; /* top always free */ - size_t mfree = m->topsize + TOP_FOOT_SIZE; - size_t sum = mfree; - msegmentptr s = &m->seg; - while (s != 0) { - mchunkptr q = align_as_chunk(s->base); - while (segment_holds(s, q) && - q != m->top && q->head != FENCEPOST_HEAD) { - size_t sz = chunksize(q); - sum += sz; - if (!is_inuse(q)) { - mfree += sz; - ++nfree; - } - q = next_chunk(q); - } - s = s->next; - } - - nm.arena = sum; - nm.ordblks = nfree; - nm.hblkhd = m->footprint - sum; - nm.usmblks = m->max_footprint; - nm.uordblks = m->footprint - mfree; - nm.fordblks = mfree; - nm.keepcost = m->topsize; - } - - POSTACTION(m); - } - return nm; -} -#endif /* !NO_MALLINFO */ - -#if !NO_MALLOC_STATS -static void internal_malloc_stats(mstate m) { - ensure_initialization(); - if (!PREACTION(m)) { - size_t maxfp = 0; - size_t fp = 0; - size_t used = 0; - check_malloc_state(m); - if (is_initialized(m)) { - msegmentptr s = &m->seg; - maxfp = m->max_footprint; - fp = m->footprint; - used = fp - (m->topsize + TOP_FOOT_SIZE); - - while (s != 0) { - mchunkptr q = align_as_chunk(s->base); - while (segment_holds(s, q) && - q != m->top && q->head != FENCEPOST_HEAD) { - if (!is_inuse(q)) - used -= chunksize(q); - q = next_chunk(q); - } - s = s->next; - } - } - POSTACTION(m); /* drop lock */ - fprintf(stderr, "max system bytes = %10lu\n", (unsigned long)(maxfp)); - fprintf(stderr, "system bytes = %10lu\n", (unsigned long)(fp)); - fprintf(stderr, "in use bytes = %10lu\n", (unsigned long)(used)); - } -} -#endif /* NO_MALLOC_STATS */ - -/* ----------------------- Operations on smallbins ----------------------- */ - -/* - Various forms of linking and unlinking are defined as macros. Even - the ones for trees, which are very long but have very short typical - paths. This is ugly but reduces reliance on inlining support of - compilers. -*/ - -/* Link a free chunk into a smallbin */ -#define insert_small_chunk(M, P, S) {\ - bindex_t I = small_index(S);\ - mchunkptr B = smallbin_at(M, I);\ - mchunkptr F = B;\ - assert(S >= MIN_CHUNK_SIZE);\ - if (!smallmap_is_marked(M, I))\ - mark_smallmap(M, I);\ - else if (RTCHECK(ok_address(M, B->fd)))\ - F = B->fd;\ - else {\ - CORRUPTION_ERROR_ACTION(M);\ - }\ - B->fd = P;\ - F->bk = P;\ - P->fd = F;\ - P->bk = B;\ -} - -/* Unlink a chunk from a smallbin */ -#define unlink_small_chunk(M, P, S) {\ - mchunkptr F = P->fd;\ - mchunkptr B = P->bk;\ - bindex_t I = small_index(S);\ - assert(P != B);\ - assert(P != F);\ - assert(chunksize(P) == small_index2size(I));\ - if (RTCHECK(F == smallbin_at(M,I) || (ok_address(M, F) && F->bk == P))) { \ - if (B == F) {\ - clear_smallmap(M, I);\ - }\ - else if (RTCHECK(B == smallbin_at(M,I) ||\ - (ok_address(M, B) && B->fd == P))) {\ - F->bk = B;\ - B->fd = F;\ - }\ - else {\ - CORRUPTION_ERROR_ACTION(M);\ - }\ - }\ - else {\ - CORRUPTION_ERROR_ACTION(M);\ - }\ -} - -/* Unlink the first chunk from a smallbin */ -#define unlink_first_small_chunk(M, B, P, I) {\ - mchunkptr F = P->fd;\ - assert(P != B);\ - assert(P != F);\ - assert(chunksize(P) == small_index2size(I));\ - if (B == F) {\ - clear_smallmap(M, I);\ - }\ - else if (RTCHECK(ok_address(M, F) && F->bk == P)) {\ - F->bk = B;\ - B->fd = F;\ - }\ - else {\ - CORRUPTION_ERROR_ACTION(M);\ - }\ -} - -/* Replace dv node, binning the old one */ -/* Used only when dvsize known to be small */ -#define replace_dv(M, P, S) {\ - size_t DVS = M->dvsize;\ - assert(is_small(DVS));\ - if (DVS != 0) {\ - mchunkptr DV = M->dv;\ - insert_small_chunk(M, DV, DVS);\ - }\ - M->dvsize = S;\ - M->dv = P;\ -} - -/* ------------------------- Operations on trees ------------------------- */ - -/* Insert chunk into tree */ -#define insert_large_chunk(M, X, S) {\ - tbinptr* H;\ - bindex_t I;\ - compute_tree_index(S, I);\ - H = treebin_at(M, I);\ - X->index = I;\ - X->child[0] = X->child[1] = 0;\ - if (!treemap_is_marked(M, I)) {\ - mark_treemap(M, I);\ - *H = X;\ - X->parent = (tchunkptr)H;\ - X->fd = X->bk = X;\ - }\ - else {\ - tchunkptr T = *H;\ - size_t K = S << leftshift_for_tree_index(I);\ - for (;;) {\ - if (chunksize(T) != S) {\ - tchunkptr* C = &(T->child[(K >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1]);\ - K <<= 1;\ - if (*C != 0)\ - T = *C;\ - else if (RTCHECK(ok_address(M, C))) {\ - *C = X;\ - X->parent = T;\ - X->fd = X->bk = X;\ - break;\ - }\ - else {\ - CORRUPTION_ERROR_ACTION(M);\ - break;\ - }\ - }\ - else {\ - tchunkptr F = T->fd;\ - if (RTCHECK(ok_address(M, T) && ok_address(M, F))) {\ - T->fd = F->bk = X;\ - X->fd = F;\ - X->bk = T;\ - X->parent = 0;\ - break;\ - }\ - else {\ - CORRUPTION_ERROR_ACTION(M);\ - break;\ - }\ - }\ - }\ - }\ -} - -/* - Unlink steps: - - 1. If x is a chained node, unlink it from its same-sized fd/bk links - and choose its bk node as its replacement. - 2. If x was the last node of its size, but not a leaf node, it must - be replaced with a leaf node (not merely one with an open left or - right), to make sure that lefts and rights of descendents - correspond properly to bit masks. We use the rightmost descendent - of x. We could use any other leaf, but this is easy to locate and - tends to counteract removal of leftmosts elsewhere, and so keeps - paths shorter than minimally guaranteed. This doesn't loop much - because on average a node in a tree is near the bottom. - 3. If x is the base of a chain (i.e., has parent links) relink - x's parent and children to x's replacement (or null if none). -*/ - -#define unlink_large_chunk(M, X) {\ - tchunkptr XP = X->parent;\ - tchunkptr R;\ - if (X->bk != X) {\ - tchunkptr F = X->fd;\ - R = X->bk;\ - if (RTCHECK(ok_address(M, F) && F->bk == X && R->fd == X)) {\ - F->bk = R;\ - R->fd = F;\ - }\ - else {\ - CORRUPTION_ERROR_ACTION(M);\ - }\ - }\ - else {\ - tchunkptr* RP;\ - if (((R = *(RP = &(X->child[1]))) != 0) ||\ - ((R = *(RP = &(X->child[0]))) != 0)) {\ - tchunkptr* CP;\ - while ((*(CP = &(R->child[1])) != 0) ||\ - (*(CP = &(R->child[0])) != 0)) {\ - R = *(RP = CP);\ - }\ - if (RTCHECK(ok_address(M, RP)))\ - *RP = 0;\ - else {\ - CORRUPTION_ERROR_ACTION(M);\ - }\ - }\ - }\ - if (XP != 0) {\ - tbinptr* H = treebin_at(M, X->index);\ - if (X == *H) {\ - if ((*H = R) == 0) \ - clear_treemap(M, X->index);\ - }\ - else if (RTCHECK(ok_address(M, XP))) {\ - if (XP->child[0] == X) \ - XP->child[0] = R;\ - else \ - XP->child[1] = R;\ - }\ - else\ - CORRUPTION_ERROR_ACTION(M);\ - if (R != 0) {\ - if (RTCHECK(ok_address(M, R))) {\ - tchunkptr C0, C1;\ - R->parent = XP;\ - if ((C0 = X->child[0]) != 0) {\ - if (RTCHECK(ok_address(M, C0))) {\ - R->child[0] = C0;\ - C0->parent = R;\ - }\ - else\ - CORRUPTION_ERROR_ACTION(M);\ - }\ - if ((C1 = X->child[1]) != 0) {\ - if (RTCHECK(ok_address(M, C1))) {\ - R->child[1] = C1;\ - C1->parent = R;\ - }\ - else\ - CORRUPTION_ERROR_ACTION(M);\ - }\ - }\ - else\ - CORRUPTION_ERROR_ACTION(M);\ - }\ - }\ -} - -/* Relays to large vs small bin operations */ - -#define insert_chunk(M, P, S)\ - if (is_small(S)) insert_small_chunk(M, P, S)\ - else { tchunkptr TP = (tchunkptr)(P); insert_large_chunk(M, TP, S); } - -#define unlink_chunk(M, P, S)\ - if (is_small(S)) unlink_small_chunk(M, P, S)\ - else { tchunkptr TP = (tchunkptr)(P); unlink_large_chunk(M, TP); } - - -/* Relays to internal calls to malloc/free from realloc, memalign etc */ - -#if ONLY_MSPACES -#define internal_malloc(m, b) mspace_malloc(m, b) -#define internal_free(m, mem) mspace_free(m,mem); -#else /* ONLY_MSPACES */ -#if MSPACES -#define internal_malloc(m, b)\ - ((m == gm)? dlmalloc(b) : mspace_malloc(m, b)) -#define internal_free(m, mem)\ - if (m == gm) dlfree(mem); else mspace_free(m,mem); -#else /* MSPACES */ -#define internal_malloc(m, b) dlmalloc(b) -#define internal_free(m, mem) dlfree(mem) -#endif /* MSPACES */ -#endif /* ONLY_MSPACES */ - -/* ----------------------- Direct-mmapping chunks ----------------------- */ - -/* - Directly mmapped chunks are set up with an offset to the start of - the mmapped region stored in the prev_foot field of the chunk. This - allows reconstruction of the required argument to MUNMAP when freed, - and also allows adjustment of the returned chunk to meet alignment - requirements (especially in memalign). -*/ - -/* Malloc using mmap */ -static void* mmap_alloc(mstate m, size_t nb) { - size_t mmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK); - if (m->footprint_limit != 0) { - size_t fp = m->footprint + mmsize; - if (fp <= m->footprint || fp > m->footprint_limit) - return 0; - } - if (mmsize > nb) { /* Check for wrap around 0 */ - char* mm = (char*)(CALL_DIRECT_MMAP(mmsize)); - if (mm != CMFAIL) { - size_t offset = align_offset(chunk2mem(mm)); - size_t psize = mmsize - offset - MMAP_FOOT_PAD; - mchunkptr p = (mchunkptr)(mm + offset); - p->prev_foot = offset; - p->head = psize; - mark_inuse_foot(m, p, psize); - chunk_plus_offset(p, psize)->head = FENCEPOST_HEAD; - chunk_plus_offset(p, psize+SIZE_T_SIZE)->head = 0; - - if (m->least_addr == 0 || mm < m->least_addr) - m->least_addr = mm; - if ((m->footprint += mmsize) > m->max_footprint) - m->max_footprint = m->footprint; - assert(is_aligned(chunk2mem(p))); - check_mmapped_chunk(m, p); - return chunk2mem(p); - } - } - return 0; -} - -/* Realloc using mmap */ -static mchunkptr mmap_resize(mstate m, mchunkptr oldp, size_t nb, int flags) { - size_t oldsize = chunksize(oldp); - (void)flags; /* placate people compiling -Wunused */ - if (is_small(nb)) /* Can't shrink mmap regions below small size */ - return 0; - /* Keep old chunk if big enough but not too big */ - if (oldsize >= nb + SIZE_T_SIZE && - (oldsize - nb) <= (mparams.granularity << 1)) - return oldp; - else { - size_t offset = oldp->prev_foot; - size_t oldmmsize = oldsize + offset + MMAP_FOOT_PAD; - size_t newmmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK); - char* cp = (char*)CALL_MREMAP((char*)oldp - offset, - oldmmsize, newmmsize, flags); - if (cp != CMFAIL) { - mchunkptr newp = (mchunkptr)(cp + offset); - size_t psize = newmmsize - offset - MMAP_FOOT_PAD; - newp->head = psize; - mark_inuse_foot(m, newp, psize); - chunk_plus_offset(newp, psize)->head = FENCEPOST_HEAD; - chunk_plus_offset(newp, psize+SIZE_T_SIZE)->head = 0; - - if (cp < m->least_addr) - m->least_addr = cp; - if ((m->footprint += newmmsize - oldmmsize) > m->max_footprint) - m->max_footprint = m->footprint; - check_mmapped_chunk(m, newp); - return newp; - } - } - return 0; -} - - -/* -------------------------- mspace management -------------------------- */ - -/* Initialize top chunk and its size */ -static void init_top(mstate m, mchunkptr p, size_t psize) { - /* Ensure alignment */ - size_t offset = align_offset(chunk2mem(p)); - p = (mchunkptr)((char*)p + offset); - psize -= offset; - - m->top = p; - m->topsize = psize; - p->head = psize | PINUSE_BIT; - /* set size of fake trailing chunk holding overhead space only once */ - chunk_plus_offset(p, psize)->head = TOP_FOOT_SIZE; - m->trim_check = mparams.trim_threshold; /* reset on each update */ -} - -/* Initialize bins for a new mstate that is otherwise zeroed out */ -static void init_bins(mstate m) { - /* Establish circular links for smallbins */ - bindex_t i; - for (i = 0; i < NSMALLBINS; ++i) { - sbinptr bin = smallbin_at(m,i); - bin->fd = bin->bk = bin; - } -} - -#if PROCEED_ON_ERROR - -/* default corruption action */ -static void reset_on_error(mstate m) { - int i; - ++malloc_corruption_error_count; - /* Reinitialize fields to forget about all memory */ - m->smallmap = m->treemap = 0; - m->dvsize = m->topsize = 0; - m->seg.base = 0; - m->seg.size = 0; - m->seg.next = 0; - m->top = m->dv = 0; - for (i = 0; i < NTREEBINS; ++i) - *treebin_at(m, i) = 0; - init_bins(m); -} -#endif /* PROCEED_ON_ERROR */ - -/* Allocate chunk and prepend remainder with chunk in successor base. */ -static void* prepend_alloc(mstate m, char* newbase, char* oldbase, - size_t nb) { - mchunkptr p = align_as_chunk(newbase); - mchunkptr oldfirst = align_as_chunk(oldbase); - size_t psize = (char*)oldfirst - (char*)p; - mchunkptr q = chunk_plus_offset(p, nb); - size_t qsize = psize - nb; - set_size_and_pinuse_of_inuse_chunk(m, p, nb); - - assert((char*)oldfirst > (char*)q); - assert(pinuse(oldfirst)); - assert(qsize >= MIN_CHUNK_SIZE); - - /* consolidate remainder with first chunk of old base */ - if (oldfirst == m->top) { - size_t tsize = m->topsize += qsize; - m->top = q; - q->head = tsize | PINUSE_BIT; - check_top_chunk(m, q); - } - else if (oldfirst == m->dv) { - size_t dsize = m->dvsize += qsize; - m->dv = q; - set_size_and_pinuse_of_free_chunk(q, dsize); - } - else { - if (!is_inuse(oldfirst)) { - size_t nsize = chunksize(oldfirst); - unlink_chunk(m, oldfirst, nsize); - oldfirst = chunk_plus_offset(oldfirst, nsize); - qsize += nsize; - } - set_free_with_pinuse(q, qsize, oldfirst); - insert_chunk(m, q, qsize); - check_free_chunk(m, q); - } - - check_malloced_chunk(m, chunk2mem(p), nb); - return chunk2mem(p); -} - -/* Add a segment to hold a new noncontiguous region */ -static void add_segment(mstate m, char* tbase, size_t tsize, flag_t mmapped) { - /* Determine locations and sizes of segment, fenceposts, old top */ - char* old_top = (char*)m->top; - msegmentptr oldsp = segment_holding(m, old_top); - char* old_end = oldsp->base + oldsp->size; - size_t ssize = pad_request(sizeof(struct malloc_segment)); - char* rawsp = old_end - (ssize + FOUR_SIZE_T_SIZES + CHUNK_ALIGN_MASK); - size_t offset = align_offset(chunk2mem(rawsp)); - char* asp = rawsp + offset; - char* csp = (asp < (old_top + MIN_CHUNK_SIZE))? old_top : asp; - mchunkptr sp = (mchunkptr)csp; - msegmentptr ss = (msegmentptr)(chunk2mem(sp)); - mchunkptr tnext = chunk_plus_offset(sp, ssize); - mchunkptr p = tnext; - int nfences = 0; - - /* reset top to new space */ - init_top(m, (mchunkptr)tbase, tsize - TOP_FOOT_SIZE); - - /* Set up segment record */ - assert(is_aligned(ss)); - set_size_and_pinuse_of_inuse_chunk(m, sp, ssize); - *ss = m->seg; /* Push current record */ - m->seg.base = tbase; - m->seg.size = tsize; - m->seg.sflags = mmapped; - m->seg.next = ss; - - /* Insert trailing fenceposts */ - for (;;) { - mchunkptr nextp = chunk_plus_offset(p, SIZE_T_SIZE); - p->head = FENCEPOST_HEAD; - ++nfences; - if ((char*)(&(nextp->head)) < old_end) - p = nextp; - else - break; - } - assert(nfences >= 2); - - /* Insert the rest of old top into a bin as an ordinary free chunk */ - if (csp != old_top) { - mchunkptr q = (mchunkptr)old_top; - size_t psize = csp - old_top; - mchunkptr tn = chunk_plus_offset(q, psize); - set_free_with_pinuse(q, psize, tn); - insert_chunk(m, q, psize); - } - - check_top_chunk(m, m->top); -} - -/* -------------------------- System allocation -------------------------- */ - -/* Get memory from system using MORECORE or MMAP */ -static void* sys_alloc(mstate m, size_t nb) { - char* tbase = CMFAIL; - size_t tsize = 0; - flag_t mmap_flag = 0; - size_t asize; /* allocation size */ - - ensure_initialization(); - - /* Directly map large chunks, but only if already initialized */ - if (use_mmap(m) && nb >= mparams.mmap_threshold && m->topsize != 0) { - void* mem = mmap_alloc(m, nb); - if (mem != 0) - return mem; - } - - asize = granularity_align(nb + SYS_ALLOC_PADDING); - if (asize <= nb) - return 0; /* wraparound */ - if (m->footprint_limit != 0) { - size_t fp = m->footprint + asize; - if (fp <= m->footprint || fp > m->footprint_limit) - return 0; - } - - /* - Try getting memory in any of three ways (in most-preferred to - least-preferred order): - 1. A call to MORECORE that can normally contiguously extend memory. - (disabled if not MORECORE_CONTIGUOUS or not HAVE_MORECORE or - or main space is mmapped or a previous contiguous call failed) - 2. A call to MMAP new space (disabled if not HAVE_MMAP). - Note that under the default settings, if MORECORE is unable to - fulfill a request, and HAVE_MMAP is true, then mmap is - used as a noncontiguous system allocator. This is a useful backup - strategy for systems with holes in address spaces -- in this case - sbrk cannot contiguously expand the heap, but mmap may be able to - find space. - 3. A call to MORECORE that cannot usually contiguously extend memory. - (disabled if not HAVE_MORECORE) - - In all cases, we need to request enough bytes from system to ensure - we can malloc nb bytes upon success, so pad with enough space for - top_foot, plus alignment-pad to make sure we don't lose bytes if - not on boundary, and round this up to a granularity unit. - */ - - if (MORECORE_CONTIGUOUS && !use_noncontiguous(m)) { - char* br = CMFAIL; - size_t ssize = asize; /* sbrk call size */ - msegmentptr ss = (m->top == 0)? 0 : segment_holding(m, (char*)m->top); - ACQUIRE_MALLOC_GLOBAL_LOCK(); - - if (ss == 0) { /* First time through or recovery */ - char* base = (char*)CALL_MORECORE(0); - if (base != CMFAIL) { - size_t fp; - /* Adjust to end on a page boundary */ - if (!is_page_aligned(base)) - ssize += (page_align((size_t)base) - (size_t)base); - fp = m->footprint + ssize; /* recheck limits */ - if (ssize > nb && ssize < HALF_MAX_SIZE_T && - (m->footprint_limit == 0 || - (fp > m->footprint && fp <= m->footprint_limit)) && - (br = (char*)(CALL_MORECORE(ssize))) == base) { - tbase = base; - tsize = ssize; - } - } - } - else { - /* Subtract out existing available top space from MORECORE request. */ - ssize = granularity_align(nb - m->topsize + SYS_ALLOC_PADDING); - /* Use mem here only if it did continuously extend old space */ - if (ssize < HALF_MAX_SIZE_T && - (br = (char*)(CALL_MORECORE(ssize))) == ss->base+ss->size) { - tbase = br; - tsize = ssize; - } - } - - if (tbase == CMFAIL) { /* Cope with partial failure */ - if (br != CMFAIL) { /* Try to use/extend the space we did get */ - if (ssize < HALF_MAX_SIZE_T && - ssize < nb + SYS_ALLOC_PADDING) { - size_t esize = granularity_align(nb + SYS_ALLOC_PADDING - ssize); - if (esize < HALF_MAX_SIZE_T) { - char* end = (char*)CALL_MORECORE(esize); - if (end != CMFAIL) - ssize += esize; - else { /* Can't use; try to release */ - (void) CALL_MORECORE(-ssize); - br = CMFAIL; - } - } - } - } - if (br != CMFAIL) { /* Use the space we did get */ - tbase = br; - tsize = ssize; - } - else - disable_contiguous(m); /* Don't try contiguous path in the future */ - } - - RELEASE_MALLOC_GLOBAL_LOCK(); - } - - if (HAVE_MMAP && tbase == CMFAIL) { /* Try MMAP */ - char* mp = (char*)(CALL_MMAP(asize)); - if (mp != CMFAIL) { - tbase = mp; - tsize = asize; - mmap_flag = USE_MMAP_BIT; - } - } - - if (HAVE_MORECORE && tbase == CMFAIL) { /* Try noncontiguous MORECORE */ - if (asize < HALF_MAX_SIZE_T) { - char* br = CMFAIL; - char* end = CMFAIL; - ACQUIRE_MALLOC_GLOBAL_LOCK(); - br = (char*)(CALL_MORECORE(asize)); - end = (char*)(CALL_MORECORE(0)); - RELEASE_MALLOC_GLOBAL_LOCK(); - if (br != CMFAIL && end != CMFAIL && br < end) { - size_t ssize = end - br; - if (ssize > nb + TOP_FOOT_SIZE) { - tbase = br; - tsize = ssize; - } - } - } - } - - if (tbase != CMFAIL) { - - if ((m->footprint += tsize) > m->max_footprint) - m->max_footprint = m->footprint; - - if (!is_initialized(m)) { /* first-time initialization */ - if (m->least_addr == 0 || tbase < m->least_addr) - m->least_addr = tbase; - m->seg.base = tbase; - m->seg.size = tsize; - m->seg.sflags = mmap_flag; - m->magic = mparams.magic; - m->release_checks = MAX_RELEASE_CHECK_RATE; - init_bins(m); -#if !ONLY_MSPACES - if (is_global(m)) - init_top(m, (mchunkptr)tbase, tsize - TOP_FOOT_SIZE); - else -#endif - { - /* Offset top by embedded malloc_state */ - mchunkptr mn = next_chunk(mem2chunk(m)); - init_top(m, mn, (size_t)((tbase + tsize) - (char*)mn) -TOP_FOOT_SIZE); - } - } - - else { - /* Try to merge with an existing segment */ - msegmentptr sp = &m->seg; - /* Only consider most recent segment if traversal suppressed */ - while (sp != 0 && tbase != sp->base + sp->size) - sp = (NO_SEGMENT_TRAVERSAL) ? 0 : sp->next; - if (sp != 0 && - !is_extern_segment(sp) && - (sp->sflags & USE_MMAP_BIT) == mmap_flag && - segment_holds(sp, m->top)) { /* append */ - sp->size += tsize; - init_top(m, m->top, m->topsize + tsize); - } - else { - if (tbase < m->least_addr) - m->least_addr = tbase; - sp = &m->seg; - while (sp != 0 && sp->base != tbase + tsize) - sp = (NO_SEGMENT_TRAVERSAL) ? 0 : sp->next; - if (sp != 0 && - !is_extern_segment(sp) && - (sp->sflags & USE_MMAP_BIT) == mmap_flag) { - char* oldbase = sp->base; - sp->base = tbase; - sp->size += tsize; - return prepend_alloc(m, tbase, oldbase, nb); - } - else - add_segment(m, tbase, tsize, mmap_flag); - } - } - - if (nb < m->topsize) { /* Allocate from new or extended top space */ - size_t rsize = m->topsize -= nb; - mchunkptr p = m->top; - mchunkptr r = m->top = chunk_plus_offset(p, nb); - r->head = rsize | PINUSE_BIT; - set_size_and_pinuse_of_inuse_chunk(m, p, nb); - check_top_chunk(m, m->top); - check_malloced_chunk(m, chunk2mem(p), nb); - return chunk2mem(p); - } - } - - MALLOC_FAILURE_ACTION; - return 0; -} - -/* ----------------------- system deallocation -------------------------- */ - -/* Unmap and unlink any mmapped segments that don't contain used chunks */ -static size_t release_unused_segments(mstate m) { - size_t released = 0; - int nsegs = 0; - msegmentptr pred = &m->seg; - msegmentptr sp = pred->next; - while (sp != 0) { - char* base = sp->base; - size_t size = sp->size; - msegmentptr next = sp->next; - ++nsegs; - if (is_mmapped_segment(sp) && !is_extern_segment(sp)) { - mchunkptr p = align_as_chunk(base); - size_t psize = chunksize(p); - /* Can unmap if first chunk holds entire segment and not pinned */ - if (!is_inuse(p) && (char*)p + psize >= base + size - TOP_FOOT_SIZE) { - tchunkptr tp = (tchunkptr)p; - assert(segment_holds(sp, (char*)sp)); - if (p == m->dv) { - m->dv = 0; - m->dvsize = 0; - } - else { - unlink_large_chunk(m, tp); - } - if (CALL_MUNMAP(base, size) == 0) { - released += size; - m->footprint -= size; - /* unlink obsoleted record */ - sp = pred; - sp->next = next; - } - else { /* back out if cannot unmap */ - insert_large_chunk(m, tp, psize); - } - } - } - if (NO_SEGMENT_TRAVERSAL) /* scan only first segment */ - break; - pred = sp; - sp = next; - } - /* Reset check counter */ - m->release_checks = (((size_t) nsegs > (size_t) MAX_RELEASE_CHECK_RATE)? - (size_t) nsegs : (size_t) MAX_RELEASE_CHECK_RATE); - return released; -} - -static int sys_trim(mstate m, size_t pad) { - size_t released = 0; - ensure_initialization(); - if (pad < MAX_REQUEST && is_initialized(m)) { - pad += TOP_FOOT_SIZE; /* ensure enough room for segment overhead */ - - if (m->topsize > pad) { - /* Shrink top space in granularity-size units, keeping at least one */ - size_t unit = mparams.granularity; - size_t extra = ((m->topsize - pad + (unit - SIZE_T_ONE)) / unit - - SIZE_T_ONE) * unit; - msegmentptr sp = segment_holding(m, (char*)m->top); - - if (!is_extern_segment(sp)) { - if (is_mmapped_segment(sp)) { - if (HAVE_MMAP && - sp->size >= extra && - !has_segment_link(m, sp)) { /* can't shrink if pinned */ - size_t newsize = sp->size - extra; - (void)newsize; /* placate people compiling -Wunused-variable */ - /* Prefer mremap, fall back to munmap */ - if ((CALL_MREMAP(sp->base, sp->size, newsize, 0) != MFAIL) || - (CALL_MUNMAP(sp->base + newsize, extra) == 0)) { - released = extra; - } - } - } - else if (HAVE_MORECORE) { - if (extra >= HALF_MAX_SIZE_T) /* Avoid wrapping negative */ - extra = (HALF_MAX_SIZE_T) + SIZE_T_ONE - unit; - ACQUIRE_MALLOC_GLOBAL_LOCK(); - { - /* Make sure end of memory is where we last set it. */ - char* old_br = (char*)(CALL_MORECORE(0)); - if (old_br == sp->base + sp->size) { - char* rel_br = (char*)(CALL_MORECORE(-extra)); - char* new_br = (char*)(CALL_MORECORE(0)); - if (rel_br != CMFAIL && new_br < old_br) - released = old_br - new_br; - } - } - RELEASE_MALLOC_GLOBAL_LOCK(); - } - } - - if (released != 0) { - sp->size -= released; - m->footprint -= released; - init_top(m, m->top, m->topsize - released); - check_top_chunk(m, m->top); - } - } - - /* Unmap any unused mmapped segments */ - if (HAVE_MMAP) - released += release_unused_segments(m); - - /* On failure, disable autotrim to avoid repeated failed future calls */ - if (released == 0 && m->topsize > m->trim_check) - m->trim_check = MAX_SIZE_T; - } - - return (released != 0)? 1 : 0; -} - -/* Consolidate and bin a chunk. Differs from exported versions - of free mainly in that the chunk need not be marked as inuse. -*/ -static void dispose_chunk(mstate m, mchunkptr p, size_t psize) { - mchunkptr next = chunk_plus_offset(p, psize); - if (!pinuse(p)) { - mchunkptr prev; - size_t prevsize = p->prev_foot; - if (is_mmapped(p)) { - psize += prevsize + MMAP_FOOT_PAD; - if (CALL_MUNMAP((char*)p - prevsize, psize) == 0) - m->footprint -= psize; - return; - } - prev = chunk_minus_offset(p, prevsize); - psize += prevsize; - p = prev; - if (RTCHECK(ok_address(m, prev))) { /* consolidate backward */ - if (p != m->dv) { - unlink_chunk(m, p, prevsize); - } - else if ((next->head & INUSE_BITS) == INUSE_BITS) { - m->dvsize = psize; - set_free_with_pinuse(p, psize, next); - return; - } - } - else { - CORRUPTION_ERROR_ACTION(m); - return; - } - } - if (RTCHECK(ok_address(m, next))) { - if (!cinuse(next)) { /* consolidate forward */ - if (next == m->top) { - size_t tsize = m->topsize += psize; - m->top = p; - p->head = tsize | PINUSE_BIT; - if (p == m->dv) { - m->dv = 0; - m->dvsize = 0; - } - return; - } - else if (next == m->dv) { - size_t dsize = m->dvsize += psize; - m->dv = p; - set_size_and_pinuse_of_free_chunk(p, dsize); - return; - } - else { - size_t nsize = chunksize(next); - psize += nsize; - unlink_chunk(m, next, nsize); - set_size_and_pinuse_of_free_chunk(p, psize); - if (p == m->dv) { - m->dvsize = psize; - return; - } - } - } - else { - set_free_with_pinuse(p, psize, next); - } - insert_chunk(m, p, psize); - } - else { - CORRUPTION_ERROR_ACTION(m); - } -} - -/* ---------------------------- malloc --------------------------- */ - -/* allocate a large request from the best fitting chunk in a treebin */ -static void* tmalloc_large(mstate m, size_t nb) { - tchunkptr v = 0; - size_t rsize = -nb; /* Unsigned negation */ - tchunkptr t; - bindex_t idx; - compute_tree_index(nb, idx); - if ((t = *treebin_at(m, idx)) != 0) { - /* Traverse tree for this bin looking for node with size == nb */ - size_t sizebits = nb << leftshift_for_tree_index(idx); - tchunkptr rst = 0; /* The deepest untaken right subtree */ - for (;;) { - tchunkptr rt; - size_t trem = chunksize(t) - nb; - if (trem < rsize) { - v = t; - if ((rsize = trem) == 0) - break; - } - rt = t->child[1]; - t = t->child[(sizebits >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1]; - if (rt != 0 && rt != t) - rst = rt; - if (t == 0) { - t = rst; /* set t to least subtree holding sizes > nb */ - break; - } - sizebits <<= 1; - } - } - if (t == 0 && v == 0) { /* set t to root of next non-empty treebin */ - binmap_t leftbits = left_bits(idx2bit(idx)) & m->treemap; - if (leftbits != 0) { - bindex_t i; - binmap_t leastbit = least_bit(leftbits); - compute_bit2idx(leastbit, i); - t = *treebin_at(m, i); - } - } - - while (t != 0) { /* find smallest of tree or subtree */ - size_t trem = chunksize(t) - nb; - if (trem < rsize) { - rsize = trem; - v = t; - } - t = leftmost_child(t); - } - - /* If dv is a better fit, return 0 so malloc will use it */ - if (v != 0 && rsize < (size_t)(m->dvsize - nb)) { - if (RTCHECK(ok_address(m, v))) { /* split */ - mchunkptr r = chunk_plus_offset(v, nb); - assert(chunksize(v) == rsize + nb); - if (RTCHECK(ok_next(v, r))) { - unlink_large_chunk(m, v); - if (rsize < MIN_CHUNK_SIZE) - set_inuse_and_pinuse(m, v, (rsize + nb)); - else { - set_size_and_pinuse_of_inuse_chunk(m, v, nb); - set_size_and_pinuse_of_free_chunk(r, rsize); - insert_chunk(m, r, rsize); - } - return chunk2mem(v); - } - } - CORRUPTION_ERROR_ACTION(m); - } - return 0; -} - -/* allocate a small request from the best fitting chunk in a treebin */ -static void* tmalloc_small(mstate m, size_t nb) { - tchunkptr t, v; - size_t rsize; - bindex_t i; - binmap_t leastbit = least_bit(m->treemap); - compute_bit2idx(leastbit, i); - v = t = *treebin_at(m, i); - rsize = chunksize(t) - nb; - - while ((t = leftmost_child(t)) != 0) { - size_t trem = chunksize(t) - nb; - if (trem < rsize) { - rsize = trem; - v = t; - } - } - - if (RTCHECK(ok_address(m, v))) { - mchunkptr r = chunk_plus_offset(v, nb); - assert(chunksize(v) == rsize + nb); - if (RTCHECK(ok_next(v, r))) { - unlink_large_chunk(m, v); - if (rsize < MIN_CHUNK_SIZE) - set_inuse_and_pinuse(m, v, (rsize + nb)); - else { - set_size_and_pinuse_of_inuse_chunk(m, v, nb); - set_size_and_pinuse_of_free_chunk(r, rsize); - replace_dv(m, r, rsize); - } - return chunk2mem(v); - } - } - - CORRUPTION_ERROR_ACTION(m); - return 0; -} - -#if !ONLY_MSPACES - -void* dlmalloc(size_t bytes) { - /* - Basic algorithm: - If a small request (< 256 bytes minus per-chunk overhead): - 1. If one exists, use a remainderless chunk in associated smallbin. - (Remainderless means that there are too few excess bytes to - represent as a chunk.) - 2. If it is big enough, use the dv chunk, which is normally the - chunk adjacent to the one used for the most recent small request. - 3. If one exists, split the smallest available chunk in a bin, - saving remainder in dv. - 4. If it is big enough, use the top chunk. - 5. If available, get memory from system and use it - Otherwise, for a large request: - 1. Find the smallest available binned chunk that fits, and use it - if it is better fitting than dv chunk, splitting if necessary. - 2. If better fitting than any binned chunk, use the dv chunk. - 3. If it is big enough, use the top chunk. - 4. If request size >= mmap threshold, try to directly mmap this chunk. - 5. If available, get memory from system and use it - - The ugly goto's here ensure that postaction occurs along all paths. - */ - -#if USE_LOCKS - ensure_initialization(); /* initialize in sys_alloc if not using locks */ -#endif - - if (!PREACTION(gm)) { - void* mem; - size_t nb; - if (bytes <= MAX_SMALL_REQUEST) { - bindex_t idx; - binmap_t smallbits; - nb = (bytes < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(bytes); - idx = small_index(nb); - smallbits = gm->smallmap >> idx; - - if ((smallbits & 0x3U) != 0) { /* Remainderless fit to a smallbin. */ - mchunkptr b, p; - idx += ~smallbits & 1; /* Uses next bin if idx empty */ - b = smallbin_at(gm, idx); - p = b->fd; - assert(chunksize(p) == small_index2size(idx)); - unlink_first_small_chunk(gm, b, p, idx); - set_inuse_and_pinuse(gm, p, small_index2size(idx)); - mem = chunk2mem(p); - check_malloced_chunk(gm, mem, nb); - goto postaction; - } - - else if (nb > gm->dvsize) { - if (smallbits != 0) { /* Use chunk in next nonempty smallbin */ - mchunkptr b, p, r; - size_t rsize; - bindex_t i; - binmap_t leftbits = (smallbits << idx) & left_bits(idx2bit(idx)); - binmap_t leastbit = least_bit(leftbits); - compute_bit2idx(leastbit, i); - b = smallbin_at(gm, i); - p = b->fd; - assert(chunksize(p) == small_index2size(i)); - unlink_first_small_chunk(gm, b, p, i); - rsize = small_index2size(i) - nb; - /* Fit here cannot be remainderless if 4byte sizes */ - if (SIZE_T_SIZE != 4 && rsize < MIN_CHUNK_SIZE) - set_inuse_and_pinuse(gm, p, small_index2size(i)); - else { - set_size_and_pinuse_of_inuse_chunk(gm, p, nb); - r = chunk_plus_offset(p, nb); - set_size_and_pinuse_of_free_chunk(r, rsize); - replace_dv(gm, r, rsize); - } - mem = chunk2mem(p); - check_malloced_chunk(gm, mem, nb); - goto postaction; - } - - else if (gm->treemap != 0 && (mem = tmalloc_small(gm, nb)) != 0) { - check_malloced_chunk(gm, mem, nb); - goto postaction; - } - } - } - else if (bytes >= MAX_REQUEST) - nb = MAX_SIZE_T; /* Too big to allocate. Force failure (in sys alloc) */ - else { - nb = pad_request(bytes); - if (gm->treemap != 0 && (mem = tmalloc_large(gm, nb)) != 0) { - check_malloced_chunk(gm, mem, nb); - goto postaction; - } - } - - if (nb <= gm->dvsize) { - size_t rsize = gm->dvsize - nb; - mchunkptr p = gm->dv; - if (rsize >= MIN_CHUNK_SIZE) { /* split dv */ - mchunkptr r = gm->dv = chunk_plus_offset(p, nb); - gm->dvsize = rsize; - set_size_and_pinuse_of_free_chunk(r, rsize); - set_size_and_pinuse_of_inuse_chunk(gm, p, nb); - } - else { /* exhaust dv */ - size_t dvs = gm->dvsize; - gm->dvsize = 0; - gm->dv = 0; - set_inuse_and_pinuse(gm, p, dvs); - } - mem = chunk2mem(p); - check_malloced_chunk(gm, mem, nb); - goto postaction; - } - - else if (nb < gm->topsize) { /* Split top */ - size_t rsize = gm->topsize -= nb; - mchunkptr p = gm->top; - mchunkptr r = gm->top = chunk_plus_offset(p, nb); - r->head = rsize | PINUSE_BIT; - set_size_and_pinuse_of_inuse_chunk(gm, p, nb); - mem = chunk2mem(p); - check_top_chunk(gm, gm->top); - check_malloced_chunk(gm, mem, nb); - goto postaction; - } - - mem = sys_alloc(gm, nb); - - postaction: - POSTACTION(gm); - return mem; - } - - return 0; -} - -/* ---------------------------- free --------------------------- */ - -void dlfree(void* mem) { - /* - Consolidate freed chunks with preceeding or succeeding bordering - free chunks, if they exist, and then place in a bin. Intermixed - with special cases for top, dv, mmapped chunks, and usage errors. - */ - - if (mem != 0) { - mchunkptr p = mem2chunk(mem); -#if FOOTERS - mstate fm = get_mstate_for(p); - if (!ok_magic(fm)) { - USAGE_ERROR_ACTION(fm, p); - return; - } -#else /* FOOTERS */ -#define fm gm -#endif /* FOOTERS */ - if (!PREACTION(fm)) { - check_inuse_chunk(fm, p); - if (RTCHECK(ok_address(fm, p) && ok_inuse(p))) { - size_t psize = chunksize(p); - mchunkptr next = chunk_plus_offset(p, psize); - if (!pinuse(p)) { - size_t prevsize = p->prev_foot; - if (is_mmapped(p)) { - psize += prevsize + MMAP_FOOT_PAD; - if (CALL_MUNMAP((char*)p - prevsize, psize) == 0) - fm->footprint -= psize; - goto postaction; - } - else { - mchunkptr prev = chunk_minus_offset(p, prevsize); - psize += prevsize; - p = prev; - if (RTCHECK(ok_address(fm, prev))) { /* consolidate backward */ - if (p != fm->dv) { - unlink_chunk(fm, p, prevsize); - } - else if ((next->head & INUSE_BITS) == INUSE_BITS) { - fm->dvsize = psize; - set_free_with_pinuse(p, psize, next); - goto postaction; - } - } - else - goto erroraction; - } - } - - if (RTCHECK(ok_next(p, next) && ok_pinuse(next))) { - if (!cinuse(next)) { /* consolidate forward */ - if (next == fm->top) { - size_t tsize = fm->topsize += psize; - fm->top = p; - p->head = tsize | PINUSE_BIT; - if (p == fm->dv) { - fm->dv = 0; - fm->dvsize = 0; - } - if (should_trim(fm, tsize)) - sys_trim(fm, 0); - goto postaction; - } - else if (next == fm->dv) { - size_t dsize = fm->dvsize += psize; - fm->dv = p; - set_size_and_pinuse_of_free_chunk(p, dsize); - goto postaction; - } - else { - size_t nsize = chunksize(next); - psize += nsize; - unlink_chunk(fm, next, nsize); - set_size_and_pinuse_of_free_chunk(p, psize); - if (p == fm->dv) { - fm->dvsize = psize; - goto postaction; - } - } - } - else - set_free_with_pinuse(p, psize, next); - - if (is_small(psize)) { - insert_small_chunk(fm, p, psize); - check_free_chunk(fm, p); - } - else { - tchunkptr tp = (tchunkptr)p; - insert_large_chunk(fm, tp, psize); - check_free_chunk(fm, p); - if (--fm->release_checks == 0) - release_unused_segments(fm); - } - goto postaction; - } - } - erroraction: - USAGE_ERROR_ACTION(fm, p); - postaction: - POSTACTION(fm); - } - } -#if !FOOTERS -#undef fm -#endif /* FOOTERS */ -} - -void* dlcalloc(size_t n_elements, size_t elem_size) { - void* mem; - size_t req = 0; - if (n_elements != 0) { - req = n_elements * elem_size; - if (((n_elements | elem_size) & ~(size_t)0xffff) && - (req / n_elements != elem_size)) - req = MAX_SIZE_T; /* force downstream failure on overflow */ - } - mem = dlmalloc(req); - if (mem != 0 && calloc_must_clear(mem2chunk(mem))) - memset(mem, 0, req); - return mem; -} - -#endif /* !ONLY_MSPACES */ - -/* ------------ Internal support for realloc, memalign, etc -------------- */ - -/* Try to realloc; only in-place unless can_move true */ -static mchunkptr try_realloc_chunk(mstate m, mchunkptr p, size_t nb, - int can_move) { - mchunkptr newp = 0; - size_t oldsize = chunksize(p); - mchunkptr next = chunk_plus_offset(p, oldsize); - if (RTCHECK(ok_address(m, p) && ok_inuse(p) && - ok_next(p, next) && ok_pinuse(next))) { - if (is_mmapped(p)) { - newp = mmap_resize(m, p, nb, can_move); - } - else if (oldsize >= nb) { /* already big enough */ - size_t rsize = oldsize - nb; - if (rsize >= MIN_CHUNK_SIZE) { /* split off remainder */ - mchunkptr r = chunk_plus_offset(p, nb); - set_inuse(m, p, nb); - set_inuse(m, r, rsize); - dispose_chunk(m, r, rsize); - } - newp = p; - } - else if (next == m->top) { /* extend into top */ - if (oldsize + m->topsize > nb) { - size_t newsize = oldsize + m->topsize; - size_t newtopsize = newsize - nb; - mchunkptr newtop = chunk_plus_offset(p, nb); - set_inuse(m, p, nb); - newtop->head = newtopsize |PINUSE_BIT; - m->top = newtop; - m->topsize = newtopsize; - newp = p; - } - } - else if (next == m->dv) { /* extend into dv */ - size_t dvs = m->dvsize; - if (oldsize + dvs >= nb) { - size_t dsize = oldsize + dvs - nb; - if (dsize >= MIN_CHUNK_SIZE) { - mchunkptr r = chunk_plus_offset(p, nb); - mchunkptr n = chunk_plus_offset(r, dsize); - set_inuse(m, p, nb); - set_size_and_pinuse_of_free_chunk(r, dsize); - clear_pinuse(n); - m->dvsize = dsize; - m->dv = r; - } - else { /* exhaust dv */ - size_t newsize = oldsize + dvs; - set_inuse(m, p, newsize); - m->dvsize = 0; - m->dv = 0; - } - newp = p; - } - } - else if (!cinuse(next)) { /* extend into next free chunk */ - size_t nextsize = chunksize(next); - if (oldsize + nextsize >= nb) { - size_t rsize = oldsize + nextsize - nb; - unlink_chunk(m, next, nextsize); - if (rsize < MIN_CHUNK_SIZE) { - size_t newsize = oldsize + nextsize; - set_inuse(m, p, newsize); - } - else { - mchunkptr r = chunk_plus_offset(p, nb); - set_inuse(m, p, nb); - set_inuse(m, r, rsize); - dispose_chunk(m, r, rsize); - } - newp = p; - } - } - } - else { - USAGE_ERROR_ACTION(m, chunk2mem(p)); - } - return newp; -} - -static void* internal_memalign(mstate m, size_t alignment, size_t bytes) { - void* mem = 0; - if (alignment < MIN_CHUNK_SIZE) /* must be at least a minimum chunk size */ - alignment = MIN_CHUNK_SIZE; - if ((alignment & (alignment-SIZE_T_ONE)) != 0) {/* Ensure a power of 2 */ - size_t a = MALLOC_ALIGNMENT << 1; - while (a < alignment) a <<= 1; - alignment = a; - } - if (bytes >= MAX_REQUEST - alignment) { - if (m != 0) { /* Test isn't needed but avoids compiler warning */ - MALLOC_FAILURE_ACTION; - } - } - else { - size_t nb = request2size(bytes); - size_t req = nb + alignment + MIN_CHUNK_SIZE - CHUNK_OVERHEAD; - mem = internal_malloc(m, req); - if (mem != 0) { - mchunkptr p = mem2chunk(mem); - if (PREACTION(m)) - return 0; - if ((((size_t)(mem)) & (alignment - 1)) != 0) { /* misaligned */ - /* - Find an aligned spot inside chunk. Since we need to give - back leading space in a chunk of at least MIN_CHUNK_SIZE, if - the first calculation places us at a spot with less than - MIN_CHUNK_SIZE leader, we can move to the next aligned spot. - We've allocated enough total room so that this is always - possible. - */ - char* br = (char*)mem2chunk((size_t)(((size_t)((char*)mem + alignment - - SIZE_T_ONE)) & - -alignment)); - char* pos = ((size_t)(br - (char*)(p)) >= MIN_CHUNK_SIZE)? - br : br+alignment; - mchunkptr newp = (mchunkptr)pos; - size_t leadsize = pos - (char*)(p); - size_t newsize = chunksize(p) - leadsize; - - if (is_mmapped(p)) { /* For mmapped chunks, just adjust offset */ - newp->prev_foot = p->prev_foot + leadsize; - newp->head = newsize; - } - else { /* Otherwise, give back leader, use the rest */ - set_inuse(m, newp, newsize); - set_inuse(m, p, leadsize); - dispose_chunk(m, p, leadsize); - } - p = newp; - } - - /* Give back spare room at the end */ - if (!is_mmapped(p)) { - size_t size = chunksize(p); - if (size > nb + MIN_CHUNK_SIZE) { - size_t remainder_size = size - nb; - mchunkptr remainder = chunk_plus_offset(p, nb); - set_inuse(m, p, nb); - set_inuse(m, remainder, remainder_size); - dispose_chunk(m, remainder, remainder_size); - } - } - - mem = chunk2mem(p); - assert (chunksize(p) >= nb); - assert(((size_t)mem & (alignment - 1)) == 0); - check_inuse_chunk(m, p); - POSTACTION(m); - } - } - return mem; -} - -/* - Common support for independent_X routines, handling - all of the combinations that can result. - The opts arg has: - bit 0 set if all elements are same size (using sizes[0]) - bit 1 set if elements should be zeroed -*/ -static void** ialloc(mstate m, - size_t n_elements, - size_t* sizes, - int opts, - void* chunks[]) { - - size_t element_size; /* chunksize of each element, if all same */ - size_t contents_size; /* total size of elements */ - size_t array_size; /* request size of pointer array */ - void* mem; /* malloced aggregate space */ - mchunkptr p; /* corresponding chunk */ - size_t remainder_size; /* remaining bytes while splitting */ - void** marray; /* either "chunks" or malloced ptr array */ - mchunkptr array_chunk; /* chunk for malloced ptr array */ - flag_t was_enabled; /* to disable mmap */ - size_t size; - size_t i; - - ensure_initialization(); - /* compute array length, if needed */ - if (chunks != 0) { - if (n_elements == 0) - return chunks; /* nothing to do */ - marray = chunks; - array_size = 0; - } - else { - /* if empty req, must still return chunk representing empty array */ - if (n_elements == 0) - return (void**)internal_malloc(m, 0); - marray = 0; - array_size = request2size(n_elements * (sizeof(void*))); - } - - /* compute total element size */ - if (opts & 0x1) { /* all-same-size */ - element_size = request2size(*sizes); - contents_size = n_elements * element_size; - } - else { /* add up all the sizes */ - element_size = 0; - contents_size = 0; - for (i = 0; i != n_elements; ++i) - contents_size += request2size(sizes[i]); - } - - size = contents_size + array_size; - - /* - Allocate the aggregate chunk. First disable direct-mmapping so - malloc won't use it, since we would not be able to later - free/realloc space internal to a segregated mmap region. - */ - was_enabled = use_mmap(m); - disable_mmap(m); - mem = internal_malloc(m, size - CHUNK_OVERHEAD); - if (was_enabled) - enable_mmap(m); - if (mem == 0) - return 0; - - if (PREACTION(m)) return 0; - p = mem2chunk(mem); - remainder_size = chunksize(p); - - assert(!is_mmapped(p)); - - if (opts & 0x2) { /* optionally clear the elements */ - memset((size_t*)mem, 0, remainder_size - SIZE_T_SIZE - array_size); - } - - /* If not provided, allocate the pointer array as final part of chunk */ - if (marray == 0) { - size_t array_chunk_size; - array_chunk = chunk_plus_offset(p, contents_size); - array_chunk_size = remainder_size - contents_size; - marray = (void**) (chunk2mem(array_chunk)); - set_size_and_pinuse_of_inuse_chunk(m, array_chunk, array_chunk_size); - remainder_size = contents_size; - } - - /* split out elements */ - for (i = 0; ; ++i) { - marray[i] = chunk2mem(p); - if (i != n_elements-1) { - if (element_size != 0) - size = element_size; - else - size = request2size(sizes[i]); - remainder_size -= size; - set_size_and_pinuse_of_inuse_chunk(m, p, size); - p = chunk_plus_offset(p, size); - } - else { /* the final element absorbs any overallocation slop */ - set_size_and_pinuse_of_inuse_chunk(m, p, remainder_size); - break; - } - } - -#if DEBUG - if (marray != chunks) { - /* final element must have exactly exhausted chunk */ - if (element_size != 0) { - assert(remainder_size == element_size); - } - else { - assert(remainder_size == request2size(sizes[i])); - } - check_inuse_chunk(m, mem2chunk(marray)); - } - for (i = 0; i != n_elements; ++i) - check_inuse_chunk(m, mem2chunk(marray[i])); - -#endif /* DEBUG */ - - POSTACTION(m); - return marray; -} - -/* Try to free all pointers in the given array. - Note: this could be made faster, by delaying consolidation, - at the price of disabling some user integrity checks, We - still optimize some consolidations by combining adjacent - chunks before freeing, which will occur often if allocated - with ialloc or the array is sorted. -*/ -static size_t internal_bulk_free(mstate m, void* array[], size_t nelem) { - size_t unfreed = 0; - if (!PREACTION(m)) { - void** a; - void** fence = &(array[nelem]); - for (a = array; a != fence; ++a) { - void* mem = *a; - if (mem != 0) { - mchunkptr p = mem2chunk(mem); - size_t psize = chunksize(p); -#if FOOTERS - if (get_mstate_for(p) != m) { - ++unfreed; - continue; - } -#endif - check_inuse_chunk(m, p); - *a = 0; - if (RTCHECK(ok_address(m, p) && ok_inuse(p))) { - void ** b = a + 1; /* try to merge with next chunk */ - mchunkptr next = next_chunk(p); - if (b != fence && *b == chunk2mem(next)) { - size_t newsize = chunksize(next) + psize; - set_inuse(m, p, newsize); - *b = chunk2mem(p); - } - else - dispose_chunk(m, p, psize); - } - else { - CORRUPTION_ERROR_ACTION(m); - break; - } - } - } - if (should_trim(m, m->topsize)) - sys_trim(m, 0); - POSTACTION(m); - } - return unfreed; -} - -/* Traversal */ -#if MALLOC_INSPECT_ALL -static void internal_inspect_all(mstate m, - void(*handler)(void *start, - void *end, - size_t used_bytes, - void* callback_arg), - void* arg) { - if (is_initialized(m)) { - mchunkptr top = m->top; - msegmentptr s; - for (s = &m->seg; s != 0; s = s->next) { - mchunkptr q = align_as_chunk(s->base); - while (segment_holds(s, q) && q->head != FENCEPOST_HEAD) { - mchunkptr next = next_chunk(q); - size_t sz = chunksize(q); - size_t used; - void* start; - if (is_inuse(q)) { - used = sz - CHUNK_OVERHEAD; /* must not be mmapped */ - start = chunk2mem(q); - } - else { - used = 0; - if (is_small(sz)) { /* offset by possible bookkeeping */ - start = (void*)((char*)q + sizeof(struct malloc_chunk)); - } - else { - start = (void*)((char*)q + sizeof(struct malloc_tree_chunk)); - } - } - if (start < (void*)next) /* skip if all space is bookkeeping */ - handler(start, next, used, arg); - if (q == top) - break; - q = next; - } - } - } -} -#endif /* MALLOC_INSPECT_ALL */ - -/* ------------------ Exported realloc, memalign, etc -------------------- */ - -#if !ONLY_MSPACES - -void* dlrealloc(void* oldmem, size_t bytes) { - void* mem = 0; - if (oldmem == 0) { - mem = dlmalloc(bytes); - } - else if (bytes >= MAX_REQUEST) { - MALLOC_FAILURE_ACTION; - } -#ifdef REALLOC_ZERO_BYTES_FREES - else if (bytes == 0) { - dlfree(oldmem); - } -#endif /* REALLOC_ZERO_BYTES_FREES */ - else { - size_t nb = request2size(bytes); - mchunkptr oldp = mem2chunk(oldmem); -#if ! FOOTERS - mstate m = gm; -#else /* FOOTERS */ - mstate m = get_mstate_for(oldp); - if (!ok_magic(m)) { - USAGE_ERROR_ACTION(m, oldmem); - return 0; - } -#endif /* FOOTERS */ - if (!PREACTION(m)) { - mchunkptr newp = try_realloc_chunk(m, oldp, nb, 1); - POSTACTION(m); - if (newp != 0) { - check_inuse_chunk(m, newp); - mem = chunk2mem(newp); - } - else { - mem = internal_malloc(m, bytes); - if (mem != 0) { - size_t oc = chunksize(oldp) - overhead_for(oldp); - memcpy(mem, oldmem, (oc < bytes)? oc : bytes); - internal_free(m, oldmem); - } - } - } - } - return mem; -} - -void* dlrealloc_in_place(void* oldmem, size_t bytes) { - void* mem = 0; - if (oldmem != 0) { - if (bytes >= MAX_REQUEST) { - MALLOC_FAILURE_ACTION; - } - else { - size_t nb = request2size(bytes); - mchunkptr oldp = mem2chunk(oldmem); -#if ! FOOTERS - mstate m = gm; -#else /* FOOTERS */ - mstate m = get_mstate_for(oldp); - if (!ok_magic(m)) { - USAGE_ERROR_ACTION(m, oldmem); - return 0; - } -#endif /* FOOTERS */ - if (!PREACTION(m)) { - mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0); - POSTACTION(m); - if (newp == oldp) { - check_inuse_chunk(m, newp); - mem = oldmem; - } - } - } - } - return mem; -} - -void* dlmemalign(size_t alignment, size_t bytes) { - if (alignment <= MALLOC_ALIGNMENT) { - return dlmalloc(bytes); - } - return internal_memalign(gm, alignment, bytes); -} - -int dlposix_memalign(void** pp, size_t alignment, size_t bytes) { - void* mem = 0; - if (alignment == MALLOC_ALIGNMENT) - mem = dlmalloc(bytes); - else { - size_t d = alignment / sizeof(void*); - size_t r = alignment % sizeof(void*); - if (r != 0 || d == 0 || (d & (d-SIZE_T_ONE)) != 0) - return EINVAL; - else if (bytes <= MAX_REQUEST - alignment) { - if (alignment < MIN_CHUNK_SIZE) - alignment = MIN_CHUNK_SIZE; - mem = internal_memalign(gm, alignment, bytes); - } - } - if (mem == 0) - return ENOMEM; - else { - *pp = mem; - return 0; - } -} - -void* dlvalloc(size_t bytes) { - size_t pagesz; - ensure_initialization(); - pagesz = mparams.page_size; - return dlmemalign(pagesz, bytes); -} - -void* dlpvalloc(size_t bytes) { - size_t pagesz; - ensure_initialization(); - pagesz = mparams.page_size; - return dlmemalign(pagesz, (bytes + pagesz - SIZE_T_ONE) & ~(pagesz - SIZE_T_ONE)); -} - -void** dlindependent_calloc(size_t n_elements, size_t elem_size, - void* chunks[]) { - size_t sz = elem_size; /* serves as 1-element array */ - return ialloc(gm, n_elements, &sz, 3, chunks); -} - -void** dlindependent_comalloc(size_t n_elements, size_t sizes[], - void* chunks[]) { - return ialloc(gm, n_elements, sizes, 0, chunks); -} - -size_t dlbulk_free(void* array[], size_t nelem) { - return internal_bulk_free(gm, array, nelem); -} - -#if MALLOC_INSPECT_ALL -void dlmalloc_inspect_all(void(*handler)(void *start, - void *end, - size_t used_bytes, - void* callback_arg), - void* arg) { - ensure_initialization(); - if (!PREACTION(gm)) { - internal_inspect_all(gm, handler, arg); - POSTACTION(gm); - } -} -#endif /* MALLOC_INSPECT_ALL */ - -int dlmalloc_trim(size_t pad) { - int result = 0; - ensure_initialization(); - if (!PREACTION(gm)) { - result = sys_trim(gm, pad); - POSTACTION(gm); - } - return result; -} - -size_t dlmalloc_footprint(void) { - return gm->footprint; -} - -size_t dlmalloc_max_footprint(void) { - return gm->max_footprint; -} - -size_t dlmalloc_footprint_limit(void) { - size_t maf = gm->footprint_limit; - return maf == 0 ? MAX_SIZE_T : maf; -} - -size_t dlmalloc_set_footprint_limit(size_t bytes) { - size_t result; /* invert sense of 0 */ - if (bytes == 0) - result = granularity_align(1); /* Use minimal size */ - if (bytes == MAX_SIZE_T) - result = 0; /* disable */ - else - result = granularity_align(bytes); - return gm->footprint_limit = result; -} - -#if !NO_MALLINFO -struct mallinfo dlmallinfo(void) { - return internal_mallinfo(gm); -} -#endif /* NO_MALLINFO */ - -#if !NO_MALLOC_STATS -void dlmalloc_stats() { - internal_malloc_stats(gm); -} -#endif /* NO_MALLOC_STATS */ - -int dlmallopt(int param_number, int value) { - return change_mparam(param_number, value); -} - -size_t dlmalloc_usable_size(void* mem) { - if (mem != 0) { - mchunkptr p = mem2chunk(mem); - if (is_inuse(p)) - return chunksize(p) - overhead_for(p); - } - return 0; -} - -#endif /* !ONLY_MSPACES */ - -/* ----------------------------- user mspaces ---------------------------- */ - -#if MSPACES - -static mstate init_user_mstate(char* tbase, size_t tsize) { - size_t msize = pad_request(sizeof(struct malloc_state)); - mchunkptr mn; - mchunkptr msp = align_as_chunk(tbase); - mstate m = (mstate)(chunk2mem(msp)); - memset(m, 0, msize); - (void)INITIAL_LOCK(&m->mutex); - msp->head = (msize|INUSE_BITS); - m->seg.base = m->least_addr = tbase; - m->seg.size = m->footprint = m->max_footprint = tsize; - m->magic = mparams.magic; - m->release_checks = MAX_RELEASE_CHECK_RATE; - m->mflags = mparams.default_mflags; - m->extp = 0; - m->exts = 0; - disable_contiguous(m); - init_bins(m); - mn = next_chunk(mem2chunk(m)); - init_top(m, mn, (size_t)((tbase + tsize) - (char*)mn) - TOP_FOOT_SIZE); - check_top_chunk(m, m->top); - return m; -} - -mspace create_mspace(size_t capacity, int locked) { - mstate m = 0; - size_t msize; - ensure_initialization(); - msize = pad_request(sizeof(struct malloc_state)); - if (capacity < (size_t) -(msize + TOP_FOOT_SIZE + mparams.page_size)) { - size_t rs = ((capacity == 0)? mparams.granularity : - (capacity + TOP_FOOT_SIZE + msize)); - size_t tsize = granularity_align(rs); - char* tbase = (char*)(CALL_MMAP(tsize)); - if (tbase != CMFAIL) { - m = init_user_mstate(tbase, tsize); - m->seg.sflags = USE_MMAP_BIT; - set_lock(m, locked); - } - } - return (mspace)m; -} - -mspace create_mspace_with_base(void* base, size_t capacity, int locked) { - mstate m = 0; - size_t msize; - ensure_initialization(); - msize = pad_request(sizeof(struct malloc_state)); - if (capacity > msize + TOP_FOOT_SIZE && - capacity < (size_t) -(msize + TOP_FOOT_SIZE + mparams.page_size)) { - m = init_user_mstate((char*)base, capacity); - m->seg.sflags = EXTERN_BIT; - set_lock(m, locked); - } - return (mspace)m; -} - -int mspace_track_large_chunks(mspace msp, int enable) { - int ret = 0; - mstate ms = (mstate)msp; - if (!PREACTION(ms)) { - if (!use_mmap(ms)) { - ret = 1; - } - if (!enable) { - enable_mmap(ms); - } else { - disable_mmap(ms); - } - POSTACTION(ms); - } - return ret; -} - -size_t destroy_mspace(mspace msp) { - size_t freed = 0; - mstate ms = (mstate)msp; - if (ok_magic(ms)) { - msegmentptr sp = &ms->seg; - (void)DESTROY_LOCK(&ms->mutex); /* destroy before unmapped */ - while (sp != 0) { - char* base = sp->base; - size_t size = sp->size; - flag_t flag = sp->sflags; - (void)base; /* placate people compiling -Wunused-variable */ - sp = sp->next; - if ((flag & USE_MMAP_BIT) && !(flag & EXTERN_BIT) && - CALL_MUNMAP(base, size) == 0) - freed += size; - } - } - else { - USAGE_ERROR_ACTION(ms,ms); - } - return freed; -} - -/* - mspace versions of routines are near-clones of the global - versions. This is not so nice but better than the alternatives. -*/ - -void* mspace_malloc(mspace msp, size_t bytes) { - mstate ms = (mstate)msp; - if (!ok_magic(ms)) { - USAGE_ERROR_ACTION(ms,ms); - return 0; - } - if (!PREACTION(ms)) { - void* mem; - size_t nb; - if (bytes <= MAX_SMALL_REQUEST) { - bindex_t idx; - binmap_t smallbits; - nb = (bytes < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(bytes); - idx = small_index(nb); - smallbits = ms->smallmap >> idx; - - if ((smallbits & 0x3U) != 0) { /* Remainderless fit to a smallbin. */ - mchunkptr b, p; - idx += ~smallbits & 1; /* Uses next bin if idx empty */ - b = smallbin_at(ms, idx); - p = b->fd; - assert(chunksize(p) == small_index2size(idx)); - unlink_first_small_chunk(ms, b, p, idx); - set_inuse_and_pinuse(ms, p, small_index2size(idx)); - mem = chunk2mem(p); - check_malloced_chunk(ms, mem, nb); - goto postaction; - } - - else if (nb > ms->dvsize) { - if (smallbits != 0) { /* Use chunk in next nonempty smallbin */ - mchunkptr b, p, r; - size_t rsize; - bindex_t i; - binmap_t leftbits = (smallbits << idx) & left_bits(idx2bit(idx)); - binmap_t leastbit = least_bit(leftbits); - compute_bit2idx(leastbit, i); - b = smallbin_at(ms, i); - p = b->fd; - assert(chunksize(p) == small_index2size(i)); - unlink_first_small_chunk(ms, b, p, i); - rsize = small_index2size(i) - nb; - /* Fit here cannot be remainderless if 4byte sizes */ - if (SIZE_T_SIZE != 4 && rsize < MIN_CHUNK_SIZE) - set_inuse_and_pinuse(ms, p, small_index2size(i)); - else { - set_size_and_pinuse_of_inuse_chunk(ms, p, nb); - r = chunk_plus_offset(p, nb); - set_size_and_pinuse_of_free_chunk(r, rsize); - replace_dv(ms, r, rsize); - } - mem = chunk2mem(p); - check_malloced_chunk(ms, mem, nb); - goto postaction; - } - - else if (ms->treemap != 0 && (mem = tmalloc_small(ms, nb)) != 0) { - check_malloced_chunk(ms, mem, nb); - goto postaction; - } - } - } - else if (bytes >= MAX_REQUEST) - nb = MAX_SIZE_T; /* Too big to allocate. Force failure (in sys alloc) */ - else { - nb = pad_request(bytes); - if (ms->treemap != 0 && (mem = tmalloc_large(ms, nb)) != 0) { - check_malloced_chunk(ms, mem, nb); - goto postaction; - } - } - - if (nb <= ms->dvsize) { - size_t rsize = ms->dvsize - nb; - mchunkptr p = ms->dv; - if (rsize >= MIN_CHUNK_SIZE) { /* split dv */ - mchunkptr r = ms->dv = chunk_plus_offset(p, nb); - ms->dvsize = rsize; - set_size_and_pinuse_of_free_chunk(r, rsize); - set_size_and_pinuse_of_inuse_chunk(ms, p, nb); - } - else { /* exhaust dv */ - size_t dvs = ms->dvsize; - ms->dvsize = 0; - ms->dv = 0; - set_inuse_and_pinuse(ms, p, dvs); - } - mem = chunk2mem(p); - check_malloced_chunk(ms, mem, nb); - goto postaction; - } - - else if (nb < ms->topsize) { /* Split top */ - size_t rsize = ms->topsize -= nb; - mchunkptr p = ms->top; - mchunkptr r = ms->top = chunk_plus_offset(p, nb); - r->head = rsize | PINUSE_BIT; - set_size_and_pinuse_of_inuse_chunk(ms, p, nb); - mem = chunk2mem(p); - check_top_chunk(ms, ms->top); - check_malloced_chunk(ms, mem, nb); - goto postaction; - } - - mem = sys_alloc(ms, nb); - - postaction: - POSTACTION(ms); - return mem; - } - - return 0; -} - -void mspace_free(mspace msp, void* mem) { - if (mem != 0) { - mchunkptr p = mem2chunk(mem); -#if FOOTERS - mstate fm = get_mstate_for(p); - (void)msp; /* placate people compiling -Wunused */ -#else /* FOOTERS */ - mstate fm = (mstate)msp; -#endif /* FOOTERS */ - if (!ok_magic(fm)) { - USAGE_ERROR_ACTION(fm, p); - return; - } - if (!PREACTION(fm)) { - check_inuse_chunk(fm, p); - if (RTCHECK(ok_address(fm, p) && ok_inuse(p))) { - size_t psize = chunksize(p); - mchunkptr next = chunk_plus_offset(p, psize); - if (!pinuse(p)) { - size_t prevsize = p->prev_foot; - if (is_mmapped(p)) { - psize += prevsize + MMAP_FOOT_PAD; - if (CALL_MUNMAP((char*)p - prevsize, psize) == 0) - fm->footprint -= psize; - goto postaction; - } - else { - mchunkptr prev = chunk_minus_offset(p, prevsize); - psize += prevsize; - p = prev; - if (RTCHECK(ok_address(fm, prev))) { /* consolidate backward */ - if (p != fm->dv) { - unlink_chunk(fm, p, prevsize); - } - else if ((next->head & INUSE_BITS) == INUSE_BITS) { - fm->dvsize = psize; - set_free_with_pinuse(p, psize, next); - goto postaction; - } - } - else - goto erroraction; - } - } - - if (RTCHECK(ok_next(p, next) && ok_pinuse(next))) { - if (!cinuse(next)) { /* consolidate forward */ - if (next == fm->top) { - size_t tsize = fm->topsize += psize; - fm->top = p; - p->head = tsize | PINUSE_BIT; - if (p == fm->dv) { - fm->dv = 0; - fm->dvsize = 0; - } - if (should_trim(fm, tsize)) - sys_trim(fm, 0); - goto postaction; - } - else if (next == fm->dv) { - size_t dsize = fm->dvsize += psize; - fm->dv = p; - set_size_and_pinuse_of_free_chunk(p, dsize); - goto postaction; - } - else { - size_t nsize = chunksize(next); - psize += nsize; - unlink_chunk(fm, next, nsize); - set_size_and_pinuse_of_free_chunk(p, psize); - if (p == fm->dv) { - fm->dvsize = psize; - goto postaction; - } - } - } - else - set_free_with_pinuse(p, psize, next); - - if (is_small(psize)) { - insert_small_chunk(fm, p, psize); - check_free_chunk(fm, p); - } - else { - tchunkptr tp = (tchunkptr)p; - insert_large_chunk(fm, tp, psize); - check_free_chunk(fm, p); - if (--fm->release_checks == 0) - release_unused_segments(fm); - } - goto postaction; - } - } - erroraction: - USAGE_ERROR_ACTION(fm, p); - postaction: - POSTACTION(fm); - } - } -} - -void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size) { - void* mem; - size_t req = 0; - mstate ms = (mstate)msp; - if (!ok_magic(ms)) { - USAGE_ERROR_ACTION(ms,ms); - return 0; - } - if (n_elements != 0) { - req = n_elements * elem_size; - if (((n_elements | elem_size) & ~(size_t)0xffff) && - (req / n_elements != elem_size)) - req = MAX_SIZE_T; /* force downstream failure on overflow */ - } - mem = internal_malloc(ms, req); - if (mem != 0 && calloc_must_clear(mem2chunk(mem))) - memset(mem, 0, req); - return mem; -} - -void* mspace_realloc(mspace msp, void* oldmem, size_t bytes) { - void* mem = 0; - if (oldmem == 0) { - mem = mspace_malloc(msp, bytes); - } - else if (bytes >= MAX_REQUEST) { - MALLOC_FAILURE_ACTION; - } -#ifdef REALLOC_ZERO_BYTES_FREES - else if (bytes == 0) { - mspace_free(msp, oldmem); - } -#endif /* REALLOC_ZERO_BYTES_FREES */ - else { - size_t nb = request2size(bytes); - mchunkptr oldp = mem2chunk(oldmem); -#if ! FOOTERS - mstate m = (mstate)msp; -#else /* FOOTERS */ - mstate m = get_mstate_for(oldp); - if (!ok_magic(m)) { - USAGE_ERROR_ACTION(m, oldmem); - return 0; - } -#endif /* FOOTERS */ - if (!PREACTION(m)) { - mchunkptr newp = try_realloc_chunk(m, oldp, nb, 1); - POSTACTION(m); - if (newp != 0) { - check_inuse_chunk(m, newp); - mem = chunk2mem(newp); - } - else { - mem = mspace_malloc(m, bytes); - if (mem != 0) { - size_t oc = chunksize(oldp) - overhead_for(oldp); - memcpy(mem, oldmem, (oc < bytes)? oc : bytes); - mspace_free(m, oldmem); - } - } - } - } - return mem; -} - -void* mspace_realloc_in_place(mspace msp, void* oldmem, size_t bytes) { - void* mem = 0; - if (oldmem != 0) { - if (bytes >= MAX_REQUEST) { - MALLOC_FAILURE_ACTION; - } - else { - size_t nb = request2size(bytes); - mchunkptr oldp = mem2chunk(oldmem); -#if ! FOOTERS - mstate m = (mstate)msp; -#else /* FOOTERS */ - mstate m = get_mstate_for(oldp); - (void)msp; /* placate people compiling -Wunused */ - if (!ok_magic(m)) { - USAGE_ERROR_ACTION(m, oldmem); - return 0; - } -#endif /* FOOTERS */ - if (!PREACTION(m)) { - mchunkptr newp = try_realloc_chunk(m, oldp, nb, 0); - POSTACTION(m); - if (newp == oldp) { - check_inuse_chunk(m, newp); - mem = oldmem; - } - } - } - } - return mem; -} - -void* mspace_memalign(mspace msp, size_t alignment, size_t bytes) { - mstate ms = (mstate)msp; - if (!ok_magic(ms)) { - USAGE_ERROR_ACTION(ms,ms); - return 0; - } - if (alignment <= MALLOC_ALIGNMENT) - return mspace_malloc(msp, bytes); - return internal_memalign(ms, alignment, bytes); -} - -void** mspace_independent_calloc(mspace msp, size_t n_elements, - size_t elem_size, void* chunks[]) { - size_t sz = elem_size; /* serves as 1-element array */ - mstate ms = (mstate)msp; - if (!ok_magic(ms)) { - USAGE_ERROR_ACTION(ms,ms); - return 0; - } - return ialloc(ms, n_elements, &sz, 3, chunks); -} - -void** mspace_independent_comalloc(mspace msp, size_t n_elements, - size_t sizes[], void* chunks[]) { - mstate ms = (mstate)msp; - if (!ok_magic(ms)) { - USAGE_ERROR_ACTION(ms,ms); - return 0; - } - return ialloc(ms, n_elements, sizes, 0, chunks); -} - -size_t mspace_bulk_free(mspace msp, void* array[], size_t nelem) { - return internal_bulk_free((mstate)msp, array, nelem); -} - -#if MALLOC_INSPECT_ALL -void mspace_inspect_all(mspace msp, - void(*handler)(void *start, - void *end, - size_t used_bytes, - void* callback_arg), - void* arg) { - mstate ms = (mstate)msp; - if (ok_magic(ms)) { - if (!PREACTION(ms)) { - internal_inspect_all(ms, handler, arg); - POSTACTION(ms); - } - } - else { - USAGE_ERROR_ACTION(ms,ms); - } -} -#endif /* MALLOC_INSPECT_ALL */ - -int mspace_trim(mspace msp, size_t pad) { - int result = 0; - mstate ms = (mstate)msp; - if (ok_magic(ms)) { - if (!PREACTION(ms)) { - result = sys_trim(ms, pad); - POSTACTION(ms); - } - } - else { - USAGE_ERROR_ACTION(ms,ms); - } - return result; -} - -#if !NO_MALLOC_STATS -void mspace_malloc_stats(mspace msp) { - mstate ms = (mstate)msp; - if (ok_magic(ms)) { - internal_malloc_stats(ms); - } - else { - USAGE_ERROR_ACTION(ms,ms); - } -} -#endif /* NO_MALLOC_STATS */ - -size_t mspace_footprint(mspace msp) { - size_t result = 0; - mstate ms = (mstate)msp; - if (ok_magic(ms)) { - result = ms->footprint; - } - else { - USAGE_ERROR_ACTION(ms,ms); - } - return result; -} - -size_t mspace_max_footprint(mspace msp) { - size_t result = 0; - mstate ms = (mstate)msp; - if (ok_magic(ms)) { - result = ms->max_footprint; - } - else { - USAGE_ERROR_ACTION(ms,ms); - } - return result; -} - -size_t mspace_footprint_limit(mspace msp) { - size_t result = 0; - mstate ms = (mstate)msp; - if (ok_magic(ms)) { - size_t maf = ms->footprint_limit; - result = (maf == 0) ? MAX_SIZE_T : maf; - } - else { - USAGE_ERROR_ACTION(ms,ms); - } - return result; -} - -size_t mspace_set_footprint_limit(mspace msp, size_t bytes) { - size_t result = 0; - mstate ms = (mstate)msp; - if (ok_magic(ms)) { - if (bytes == 0) - result = granularity_align(1); /* Use minimal size */ - if (bytes == MAX_SIZE_T) - result = 0; /* disable */ - else - result = granularity_align(bytes); - ms->footprint_limit = result; - } - else { - USAGE_ERROR_ACTION(ms,ms); - } - return result; -} - -#if !NO_MALLINFO -struct mallinfo mspace_mallinfo(mspace msp) { - mstate ms = (mstate)msp; - if (!ok_magic(ms)) { - USAGE_ERROR_ACTION(ms,ms); - } - return internal_mallinfo(ms); -} -#endif /* NO_MALLINFO */ - -size_t mspace_usable_size(const void* mem) { - if (mem != 0) { - mchunkptr p = mem2chunk(mem); - if (is_inuse(p)) - return chunksize(p) - overhead_for(p); - } - return 0; -} - -int mspace_mallopt(int param_number, int value) { - return change_mparam(param_number, value); -} - -#endif /* MSPACES */ - - -/* -------------------- Alternative MORECORE functions ------------------- */ - -/* - Guidelines for creating a custom version of MORECORE: - - * For best performance, MORECORE should allocate in multiples of pagesize. - * MORECORE may allocate more memory than requested. (Or even less, - but this will usually result in a malloc failure.) - * MORECORE must not allocate memory when given argument zero, but - instead return one past the end address of memory from previous - nonzero call. - * For best performance, consecutive calls to MORECORE with positive - arguments should return increasing addresses, indicating that - space has been contiguously extended. - * Even though consecutive calls to MORECORE need not return contiguous - addresses, it must be OK for malloc'ed chunks to span multiple - regions in those cases where they do happen to be contiguous. - * MORECORE need not handle negative arguments -- it may instead - just return MFAIL when given negative arguments. - Negative arguments are always multiples of pagesize. MORECORE - must not misinterpret negative args as large positive unsigned - args. You can suppress all such calls from even occurring by defining - MORECORE_CANNOT_TRIM, - - As an example alternative MORECORE, here is a custom allocator - kindly contributed for pre-OSX macOS. It uses virtually but not - necessarily physically contiguous non-paged memory (locked in, - present and won't get swapped out). You can use it by uncommenting - this section, adding some #includes, and setting up the appropriate - defines above: - - #define MORECORE osMoreCore - - There is also a shutdown routine that should somehow be called for - cleanup upon program exit. - - #define MAX_POOL_ENTRIES 100 - #define MINIMUM_MORECORE_SIZE (64 * 1024U) - static int next_os_pool; - void *our_os_pools[MAX_POOL_ENTRIES]; - - void *osMoreCore(int size) - { - void *ptr = 0; - static void *sbrk_top = 0; - - if (size > 0) - { - if (size < MINIMUM_MORECORE_SIZE) - size = MINIMUM_MORECORE_SIZE; - if (CurrentExecutionLevel() == kTaskLevel) - ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0); - if (ptr == 0) - { - return (void *) MFAIL; - } - // save ptrs so they can be freed during cleanup - our_os_pools[next_os_pool] = ptr; - next_os_pool++; - ptr = (void *) ((((size_t) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK); - sbrk_top = (char *) ptr + size; - return ptr; - } - else if (size < 0) - { - // we don't currently support shrink behavior - return (void *) MFAIL; - } - else - { - return sbrk_top; - } - } - - // cleanup any allocated memory pools - // called as last thing before shutting down driver - - void osCleanupMem(void) - { - void **ptr; - - for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++) - if (*ptr) - { - PoolDeallocate(*ptr); - *ptr = 0; - } - } - -*/ - - -/* ----------------------------------------------------------------------- -History: - v2.8.6 Wed Aug 29 06:57:58 2012 Doug Lea - * fix bad comparison in dlposix_memalign - * don't reuse adjusted asize in sys_alloc - * add LOCK_AT_FORK -- thanks to Kirill Artamonov for the suggestion - * reduce compiler warnings -- thanks to all who reported/suggested these - - v2.8.5 Sun May 22 10:26:02 2011 Doug Lea (dl at gee) - * Always perform unlink checks unless INSECURE - * Add posix_memalign. - * Improve realloc to expand in more cases; expose realloc_in_place. - Thanks to Peter Buhr for the suggestion. - * Add footprint_limit, inspect_all, bulk_free. Thanks - to Barry Hayes and others for the suggestions. - * Internal refactorings to avoid calls while holding locks - * Use non-reentrant locks by default. Thanks to Roland McGrath - for the suggestion. - * Small fixes to mspace_destroy, reset_on_error. - * Various configuration extensions/changes. Thanks - to all who contributed these. - - V2.8.4a Thu Apr 28 14:39:43 2011 (dl at gee.cs.oswego.edu) - * Update Creative Commons URL - - V2.8.4 Wed May 27 09:56:23 2009 Doug Lea (dl at gee) - * Use zeros instead of prev foot for is_mmapped - * Add mspace_track_large_chunks; thanks to Jean Brouwers - * Fix set_inuse in internal_realloc; thanks to Jean Brouwers - * Fix insufficient sys_alloc padding when using 16byte alignment - * Fix bad error check in mspace_footprint - * Adaptations for ptmalloc; thanks to Wolfram Gloger. - * Reentrant spin locks; thanks to Earl Chew and others - * Win32 improvements; thanks to Niall Douglas and Earl Chew - * Add NO_SEGMENT_TRAVERSAL and MAX_RELEASE_CHECK_RATE options - * Extension hook in malloc_state - * Various small adjustments to reduce warnings on some compilers - * Various configuration extensions/changes for more platforms. Thanks - to all who contributed these. - - V2.8.3 Thu Sep 22 11:16:32 2005 Doug Lea (dl at gee) - * Add max_footprint functions - * Ensure all appropriate literals are size_t - * Fix conditional compilation problem for some #define settings - * Avoid concatenating segments with the one provided - in create_mspace_with_base - * Rename some variables to avoid compiler shadowing warnings - * Use explicit lock initialization. - * Better handling of sbrk interference. - * Simplify and fix segment insertion, trimming and mspace_destroy - * Reinstate REALLOC_ZERO_BYTES_FREES option from 2.7.x - * Thanks especially to Dennis Flanagan for help on these. - - V2.8.2 Sun Jun 12 16:01:10 2005 Doug Lea (dl at gee) - * Fix memalign brace error. - - V2.8.1 Wed Jun 8 16:11:46 2005 Doug Lea (dl at gee) - * Fix improper #endif nesting in C++ - * Add explicit casts needed for C++ - - V2.8.0 Mon May 30 14:09:02 2005 Doug Lea (dl at gee) - * Use trees for large bins - * Support mspaces - * Use segments to unify sbrk-based and mmap-based system allocation, - removing need for emulation on most platforms without sbrk. - * Default safety checks - * Optional footer checks. Thanks to William Robertson for the idea. - * Internal code refactoring - * Incorporate suggestions and platform-specific changes. - Thanks to Dennis Flanagan, Colin Plumb, Niall Douglas, - Aaron Bachmann, Emery Berger, and others. - * Speed up non-fastbin processing enough to remove fastbins. - * Remove useless cfree() to avoid conflicts with other apps. - * Remove internal memcpy, memset. Compilers handle builtins better. - * Remove some options that no one ever used and rename others. - - V2.7.2 Sat Aug 17 09:07:30 2002 Doug Lea (dl at gee) - * Fix malloc_state bitmap array misdeclaration - - V2.7.1 Thu Jul 25 10:58:03 2002 Doug Lea (dl at gee) - * Allow tuning of FIRST_SORTED_BIN_SIZE - * Use PTR_UINT as type for all ptr->int casts. Thanks to John Belmonte. - * Better detection and support for non-contiguousness of MORECORE. - Thanks to Andreas Mueller, Conal Walsh, and Wolfram Gloger - * Bypass most of malloc if no frees. Thanks To Emery Berger. - * Fix freeing of old top non-contiguous chunk im sysmalloc. - * Raised default trim and map thresholds to 256K. - * Fix mmap-related #defines. Thanks to Lubos Lunak. - * Fix copy macros; added LACKS_FCNTL_H. Thanks to Neal Walfield. - * Branch-free bin calculation - * Default trim and mmap thresholds now 256K. - - V2.7.0 Sun Mar 11 14:14:06 2001 Doug Lea (dl at gee) - * Introduce independent_comalloc and independent_calloc. - Thanks to Michael Pachos for motivation and help. - * Make optional .h file available - * Allow > 2GB requests on 32bit systems. - * new WIN32 sbrk, mmap, munmap, lock code from . - Thanks also to Andreas Mueller , - and Anonymous. - * Allow override of MALLOC_ALIGNMENT (Thanks to Ruud Waij for - helping test this.) - * memalign: check alignment arg - * realloc: don't try to shift chunks backwards, since this - leads to more fragmentation in some programs and doesn't - seem to help in any others. - * Collect all cases in malloc requiring system memory into sysmalloc - * Use mmap as backup to sbrk - * Place all internal state in malloc_state - * Introduce fastbins (although similar to 2.5.1) - * Many minor tunings and cosmetic improvements - * Introduce USE_PUBLIC_MALLOC_WRAPPERS, USE_MALLOC_LOCK - * Introduce MALLOC_FAILURE_ACTION, MORECORE_CONTIGUOUS - Thanks to Tony E. Bennett and others. - * Include errno.h to support default failure action. - - V2.6.6 Sun Dec 5 07:42:19 1999 Doug Lea (dl at gee) - * return null for negative arguments - * Added Several WIN32 cleanups from Martin C. Fong - * Add 'LACKS_SYS_PARAM_H' for those systems without 'sys/param.h' - (e.g. WIN32 platforms) - * Cleanup header file inclusion for WIN32 platforms - * Cleanup code to avoid Microsoft Visual C++ compiler complaints - * Add 'USE_DL_PREFIX' to quickly allow co-existence with existing - memory allocation routines - * Set 'malloc_getpagesize' for WIN32 platforms (needs more work) - * Use 'assert' rather than 'ASSERT' in WIN32 code to conform to - usage of 'assert' in non-WIN32 code - * Improve WIN32 'sbrk()' emulation's 'findRegion()' routine to - avoid infinite loop - * Always call 'fREe()' rather than 'free()' - - V2.6.5 Wed Jun 17 15:57:31 1998 Doug Lea (dl at gee) - * Fixed ordering problem with boundary-stamping - - V2.6.3 Sun May 19 08:17:58 1996 Doug Lea (dl at gee) - * Added pvalloc, as recommended by H.J. Liu - * Added 64bit pointer support mainly from Wolfram Gloger - * Added anonymously donated WIN32 sbrk emulation - * Malloc, calloc, getpagesize: add optimizations from Raymond Nijssen - * malloc_extend_top: fix mask error that caused wastage after - foreign sbrks - * Add linux mremap support code from HJ Liu - - V2.6.2 Tue Dec 5 06:52:55 1995 Doug Lea (dl at gee) - * Integrated most documentation with the code. - * Add support for mmap, with help from - Wolfram Gloger (Gloger@lrz.uni-muenchen.de). - * Use last_remainder in more cases. - * Pack bins using idea from colin@nyx10.cs.du.edu - * Use ordered bins instead of best-fit threshhold - * Eliminate block-local decls to simplify tracing and debugging. - * Support another case of realloc via move into top - * Fix error occuring when initial sbrk_base not word-aligned. - * Rely on page size for units instead of SBRK_UNIT to - avoid surprises about sbrk alignment conventions. - * Add mallinfo, mallopt. Thanks to Raymond Nijssen - (raymond@es.ele.tue.nl) for the suggestion. - * Add `pad' argument to malloc_trim and top_pad mallopt parameter. - * More precautions for cases where other routines call sbrk, - courtesy of Wolfram Gloger (Gloger@lrz.uni-muenchen.de). - * Added macros etc., allowing use in linux libc from - H.J. Lu (hjl@gnu.ai.mit.edu) - * Inverted this history list - - V2.6.1 Sat Dec 2 14:10:57 1995 Doug Lea (dl at gee) - * Re-tuned and fixed to behave more nicely with V2.6.0 changes. - * Removed all preallocation code since under current scheme - the work required to undo bad preallocations exceeds - the work saved in good cases for most test programs. - * No longer use return list or unconsolidated bins since - no scheme using them consistently outperforms those that don't - given above changes. - * Use best fit for very large chunks to prevent some worst-cases. - * Added some support for debugging - - V2.6.0 Sat Nov 4 07:05:23 1995 Doug Lea (dl at gee) - * Removed footers when chunks are in use. Thanks to - Paul Wilson (wilson@cs.texas.edu) for the suggestion. - - V2.5.4 Wed Nov 1 07:54:51 1995 Doug Lea (dl at gee) - * Added malloc_trim, with help from Wolfram Gloger - (wmglo@Dent.MED.Uni-Muenchen.DE). - - V2.5.3 Tue Apr 26 10:16:01 1994 Doug Lea (dl at g) - - V2.5.2 Tue Apr 5 16:20:40 1994 Doug Lea (dl at g) - * realloc: try to expand in both directions - * malloc: swap order of clean-bin strategy; - * realloc: only conditionally expand backwards - * Try not to scavenge used bins - * Use bin counts as a guide to preallocation - * Occasionally bin return list chunks in first scan - * Add a few optimizations from colin@nyx10.cs.du.edu - - V2.5.1 Sat Aug 14 15:40:43 1993 Doug Lea (dl at g) - * faster bin computation & slightly different binning - * merged all consolidations to one part of malloc proper - (eliminating old malloc_find_space & malloc_clean_bin) - * Scan 2 returns chunks (not just 1) - * Propagate failure in realloc if malloc returns 0 - * Add stuff to allow compilation on non-ANSI compilers - from kpv@research.att.com - - V2.5 Sat Aug 7 07:41:59 1993 Doug Lea (dl at g.oswego.edu) - * removed potential for odd address access in prev_chunk - * removed dependency on getpagesize.h - * misc cosmetics and a bit more internal documentation - * anticosmetics: mangled names in macros to evade debugger strangeness - * tested on sparc, hp-700, dec-mips, rs6000 - with gcc & native cc (hp, dec only) allowing - Detlefs & Zorn comparison study (in SIGPLAN Notices.) - - Trial version Fri Aug 28 13:14:29 1992 Doug Lea (dl at g.oswego.edu) - * Based loosely on libg++-1.2X malloc. (It retains some of the overall - structure of old version, but most details differ.) - -*/ diff --git a/native/src/crt0/dlmalloc/malloc.h b/native/src/crt0/dlmalloc/malloc.h deleted file mode 100644 index 877d28f45..000000000 --- a/native/src/crt0/dlmalloc/malloc.h +++ /dev/null @@ -1,637 +0,0 @@ -/* -Copyright 2023 Doug Lea - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - Default header file for malloc-2.8.x - Re-licensed 25 Sep 2023 with MIT-0 replacing obsolete CC0 - See https://opensource.org/license/mit-0/ - - This header is for ANSI C/C++ only. You can set any of - the following #defines before including: - - * If USE_DL_PREFIX is defined, it is assumed that malloc.c - was also compiled with this option, so all routines - have names starting with "dl". - - * If HAVE_USR_INCLUDE_MALLOC_H is defined, it is assumed that this - file will be #included AFTER . This is needed only if - your system defines a struct mallinfo that is incompatible with the - standard one declared here. Otherwise, you can include this file - INSTEAD of your system system . At least on ANSI, all - declarations should be compatible with system versions - - * If MSPACES is defined, declarations for mspace versions are included. -*/ - -#ifndef MALLOC_280_H -#define MALLOC_280_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include /* for size_t */ - -#ifndef ONLY_MSPACES -#define ONLY_MSPACES 0 /* define to a value */ -#elif ONLY_MSPACES != 0 -#define ONLY_MSPACES 1 -#endif /* ONLY_MSPACES */ -#ifndef NO_MALLINFO -#define NO_MALLINFO 0 -#endif /* NO_MALLINFO */ - -#ifndef MSPACES -#if ONLY_MSPACES -#define MSPACES 1 -#else /* ONLY_MSPACES */ -#define MSPACES 0 -#endif /* ONLY_MSPACES */ -#endif /* MSPACES */ - -#if !ONLY_MSPACES - -#ifndef USE_DL_PREFIX -#define dlcalloc calloc -#define dlfree free -#define dlmalloc malloc -#define dlmemalign memalign -#define dlposix_memalign posix_memalign -#define dlrealloc realloc -#define dlvalloc valloc -#define dlpvalloc pvalloc -#define dlmallinfo mallinfo -#define dlmallopt mallopt -#define dlmalloc_trim malloc_trim -#define dlmalloc_stats malloc_stats -#define dlmalloc_usable_size malloc_usable_size -#define dlmalloc_footprint malloc_footprint -#define dlmalloc_max_footprint malloc_max_footprint -#define dlmalloc_footprint_limit malloc_footprint_limit -#define dlmalloc_set_footprint_limit malloc_set_footprint_limit -#define dlmalloc_inspect_all malloc_inspect_all -#define dlindependent_calloc independent_calloc -#define dlindependent_comalloc independent_comalloc -#define dlbulk_free bulk_free -#endif /* USE_DL_PREFIX */ - -#if !NO_MALLINFO -#ifndef HAVE_USR_INCLUDE_MALLOC_H -#ifndef _MALLOC_H -#ifndef MALLINFO_FIELD_TYPE -#define MALLINFO_FIELD_TYPE size_t -#endif /* MALLINFO_FIELD_TYPE */ -#ifndef STRUCT_MALLINFO_DECLARED -#define STRUCT_MALLINFO_DECLARED 1 -struct mallinfo { - MALLINFO_FIELD_TYPE arena; /* non-mmapped space allocated from system */ - MALLINFO_FIELD_TYPE ordblks; /* number of free chunks */ - MALLINFO_FIELD_TYPE smblks; /* always 0 */ - MALLINFO_FIELD_TYPE hblks; /* always 0 */ - MALLINFO_FIELD_TYPE hblkhd; /* space in mmapped regions */ - MALLINFO_FIELD_TYPE usmblks; /* maximum total allocated space */ - MALLINFO_FIELD_TYPE fsmblks; /* always 0 */ - MALLINFO_FIELD_TYPE uordblks; /* total allocated space */ - MALLINFO_FIELD_TYPE fordblks; /* total free space */ - MALLINFO_FIELD_TYPE keepcost; /* releasable (via malloc_trim) space */ -}; -#endif /* STRUCT_MALLINFO_DECLARED */ -#endif /* _MALLOC_H */ -#endif /* HAVE_USR_INCLUDE_MALLOC_H */ -#endif /* !NO_MALLINFO */ - -/* - malloc(size_t n) - Returns a pointer to a newly allocated chunk of at least n bytes, or - null if no space is available, in which case errno is set to ENOMEM - on ANSI C systems. - - If n is zero, malloc returns a minimum-sized chunk. (The minimum - size is 16 bytes on most 32bit systems, and 32 bytes on 64bit - systems.) Note that size_t is an unsigned type, so calls with - arguments that would be negative if signed are interpreted as - requests for huge amounts of space, which will often fail. The - maximum supported value of n differs across systems, but is in all - cases less than the maximum representable value of a size_t. -*/ -void* dlmalloc(size_t); - -/* - free(void* p) - Releases the chunk of memory pointed to by p, that had been previously - allocated using malloc or a related routine such as realloc. - It has no effect if p is null. If p was not malloced or already - freed, free(p) will by default cuase the current program to abort. -*/ -void dlfree(void*); - -/* - calloc(size_t n_elements, size_t element_size); - Returns a pointer to n_elements * element_size bytes, with all locations - set to zero. -*/ -void* dlcalloc(size_t, size_t); - -/* - realloc(void* p, size_t n) - Returns a pointer to a chunk of size n that contains the same data - as does chunk p up to the minimum of (n, p's size) bytes, or null - if no space is available. - - The returned pointer may or may not be the same as p. The algorithm - prefers extending p in most cases when possible, otherwise it - employs the equivalent of a malloc-copy-free sequence. - - If p is null, realloc is equivalent to malloc. - - If space is not available, realloc returns null, errno is set (if on - ANSI) and p is NOT freed. - - if n is for fewer bytes than already held by p, the newly unused - space is lopped off and freed if possible. realloc with a size - argument of zero (re)allocates a minimum-sized chunk. - - The old unix realloc convention of allowing the last-free'd chunk - to be used as an argument to realloc is not supported. -*/ -void* dlrealloc(void*, size_t); - -/* - realloc_in_place(void* p, size_t n) - Resizes the space allocated for p to size n, only if this can be - done without moving p (i.e., only if there is adjacent space - available if n is greater than p's current allocated size, or n is - less than or equal to p's size). This may be used instead of plain - realloc if an alternative allocation strategy is needed upon failure - to expand space; for example, reallocation of a buffer that must be - memory-aligned or cleared. You can use realloc_in_place to trigger - these alternatives only when needed. - - Returns p if successful; otherwise null. -*/ -void* dlrealloc_in_place(void*, size_t); - -/* - memalign(size_t alignment, size_t n); - Returns a pointer to a newly allocated chunk of n bytes, aligned - in accord with the alignment argument. - - The alignment argument should be a power of two. If the argument is - not a power of two, the nearest greater power is used. - 8-byte alignment is guaranteed by normal malloc calls, so don't - bother calling memalign with an argument of 8 or less. - - Overreliance on memalign is a sure way to fragment space. -*/ -void* dlmemalign(size_t, size_t); - -/* - int posix_memalign(void** pp, size_t alignment, size_t n); - Allocates a chunk of n bytes, aligned in accord with the alignment - argument. Differs from memalign only in that it (1) assigns the - allocated memory to *pp rather than returning it, (2) fails and - returns EINVAL if the alignment is not a power of two (3) fails and - returns ENOMEM if memory cannot be allocated. -*/ -int dlposix_memalign(void**, size_t, size_t); - -/* - valloc(size_t n); - Equivalent to memalign(pagesize, n), where pagesize is the page - size of the system. If the pagesize is unknown, 4096 is used. -*/ -void* dlvalloc(size_t); - -/* - mallopt(int parameter_number, int parameter_value) - Sets tunable parameters The format is to provide a - (parameter-number, parameter-value) pair. mallopt then sets the - corresponding parameter to the argument value if it can (i.e., so - long as the value is meaningful), and returns 1 if successful else - 0. SVID/XPG/ANSI defines four standard param numbers for mallopt, - normally defined in malloc.h. None of these are use in this malloc, - so setting them has no effect. But this malloc also supports other - options in mallopt: - - Symbol param # default allowed param values - M_TRIM_THRESHOLD -1 2*1024*1024 any (-1U disables trimming) - M_GRANULARITY -2 page size any power of 2 >= page size - M_MMAP_THRESHOLD -3 256*1024 any (or 0 if no MMAP support) -*/ -int dlmallopt(int, int); - -#define M_TRIM_THRESHOLD (-1) -#define M_GRANULARITY (-2) -#define M_MMAP_THRESHOLD (-3) - - -/* - malloc_footprint(); - Returns the number of bytes obtained from the system. The total - number of bytes allocated by malloc, realloc etc., is less than this - value. Unlike mallinfo, this function returns only a precomputed - result, so can be called frequently to monitor memory consumption. - Even if locks are otherwise defined, this function does not use them, - so results might not be up to date. -*/ -size_t dlmalloc_footprint(void); - -/* - malloc_max_footprint(); - Returns the maximum number of bytes obtained from the system. This - value will be greater than current footprint if deallocated space - has been reclaimed by the system. The peak number of bytes allocated - by malloc, realloc etc., is less than this value. Unlike mallinfo, - this function returns only a precomputed result, so can be called - frequently to monitor memory consumption. Even if locks are - otherwise defined, this function does not use them, so results might - not be up to date. -*/ -size_t dlmalloc_max_footprint(void); - -/* - malloc_footprint_limit(); - Returns the number of bytes that the heap is allowed to obtain from - the system, returning the last value returned by - malloc_set_footprint_limit, or the maximum size_t value if - never set. The returned value reflects a permission. There is no - guarantee that this number of bytes can actually be obtained from - the system. -*/ -size_t dlmalloc_footprint_limit(void); - -/* - malloc_set_footprint_limit(); - Sets the maximum number of bytes to obtain from the system, causing - failure returns from malloc and related functions upon attempts to - exceed this value. The argument value may be subject to page - rounding to an enforceable limit; this actual value is returned. - Using an argument of the maximum possible size_t effectively - disables checks. If the argument is less than or equal to the - current malloc_footprint, then all future allocations that require - additional system memory will fail. However, invocation cannot - retroactively deallocate existing used memory. -*/ -size_t dlmalloc_set_footprint_limit(size_t bytes); - -/* - malloc_inspect_all(void(*handler)(void *start, - void *end, - size_t used_bytes, - void* callback_arg), - void* arg); - Traverses the heap and calls the given handler for each managed - region, skipping all bytes that are (or may be) used for bookkeeping - purposes. Traversal does not include include chunks that have been - directly memory mapped. Each reported region begins at the start - address, and continues up to but not including the end address. The - first used_bytes of the region contain allocated data. If - used_bytes is zero, the region is unallocated. The handler is - invoked with the given callback argument. If locks are defined, they - are held during the entire traversal. It is a bad idea to invoke - other malloc functions from within the handler. - - For example, to count the number of in-use chunks with size greater - than 1000, you could write: - static int count = 0; - void count_chunks(void* start, void* end, size_t used, void* arg) { - if (used >= 1000) ++count; - } - then: - malloc_inspect_all(count_chunks, NULL); - - malloc_inspect_all is compiled only if MALLOC_INSPECT_ALL is defined. -*/ -void dlmalloc_inspect_all(void(*handler)(void*, void *, size_t, void*), - void* arg); - -#if !NO_MALLINFO -/* - mallinfo() - Returns (by copy) a struct containing various summary statistics: - - arena: current total non-mmapped bytes allocated from system - ordblks: the number of free chunks - smblks: always zero. - hblks: current number of mmapped regions - hblkhd: total bytes held in mmapped regions - usmblks: the maximum total allocated space. This will be greater - than current total if trimming has occurred. - fsmblks: always zero - uordblks: current total allocated space (normal or mmapped) - fordblks: total free space - keepcost: the maximum number of bytes that could ideally be released - back to system via malloc_trim. ("ideally" means that - it ignores page restrictions etc.) - - Because these fields are ints, but internal bookkeeping may - be kept as longs, the reported values may wrap around zero and - thus be inaccurate. -*/ - -struct mallinfo dlmallinfo(void); -#endif /* NO_MALLINFO */ - -/* - independent_calloc(size_t n_elements, size_t element_size, void* chunks[]); - - independent_calloc is similar to calloc, but instead of returning a - single cleared space, it returns an array of pointers to n_elements - independent elements that can hold contents of size elem_size, each - of which starts out cleared, and can be independently freed, - realloc'ed etc. The elements are guaranteed to be adjacently - allocated (this is not guaranteed to occur with multiple callocs or - mallocs), which may also improve cache locality in some - applications. - - The "chunks" argument is optional (i.e., may be null, which is - probably the most typical usage). If it is null, the returned array - is itself dynamically allocated and should also be freed when it is - no longer needed. Otherwise, the chunks array must be of at least - n_elements in length. It is filled in with the pointers to the - chunks. - - In either case, independent_calloc returns this pointer array, or - null if the allocation failed. If n_elements is zero and "chunks" - is null, it returns a chunk representing an array with zero elements - (which should be freed if not wanted). - - Each element must be freed when it is no longer needed. This can be - done all at once using bulk_free. - - independent_calloc simplifies and speeds up implementations of many - kinds of pools. It may also be useful when constructing large data - structures that initially have a fixed number of fixed-sized nodes, - but the number is not known at compile time, and some of the nodes - may later need to be freed. For example: - - struct Node { int item; struct Node* next; }; - - struct Node* build_list() { - struct Node** pool; - int n = read_number_of_nodes_needed(); - if (n <= 0) return 0; - pool = (struct Node**)(independent_calloc(n, sizeof(struct Node), 0); - if (pool == 0) die(); - // organize into a linked list... - struct Node* first = pool[0]; - for (i = 0; i < n-1; ++i) - pool[i]->next = pool[i+1]; - free(pool); // Can now free the array (or not, if it is needed later) - return first; - } -*/ -void** dlindependent_calloc(size_t, size_t, void**); - -/* - independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]); - - independent_comalloc allocates, all at once, a set of n_elements - chunks with sizes indicated in the "sizes" array. It returns - an array of pointers to these elements, each of which can be - independently freed, realloc'ed etc. The elements are guaranteed to - be adjacently allocated (this is not guaranteed to occur with - multiple callocs or mallocs), which may also improve cache locality - in some applications. - - The "chunks" argument is optional (i.e., may be null). If it is null - the returned array is itself dynamically allocated and should also - be freed when it is no longer needed. Otherwise, the chunks array - must be of at least n_elements in length. It is filled in with the - pointers to the chunks. - - In either case, independent_comalloc returns this pointer array, or - null if the allocation failed. If n_elements is zero and chunks is - null, it returns a chunk representing an array with zero elements - (which should be freed if not wanted). - - Each element must be freed when it is no longer needed. This can be - done all at once using bulk_free. - - independent_comallac differs from independent_calloc in that each - element may have a different size, and also that it does not - automatically clear elements. - - independent_comalloc can be used to speed up allocation in cases - where several structs or objects must always be allocated at the - same time. For example: - - struct Head { ... } - struct Foot { ... } - - void send_message(char* msg) { - int msglen = strlen(msg); - size_t sizes[3] = { sizeof(struct Head), msglen, sizeof(struct Foot) }; - void* chunks[3]; - if (independent_comalloc(3, sizes, chunks) == 0) - die(); - struct Head* head = (struct Head*)(chunks[0]); - char* body = (char*)(chunks[1]); - struct Foot* foot = (struct Foot*)(chunks[2]); - // ... - } - - In general though, independent_comalloc is worth using only for - larger values of n_elements. For small values, you probably won't - detect enough difference from series of malloc calls to bother. - - Overuse of independent_comalloc can increase overall memory usage, - since it cannot reuse existing noncontiguous small chunks that - might be available for some of the elements. -*/ -void** dlindependent_comalloc(size_t, size_t*, void**); - -/* - bulk_free(void* array[], size_t n_elements) - Frees and clears (sets to null) each non-null pointer in the given - array. This is likely to be faster than freeing them one-by-one. - If footers are used, pointers that have been allocated in different - mspaces are not freed or cleared, and the count of all such pointers - is returned. For large arrays of pointers with poor locality, it - may be worthwhile to sort this array before calling bulk_free. -*/ -size_t dlbulk_free(void**, size_t n_elements); - -/* - pvalloc(size_t n); - Equivalent to valloc(minimum-page-that-holds(n)), that is, - round up n to nearest pagesize. - */ -void* dlpvalloc(size_t); - -/* - malloc_trim(size_t pad); - - If possible, gives memory back to the system (via negative arguments - to sbrk) if there is unused memory at the `high' end of the malloc - pool or in unused MMAP segments. You can call this after freeing - large blocks of memory to potentially reduce the system-level memory - requirements of a program. However, it cannot guarantee to reduce - memory. Under some allocation patterns, some large free blocks of - memory will be locked between two used chunks, so they cannot be - given back to the system. - - The `pad' argument to malloc_trim represents the amount of free - trailing space to leave untrimmed. If this argument is zero, only - the minimum amount of memory to maintain internal data structures - will be left. Non-zero arguments can be supplied to maintain enough - trailing space to service future expected allocations without having - to re-obtain memory from the system. - - Malloc_trim returns 1 if it actually released any memory, else 0. -*/ -int dlmalloc_trim(size_t); - -/* - malloc_stats(); - Prints on stderr the amount of space obtained from the system (both - via sbrk and mmap), the maximum amount (which may be more than - current if malloc_trim and/or munmap got called), and the current - number of bytes allocated via malloc (or realloc, etc) but not yet - freed. Note that this is the number of bytes allocated, not the - number requested. It will be larger than the number requested - because of alignment and bookkeeping overhead. Because it includes - alignment wastage as being in use, this figure may be greater than - zero even when no user-level chunks are allocated. - - The reported current and maximum system memory can be inaccurate if - a program makes other calls to system memory allocation functions - (normally sbrk) outside of malloc. - - malloc_stats prints only the most commonly interesting statistics. - More information can be obtained by calling mallinfo. - - malloc_stats is not compiled if NO_MALLOC_STATS is defined. -*/ -void dlmalloc_stats(void); - -#endif /* !ONLY_MSPACES */ - -/* - malloc_usable_size(void* p); - - Returns the number of bytes you can actually use in - an allocated chunk, which may be more than you requested (although - often not) due to alignment and minimum size constraints. - You can use this many bytes without worrying about - overwriting other allocated objects. This is not a particularly great - programming practice. malloc_usable_size can be more useful in - debugging and assertions, for example: - - p = malloc(n); - assert(malloc_usable_size(p) >= 256); -*/ -size_t dlmalloc_usable_size(const void*); - -#if MSPACES - -/* - mspace is an opaque type representing an independent - region of space that supports mspace_malloc, etc. -*/ -typedef void* mspace; - -/* - create_mspace creates and returns a new independent space with the - given initial capacity, or, if 0, the default granularity size. It - returns null if there is no system memory available to create the - space. If argument locked is non-zero, the space uses a separate - lock to control access. The capacity of the space will grow - dynamically as needed to service mspace_malloc requests. You can - control the sizes of incremental increases of this space by - compiling with a different DEFAULT_GRANULARITY or dynamically - setting with mallopt(M_GRANULARITY, value). -*/ -mspace create_mspace(size_t capacity, int locked); - -/* - destroy_mspace destroys the given space, and attempts to return all - of its memory back to the system, returning the total number of - bytes freed. After destruction, the results of access to all memory - used by the space become undefined. -*/ -size_t destroy_mspace(mspace msp); - -/* - create_mspace_with_base uses the memory supplied as the initial base - of a new mspace. Part (less than 128*sizeof(size_t) bytes) of this - space is used for bookkeeping, so the capacity must be at least this - large. (Otherwise 0 is returned.) When this initial space is - exhausted, additional memory will be obtained from the system. - Destroying this space will deallocate all additionally allocated - space (if possible) but not the initial base. -*/ -mspace create_mspace_with_base(void* base, size_t capacity, int locked); - -/* - mspace_track_large_chunks controls whether requests for large chunks - are allocated in their own untracked mmapped regions, separate from - others in this mspace. By default large chunks are not tracked, - which reduces fragmentation. However, such chunks are not - necessarily released to the system upon destroy_mspace. Enabling - tracking by setting to true may increase fragmentation, but avoids - leakage when relying on destroy_mspace to release all memory - allocated using this space. The function returns the previous - setting. -*/ -int mspace_track_large_chunks(mspace msp, int enable); - -#if !NO_MALLINFO -/* - mspace_mallinfo behaves as mallinfo, but reports properties of - the given space. -*/ -struct mallinfo mspace_mallinfo(mspace msp); -#endif /* NO_MALLINFO */ - -/* - An alias for mallopt. -*/ -int mspace_mallopt(int, int); - -/* - The following operate identically to their malloc counterparts - but operate only for the given mspace argument -*/ -void* mspace_malloc(mspace msp, size_t bytes); -void mspace_free(mspace msp, void* mem); -void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size); -void* mspace_realloc(mspace msp, void* mem, size_t newsize); -void* mspace_realloc_in_place(mspace msp, void* mem, size_t newsize); -void* mspace_memalign(mspace msp, size_t alignment, size_t bytes); -void** mspace_independent_calloc(mspace msp, size_t n_elements, - size_t elem_size, void* chunks[]); -void** mspace_independent_comalloc(mspace msp, size_t n_elements, - size_t sizes[], void* chunks[]); -size_t mspace_bulk_free(mspace msp, void**, size_t n_elements); -size_t mspace_usable_size(const void* mem); -void mspace_malloc_stats(mspace msp); -int mspace_trim(mspace msp, size_t pad); -size_t mspace_footprint(mspace msp); -size_t mspace_max_footprint(mspace msp); -size_t mspace_footprint_limit(mspace msp); -size_t mspace_set_footprint_limit(mspace msp, size_t bytes); -void mspace_inspect_all(mspace msp, - void(*handler)(void *, void *, size_t, void*), - void* arg); -#endif /* MSPACES */ - -#ifdef __cplusplus -}; /* end of extern "C" */ -#endif - -#endif /* MALLOC_280_H */ diff --git a/native/src/crt0/linux_syscall_support.h b/native/src/crt0/linux_syscall_support.h deleted file mode 100644 index c80fbfd73..000000000 --- a/native/src/crt0/linux_syscall_support.h +++ /dev/null @@ -1,5375 +0,0 @@ -/* Copyright 2005-2011 Google LLC - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are - * met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following disclaimer - * in the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Google LLC nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - * --- - * Author: Markus Gutschke - */ - -/* This file includes Linux-specific support functions common to the - * coredumper and the thread lister; primarily, this is a collection - * of direct system calls, and a couple of symbols missing from - * standard header files. - * There are a few options that the including file can set to control - * the behavior of this file: - * - * SYS_CPLUSPLUS: - * The entire header file will normally be wrapped in 'extern "C" { }", - * making it suitable for compilation as both C and C++ source. If you - * do not want to do this, you can set the SYS_CPLUSPLUS macro to inhibit - * the wrapping. N.B. doing so will suppress inclusion of all prerequisite - * system header files, too. It is the caller's responsibility to provide - * the necessary definitions. - * - * SYS_ERRNO: - * All system calls will update "errno" unless overridden by setting the - * SYS_ERRNO macro prior to including this file. SYS_ERRNO should be - * an l-value. - * - * SYS_INLINE: - * New symbols will be defined "static inline", unless overridden by - * the SYS_INLINE macro. - * - * SYS_LINUX_SYSCALL_SUPPORT_H - * This macro is used to avoid multiple inclusions of this header file. - * If you need to include this file more than once, make sure to - * unset SYS_LINUX_SYSCALL_SUPPORT_H before each inclusion. - * - * SYS_PREFIX: - * New system calls will have a prefix of "sys_" unless overridden by - * the SYS_PREFIX macro. Valid values for this macro are [0..9] which - * results in prefixes "sys[0..9]_". It is also possible to set this - * macro to -1, which avoids all prefixes. - * - * SYS_SYSCALL_ENTRYPOINT: - * Some applications (such as sandboxes that filter system calls), need - * to be able to run custom-code each time a system call is made. If this - * macro is defined, it expands to the name of a "common" symbol. If - * this symbol is assigned a non-NULL pointer value, it is used as the - * address of the system call entrypoint. - * A pointer to this symbol can be obtained by calling - * get_syscall_entrypoint() - * - * This file defines a few internal symbols that all start with "LSS_". - * Do not access these symbols from outside this file. They are not part - * of the supported API. - */ -#ifndef SYS_LINUX_SYSCALL_SUPPORT_H -#define SYS_LINUX_SYSCALL_SUPPORT_H - -/* We currently only support x86-32, x86-64, ARM, MIPS, PPC, s390 and s390x - * on Linux. - * Porting to other related platforms should not be difficult. - */ -#if (defined(__i386__) || defined(__x86_64__) || defined(__ARM_ARCH_3__) || \ - defined(__mips__) || defined(__PPC__) || defined(__ARM_EABI__) || \ - defined(__aarch64__) || defined(__s390__) || defined(__e2k__) || \ - (defined(__riscv) && __riscv_xlen == 64) || defined(__loongarch_lp64)) \ - && (defined(__linux) || defined(__ANDROID__)) - -#ifndef SYS_CPLUSPLUS -#ifdef __cplusplus -/* Some system header files in older versions of gcc neglect to properly - * handle being included from C++. As it appears to be harmless to have - * multiple nested 'extern "C"' blocks, just add another one here. - */ -extern "C" { -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __mips__ -/* Include definitions of the ABI currently in use. */ -#ifdef __ANDROID__ -/* Android doesn't have sgidefs.h, but does have asm/sgidefs.h, - * which has the definitions we need. - */ -#include -#else -#include -#endif -#endif -#endif - -/* Some libcs, for example Android NDK and musl, #define these - * macros as aliases to their non-64 counterparts. To avoid naming - * conflict, remove them. - * - * These are restored by the corresponding #pragma pop_macro near - * the end of this file. - */ -#pragma push_macro("stat64") -#pragma push_macro("fstat64") -#pragma push_macro("lstat64") -#pragma push_macro("pread64") -#pragma push_macro("pwrite64") -#pragma push_macro("getdents64") -#undef stat64 -#undef fstat64 -#undef lstat64 -#undef pread64 -#undef pwrite64 -#undef getdents64 - -#if defined(__ANDROID__) && defined(__x86_64__) -// A number of x86_64 syscalls are blocked by seccomp on recent Android; -// undefine them so that modern alternatives will be used instead where -// possible. -// The alternative syscalls have been sanity checked against linux-3.4+; -// older versions might not work. -# undef __NR_getdents -# undef __NR_dup2 -# undef __NR_fork -# undef __NR_getpgrp -# undef __NR_open -# undef __NR_poll -# undef __NR_readlink -# undef __NR_stat -# undef __NR_unlink -# undef __NR_pipe -#endif - -#if defined(__ANDROID__) -// waitpid is blocked by seccomp on all architectures on recent Android. -# undef __NR_waitpid -#endif - -/* As glibc often provides subtly incompatible data structures (and implicit - * wrapper functions that convert them), we provide our own kernel data - * structures for use by the system calls. - * These structures have been developed by using Linux 2.6.23 headers for - * reference. Note though, we do not care about exact API compatibility - * with the kernel, and in fact the kernel often does not have a single - * API that works across architectures. Instead, we try to mimic the glibc - * API where reasonable, and only guarantee ABI compatibility with the - * kernel headers. - * Most notably, here are a few changes that were made to the structures - * defined by kernel headers: - * - * - we only define structures, but not symbolic names for kernel data - * types. For the latter, we directly use the native C datatype - * (i.e. "unsigned" instead of "mode_t"). - * - in a few cases, it is possible to define identical structures for - * both 32bit (e.g. i386) and 64bit (e.g. x86-64) platforms by - * standardizing on the 64bit version of the data types. In particular, - * this means that we use "unsigned" where the 32bit headers say - * "unsigned long". - * - overall, we try to minimize the number of cases where we need to - * conditionally define different structures. - * - the "struct kernel_sigaction" class of structures have been - * modified to more closely mimic glibc's API by introducing an - * anonymous union for the function pointer. - * - a small number of field names had to have an underscore appended to - * them, because glibc defines a global macro by the same name. - */ - -/* include/linux/dirent.h */ -struct kernel_dirent64 { - unsigned long long d_ino; - long long d_off; - unsigned short d_reclen; - unsigned char d_type; - char d_name[256]; -}; - -/* include/linux/dirent.h */ -#if !defined(__NR_getdents) -// when getdents is not available, getdents64 is used for both. -#define kernel_dirent kernel_dirent64 -#else -struct kernel_dirent { - long d_ino; - long d_off; - unsigned short d_reclen; - char d_name[256]; -}; -#endif - -/* include/linux/uio.h */ -struct kernel_iovec { - void *iov_base; - unsigned long iov_len; -}; - -/* include/linux/socket.h */ -struct kernel_msghdr { - void *msg_name; - int msg_namelen; - struct kernel_iovec*msg_iov; - unsigned long msg_iovlen; - void *msg_control; - unsigned long msg_controllen; - unsigned msg_flags; -}; - -/* include/asm-generic/poll.h */ -struct kernel_pollfd { - int fd; - short events; - short revents; -}; - -/* include/linux/resource.h */ -struct kernel_rlimit { - unsigned long rlim_cur; - unsigned long rlim_max; -}; - -/* include/linux/time.h */ -struct kernel_timespec { - long tv_sec; - long tv_nsec; -}; - -/* include/linux/time.h */ -struct kernel_timeval { - long tv_sec; - long tv_usec; -}; - -/* include/linux/time.h */ -struct kernel_itimerval { - struct kernel_timeval it_interval; - struct kernel_timeval it_value; -}; - -/* include/linux/resource.h */ -struct kernel_rusage { - struct kernel_timeval ru_utime; - struct kernel_timeval ru_stime; - long ru_maxrss; - long ru_ixrss; - long ru_idrss; - long ru_isrss; - long ru_minflt; - long ru_majflt; - long ru_nswap; - long ru_inblock; - long ru_oublock; - long ru_msgsnd; - long ru_msgrcv; - long ru_nsignals; - long ru_nvcsw; - long ru_nivcsw; -}; - -#if defined(__i386__) || defined(__ARM_EABI__) || defined(__ARM_ARCH_3__) \ - || defined(__PPC__) || (defined(__s390__) && !defined(__s390x__)) \ - || defined(__e2k__) - -/* include/asm-{arm,i386,mips,ppc}/signal.h */ -struct kernel_old_sigaction { - union { - void (*sa_handler_)(int); - void (*sa_sigaction_)(int, siginfo_t *, void *); - }; - unsigned long sa_mask; - unsigned long sa_flags; - void (*sa_restorer)(void); -} __attribute__((packed,aligned(4))); -#elif (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI32) - #define kernel_old_sigaction kernel_sigaction -#elif defined(__aarch64__) || defined(__riscv) || defined(__loongarch_lp64) - // No kernel_old_sigaction defined for arm64 riscv and loongarch64. -#endif - -/* Some kernel functions (e.g. sigaction() in 2.6.23) require that the - * exactly match the size of the signal set, even though the API was - * intended to be extensible. We define our own KERNEL_NSIG to deal with - * this. - * Please note that glibc provides signals [1.._NSIG-1], whereas the - * kernel (and this header) provides the range [1..KERNEL_NSIG]. The - * actual number of signals is obviously the same, but the constants - * differ by one. - */ -#ifdef __mips__ -#define KERNEL_NSIG 128 -#else -#define KERNEL_NSIG 64 -#endif - -/* include/asm-{arm,aarch64,i386,mips,x86_64}/signal.h */ -struct kernel_sigset_t { - unsigned long sig[(KERNEL_NSIG + 8*sizeof(unsigned long) - 1)/ - (8*sizeof(unsigned long))]; -}; - -/* include/asm-{arm,i386,mips,x86_64,ppc}/signal.h */ -struct kernel_sigaction { -#ifdef __mips__ - unsigned long sa_flags; - union { - void (*sa_handler_)(int); - void (*sa_sigaction_)(int, siginfo_t *, void *); - }; - struct kernel_sigset_t sa_mask; -#else - union { - void (*sa_handler_)(int); - void (*sa_sigaction_)(int, siginfo_t *, void *); - }; - unsigned long sa_flags; -#if !defined(__riscv) && !defined(__loongarch_lp64) - void (*sa_restorer)(void); -#endif - struct kernel_sigset_t sa_mask; -#endif -}; - -/* include/linux/socket.h */ -struct kernel_sockaddr { - unsigned short sa_family; - char sa_data[14]; -}; - -/* include/asm-{arm,aarch64,i386,mips,ppc,s390}/stat.h */ -#ifdef __mips__ -#if _MIPS_SIM == _MIPS_SIM_ABI64 -typedef unsigned long long kernel_blkcnt_t; -typedef unsigned kernel_blksize_t; -typedef unsigned kernel_dev_t; -typedef unsigned kernel_gid_t; -typedef unsigned long long kernel_ino_t; -typedef unsigned kernel_mode_t; -typedef unsigned kernel_nlink_t; -typedef long long kernel_off_t; -typedef unsigned kernel_time_t; -typedef unsigned kernel_uid_t; -struct kernel_stat { -#else -struct kernel_stat64 { -#endif - unsigned st_dev; - unsigned __pad0[3]; - unsigned long long st_ino; - unsigned st_mode; - unsigned st_nlink; - unsigned st_uid; - unsigned st_gid; - unsigned st_rdev; - unsigned __pad1[3]; - long long st_size; - unsigned st_atime_; - unsigned st_atime_nsec_; - unsigned st_mtime_; - unsigned st_mtime_nsec_; - unsigned st_ctime_; - unsigned st_ctime_nsec_; - unsigned st_blksize; - unsigned __pad2; - unsigned long long st_blocks; -}; -#elif defined __PPC__ -struct kernel_stat64 { - unsigned long long st_dev; - unsigned long long st_ino; - unsigned st_mode; - unsigned st_nlink; - unsigned st_uid; - unsigned st_gid; - unsigned long long st_rdev; - unsigned short int __pad2; - long long st_size; - long st_blksize; - long long st_blocks; - long st_atime_; - unsigned long st_atime_nsec_; - long st_mtime_; - unsigned long st_mtime_nsec_; - long st_ctime_; - unsigned long st_ctime_nsec_; - unsigned long __unused4; - unsigned long __unused5; -}; -#elif defined(__e2k__) -struct kernel_stat64 { - unsigned long long st_dev; - unsigned long long st_ino; - unsigned int st_mode; - unsigned int st_nlink; - unsigned int st_uid; - unsigned int st_gid; - unsigned long long st_rdev; - long long st_size; - int st_blksize; - int __pad2; - unsigned long long st_blocks; - int st_atime_; - unsigned int st_atime_nsec_; - int st_mtime_; - unsigned int st_mtime_nsec_; - int st_ctime_; - unsigned int st_ctime_nsec_; - unsigned int __unused4; - unsigned int __unused5; -}; -#else -struct kernel_stat64 { - unsigned long long st_dev; - unsigned char __pad0[4]; - unsigned __st_ino; - unsigned st_mode; - unsigned st_nlink; - unsigned st_uid; - unsigned st_gid; - unsigned long long st_rdev; - unsigned char __pad3[4]; - long long st_size; - unsigned st_blksize; - unsigned long long st_blocks; - unsigned st_atime_; - unsigned st_atime_nsec_; - unsigned st_mtime_; - unsigned st_mtime_nsec_; - unsigned st_ctime_; - unsigned st_ctime_nsec_; - unsigned long long st_ino; -}; -#endif - -/* include/asm-{arm,aarch64,i386,mips,x86_64,ppc,s390}/stat.h */ -#if defined(__i386__) || defined(__ARM_ARCH_3__) || defined(__ARM_EABI__) -typedef unsigned kernel_blkcnt_t; -typedef unsigned kernel_blksize_t; -typedef unsigned short kernel_dev_t; -typedef unsigned short kernel_gid_t; -typedef unsigned kernel_ino_t; -typedef unsigned short kernel_mode_t; -typedef unsigned short kernel_nlink_t; -typedef unsigned kernel_off_t; -typedef unsigned kernel_time_t; -typedef unsigned short kernel_uid_t; -struct kernel_stat { - /* The kernel headers suggest that st_dev and st_rdev should be 32bit - * quantities encoding 12bit major and 20bit minor numbers in an interleaved - * format. In reality, we do not see useful data in the top bits. So, - * we'll leave the padding in here, until we find a better solution. - */ - kernel_dev_t st_dev; - short pad1; - kernel_ino_t st_ino; - kernel_mode_t st_mode; - kernel_nlink_t st_nlink; - kernel_uid_t st_uid; - kernel_gid_t st_gid; - kernel_dev_t st_rdev; - short pad2; - kernel_off_t st_size; - kernel_blksize_t st_blksize; - kernel_blkcnt_t st_blocks; - kernel_time_t st_atime_; - unsigned st_atime_nsec_; - kernel_time_t st_mtime_; - unsigned st_mtime_nsec_; - kernel_time_t st_ctime_; - unsigned st_ctime_nsec_; - unsigned __unused4; - unsigned __unused5; -}; -#elif defined(__x86_64__) -typedef int64_t kernel_blkcnt_t; -typedef int64_t kernel_blksize_t; -typedef uint64_t kernel_dev_t; -typedef unsigned kernel_gid_t; -typedef uint64_t kernel_ino_t; -typedef unsigned kernel_mode_t; -typedef uint64_t kernel_nlink_t; -typedef int64_t kernel_off_t; -typedef uint64_t kernel_time_t; -typedef unsigned kernel_uid_t; -struct kernel_stat { - kernel_dev_t st_dev; - kernel_ino_t st_ino; - kernel_nlink_t st_nlink; - kernel_mode_t st_mode; - kernel_uid_t st_uid; - kernel_gid_t st_gid; - unsigned __pad0; - kernel_dev_t st_rdev; - kernel_off_t st_size; - kernel_blksize_t st_blksize; - kernel_blkcnt_t st_blocks; - kernel_time_t st_atime_; - uint64_t st_atime_nsec_; - kernel_time_t st_mtime_; - uint64_t st_mtime_nsec_; - kernel_time_t st_ctime_; - uint64_t st_ctime_nsec_; - int64_t __unused4[3]; -}; -#elif defined(__PPC__) -typedef unsigned long kernel_blkcnt_t; -typedef unsigned long kernel_blksize_t; -typedef unsigned kernel_dev_t; -typedef unsigned kernel_gid_t; -typedef unsigned long kernel_ino_t; -typedef unsigned long kernel_mode_t; -typedef unsigned short kernel_nlink_t; -typedef long kernel_off_t; -typedef unsigned long kernel_time_t; -typedef unsigned kernel_uid_t; -struct kernel_stat { - kernel_dev_t st_dev; - kernel_ino_t st_ino; - kernel_mode_t st_mode; - kernel_nlink_t st_nlink; - kernel_gid_t st_uid; - kernel_uid_t st_gid; - kernel_dev_t st_rdev; - kernel_off_t st_size; - kernel_blksize_t st_blksize; - kernel_blkcnt_t st_blocks; - kernel_time_t st_atime_; - unsigned long st_atime_nsec_; - kernel_time_t st_mtime_; - unsigned long st_mtime_nsec_; - kernel_time_t st_ctime_; - unsigned long st_ctime_nsec_; - unsigned long __unused4; - unsigned long __unused5; -}; -#elif (defined(__mips__) && _MIPS_SIM != _MIPS_SIM_ABI64) -typedef int kernel_blkcnt_t; -typedef int kernel_blksize_t; -typedef unsigned kernel_dev_t; -typedef unsigned kernel_gid_t; -typedef unsigned kernel_ino_t; -typedef unsigned kernel_mode_t; -typedef unsigned kernel_nlink_t; -typedef long kernel_off_t; -typedef long kernel_time_t; -typedef unsigned kernel_uid_t; -struct kernel_stat { - kernel_dev_t st_dev; - int st_pad1[3]; - kernel_ino_t st_ino; - kernel_mode_t st_mode; - kernel_nlink_t st_nlink; - kernel_uid_t st_uid; - kernel_gid_t st_gid; - kernel_dev_t st_rdev; - int st_pad2[2]; - kernel_off_t st_size; - int st_pad3; - kernel_time_t st_atime_; - long st_atime_nsec_; - kernel_time_t st_mtime_; - long st_mtime_nsec_; - kernel_time_t st_ctime_; - long st_ctime_nsec_; - kernel_blksize_t st_blksize; - kernel_blkcnt_t st_blocks; - int st_pad4[14]; -}; -#elif defined(__aarch64__) || defined(__riscv) || defined(__loongarch_lp64) -typedef long kernel_blkcnt_t; -typedef int kernel_blksize_t; -typedef unsigned long kernel_dev_t; -typedef unsigned int kernel_gid_t; -typedef unsigned long kernel_ino_t; -typedef unsigned int kernel_mode_t; -typedef unsigned int kernel_nlink_t; -typedef long kernel_off_t; -typedef long kernel_time_t; -typedef unsigned int kernel_uid_t; -struct kernel_stat { - kernel_dev_t st_dev; - kernel_ino_t st_ino; - kernel_mode_t st_mode; - kernel_nlink_t st_nlink; - kernel_uid_t st_uid; - kernel_gid_t st_gid; - kernel_dev_t st_rdev; - unsigned long __pad1; - kernel_off_t st_size; - kernel_blksize_t st_blksize; - int __pad2; - kernel_blkcnt_t st_blocks; - kernel_time_t st_atime_; - unsigned long st_atime_nsec_; - kernel_time_t st_mtime_; - unsigned long st_mtime_nsec_; - kernel_time_t st_ctime_; - unsigned long st_ctime_nsec_; - unsigned int __unused4; - unsigned int __unused5; -}; -#elif defined(__s390x__) -typedef long kernel_blkcnt_t; -typedef unsigned long kernel_blksize_t; -typedef unsigned long kernel_dev_t; -typedef unsigned int kernel_gid_t; -typedef unsigned long kernel_ino_t; -typedef unsigned int kernel_mode_t; -typedef unsigned long kernel_nlink_t; -typedef unsigned long kernel_off_t; -typedef unsigned long kernel_time_t; -typedef unsigned int kernel_uid_t; -struct kernel_stat { - kernel_dev_t st_dev; - kernel_ino_t st_ino; - kernel_nlink_t st_nlink; - kernel_mode_t st_mode; - kernel_uid_t st_uid; - kernel_gid_t st_gid; - unsigned int __pad1; - kernel_dev_t st_rdev; - kernel_off_t st_size; - kernel_time_t st_atime_; - unsigned long st_atime_nsec_; - kernel_time_t st_mtime_; - unsigned long st_mtime_nsec_; - kernel_time_t st_ctime_; - unsigned long st_ctime_nsec_; - kernel_blksize_t st_blksize; - kernel_blkcnt_t st_blocks; - unsigned long __unused[3]; -}; -#elif defined(__s390__) -typedef unsigned long kernel_blkcnt_t; -typedef unsigned long kernel_blksize_t; -typedef unsigned short kernel_dev_t; -typedef unsigned short kernel_gid_t; -typedef unsigned long kernel_ino_t; -typedef unsigned short kernel_mode_t; -typedef unsigned short kernel_nlink_t; -typedef unsigned long kernel_off_t; -typedef unsigned long kernel_time_t; -typedef unsigned short kernel_uid_t; -struct kernel_stat { - kernel_dev_t st_dev; - unsigned short __pad1; - kernel_ino_t st_ino; - kernel_mode_t st_mode; - kernel_nlink_t st_nlink; - kernel_uid_t st_uid; - kernel_gid_t st_gid; - kernel_dev_t st_rdev; - unsigned short __pad2; - kernel_off_t st_size; - kernel_blksize_t st_blksize; - kernel_blkcnt_t st_blocks; - kernel_time_t st_atime_; - unsigned long st_atime_nsec_; - kernel_time_t st_mtime_; - unsigned long st_mtime_nsec_; - kernel_time_t st_ctime_; - unsigned long st_ctime_nsec_; - unsigned long __unused4; - unsigned long __unused5; -}; -#elif defined(__e2k__) -typedef unsigned long kernel_blkcnt_t; -typedef unsigned long kernel_blksize_t; -typedef unsigned long kernel_dev_t; -typedef unsigned int kernel_gid_t; -typedef unsigned long kernel_ino_t; -typedef unsigned int kernel_mode_t; -typedef unsigned long kernel_nlink_t; -typedef unsigned long kernel_off_t; -typedef unsigned long kernel_time_t; -typedef unsigned int kernel_uid_t; -struct kernel_stat { - kernel_dev_t st_dev; - kernel_ino_t st_ino; - kernel_mode_t st_mode; - kernel_nlink_t st_nlink; - kernel_uid_t st_uid; - kernel_gid_t st_gid; - kernel_dev_t st_rdev; - kernel_off_t st_size; - kernel_blksize_t st_blksize; - kernel_blkcnt_t st_blocks; - kernel_time_t st_atime_; - unsigned long st_atime_nsec_; - kernel_time_t st_mtime_; - unsigned long st_mtime_nsec_; - kernel_time_t st_ctime_; - unsigned long st_ctime_nsec_; -}; -#endif - -/* include/asm-{arm,aarch64,i386,mips,x86_64,ppc,s390}/statfs.h */ -#ifdef __mips__ -#if _MIPS_SIM != _MIPS_SIM_ABI64 -struct kernel_statfs64 { - unsigned long f_type; - unsigned long f_bsize; - unsigned long f_frsize; - unsigned long __pad; - unsigned long long f_blocks; - unsigned long long f_bfree; - unsigned long long f_files; - unsigned long long f_ffree; - unsigned long long f_bavail; - struct { int val[2]; } f_fsid; - unsigned long f_namelen; - unsigned long f_spare[6]; -}; -#endif -#elif defined(__s390__) -/* See also arch/s390/include/asm/compat.h */ -struct kernel_statfs64 { - unsigned int f_type; - unsigned int f_bsize; - unsigned long long f_blocks; - unsigned long long f_bfree; - unsigned long long f_bavail; - unsigned long long f_files; - unsigned long long f_ffree; - struct { int val[2]; } f_fsid; - unsigned int f_namelen; - unsigned int f_frsize; - unsigned int f_flags; - unsigned int f_spare[4]; -}; -#elif !defined(__x86_64__) -struct kernel_statfs64 { - unsigned long f_type; - unsigned long f_bsize; - unsigned long long f_blocks; - unsigned long long f_bfree; - unsigned long long f_bavail; - unsigned long long f_files; - unsigned long long f_ffree; - struct { int val[2]; } f_fsid; - unsigned long f_namelen; - unsigned long f_frsize; - unsigned long f_spare[5]; -}; -#endif - -/* include/asm-{arm,i386,mips,x86_64,ppc,generic,s390}/statfs.h */ -#ifdef __mips__ -struct kernel_statfs { - long f_type; - long f_bsize; - long f_frsize; - long f_blocks; - long f_bfree; - long f_files; - long f_ffree; - long f_bavail; - struct { int val[2]; } f_fsid; - long f_namelen; - long f_spare[6]; -}; -#elif defined(__x86_64__) -struct kernel_statfs { - /* x86_64 actually defines all these fields as signed, whereas all other */ - /* platforms define them as unsigned. Leaving them at unsigned should not */ - /* cause any problems. Make sure these are 64-bit even on x32. */ - uint64_t f_type; - uint64_t f_bsize; - uint64_t f_blocks; - uint64_t f_bfree; - uint64_t f_bavail; - uint64_t f_files; - uint64_t f_ffree; - struct { int val[2]; } f_fsid; - uint64_t f_namelen; - uint64_t f_frsize; - uint64_t f_spare[5]; -}; -#elif defined(__s390__) -struct kernel_statfs { - unsigned int f_type; - unsigned int f_bsize; - unsigned long f_blocks; - unsigned long f_bfree; - unsigned long f_bavail; - unsigned long f_files; - unsigned long f_ffree; - struct { int val[2]; } f_fsid; - unsigned int f_namelen; - unsigned int f_frsize; - unsigned int f_flags; - unsigned int f_spare[4]; -}; -#else -struct kernel_statfs { - unsigned long f_type; - unsigned long f_bsize; - unsigned long f_blocks; - unsigned long f_bfree; - unsigned long f_bavail; - unsigned long f_files; - unsigned long f_ffree; - struct { int val[2]; } f_fsid; - unsigned long f_namelen; - unsigned long f_frsize; - unsigned long f_spare[5]; -}; -#endif - -struct kernel_statx_timestamp { - int64_t tv_sec; - uint32_t tv_nsec; - int32_t __reserved; -}; - -struct kernel_statx { - uint32_t stx_mask; - uint32_t stx_blksize; - uint64_t stx_attributes; - uint32_t stx_nlink; - uint32_t stx_uid; - uint32_t stx_gid; - uint16_t stx_mode; - uint16_t __spare0[1]; - uint64_t stx_ino; - uint64_t stx_size; - uint64_t stx_blocks; - uint64_t stx_attributes_mask; - struct kernel_statx_timestamp stx_atime; - struct kernel_statx_timestamp stx_btime; - struct kernel_statx_timestamp stx_ctime; - struct kernel_statx_timestamp stx_mtime; - uint32_t stx_rdev_major; - uint32_t stx_rdev_minor; - uint32_t stx_dev_major; - uint32_t stx_dev_minor; - uint64_t stx_mnt_id; - uint64_t __spare2; - uint64_t __spare3[12]; -}; - -/* Definitions missing from the standard header files */ -#ifndef O_DIRECTORY -#if defined(__ARM_ARCH_3__) || defined(__ARM_EABI__) || defined(__aarch64__) -#define O_DIRECTORY 0040000 -#else -#define O_DIRECTORY 0200000 -#endif -#endif -#ifndef NT_PRXFPREG -#define NT_PRXFPREG 0x46e62b7f -#endif -#ifndef PTRACE_GETFPXREGS -#define PTRACE_GETFPXREGS ((enum __ptrace_request)18) -#endif -#ifndef PR_GET_DUMPABLE -#define PR_GET_DUMPABLE 3 -#endif -#ifndef PR_SET_DUMPABLE -#define PR_SET_DUMPABLE 4 -#endif -#ifndef PR_GET_SECCOMP -#define PR_GET_SECCOMP 21 -#endif -#ifndef PR_SET_SECCOMP -#define PR_SET_SECCOMP 22 -#endif -#ifndef AT_FDCWD -#define AT_FDCWD (-100) -#endif -#ifndef AT_SYMLINK_NOFOLLOW -#define AT_SYMLINK_NOFOLLOW 0x100 -#endif -#ifndef AT_REMOVEDIR -#define AT_REMOVEDIR 0x200 -#endif -#ifndef AT_NO_AUTOMOUNT -#define AT_NO_AUTOMOUNT 0x800 -#endif -#ifndef AT_EMPTY_PATH -#define AT_EMPTY_PATH 0x1000 -#endif -#ifndef STATX_BASIC_STATS -#define STATX_BASIC_STATS 0x000007ffU -#endif -#ifndef AT_STATX_SYNC_AS_STAT -#define AT_STATX_SYNC_AS_STAT 0x0000 -#endif -#ifndef MREMAP_FIXED -#define MREMAP_FIXED 2 -#endif -#ifndef SA_RESTORER -#define SA_RESTORER 0x04000000 -#endif -#ifndef CPUCLOCK_PROF -#define CPUCLOCK_PROF 0 -#endif -#ifndef CPUCLOCK_VIRT -#define CPUCLOCK_VIRT 1 -#endif -#ifndef CPUCLOCK_SCHED -#define CPUCLOCK_SCHED 2 -#endif -#ifndef CPUCLOCK_PERTHREAD_MASK -#define CPUCLOCK_PERTHREAD_MASK 4 -#endif -#ifndef MAKE_PROCESS_CPUCLOCK -#define MAKE_PROCESS_CPUCLOCK(pid, clock) \ - ((int)(~(unsigned)(pid) << 3) | (int)(clock)) -#endif -#ifndef MAKE_THREAD_CPUCLOCK -#define MAKE_THREAD_CPUCLOCK(tid, clock) \ - ((int)(~(unsigned)(tid) << 3) | \ - (int)((clock) | CPUCLOCK_PERTHREAD_MASK)) -#endif - -#ifndef FUTEX_WAIT -#define FUTEX_WAIT 0 -#endif -#ifndef FUTEX_WAKE -#define FUTEX_WAKE 1 -#endif -#ifndef FUTEX_FD -#define FUTEX_FD 2 -#endif -#ifndef FUTEX_REQUEUE -#define FUTEX_REQUEUE 3 -#endif -#ifndef FUTEX_CMP_REQUEUE -#define FUTEX_CMP_REQUEUE 4 -#endif -#ifndef FUTEX_WAKE_OP -#define FUTEX_WAKE_OP 5 -#endif -#ifndef FUTEX_LOCK_PI -#define FUTEX_LOCK_PI 6 -#endif -#ifndef FUTEX_UNLOCK_PI -#define FUTEX_UNLOCK_PI 7 -#endif -#ifndef FUTEX_TRYLOCK_PI -#define FUTEX_TRYLOCK_PI 8 -#endif -#ifndef FUTEX_PRIVATE_FLAG -#define FUTEX_PRIVATE_FLAG 128 -#endif -#ifndef FUTEX_CMD_MASK -#define FUTEX_CMD_MASK ~FUTEX_PRIVATE_FLAG -#endif -#ifndef FUTEX_WAIT_PRIVATE -#define FUTEX_WAIT_PRIVATE (FUTEX_WAIT | FUTEX_PRIVATE_FLAG) -#endif -#ifndef FUTEX_WAKE_PRIVATE -#define FUTEX_WAKE_PRIVATE (FUTEX_WAKE | FUTEX_PRIVATE_FLAG) -#endif -#ifndef FUTEX_REQUEUE_PRIVATE -#define FUTEX_REQUEUE_PRIVATE (FUTEX_REQUEUE | FUTEX_PRIVATE_FLAG) -#endif -#ifndef FUTEX_CMP_REQUEUE_PRIVATE -#define FUTEX_CMP_REQUEUE_PRIVATE (FUTEX_CMP_REQUEUE | FUTEX_PRIVATE_FLAG) -#endif -#ifndef FUTEX_WAKE_OP_PRIVATE -#define FUTEX_WAKE_OP_PRIVATE (FUTEX_WAKE_OP | FUTEX_PRIVATE_FLAG) -#endif -#ifndef FUTEX_LOCK_PI_PRIVATE -#define FUTEX_LOCK_PI_PRIVATE (FUTEX_LOCK_PI | FUTEX_PRIVATE_FLAG) -#endif -#ifndef FUTEX_UNLOCK_PI_PRIVATE -#define FUTEX_UNLOCK_PI_PRIVATE (FUTEX_UNLOCK_PI | FUTEX_PRIVATE_FLAG) -#endif -#ifndef FUTEX_TRYLOCK_PI_PRIVATE -#define FUTEX_TRYLOCK_PI_PRIVATE (FUTEX_TRYLOCK_PI | FUTEX_PRIVATE_FLAG) -#endif - - -#if defined(__x86_64__) -#ifndef ARCH_SET_GS -#define ARCH_SET_GS 0x1001 -#endif -#ifndef ARCH_GET_GS -#define ARCH_GET_GS 0x1004 -#endif -#endif - -#if defined(__i386__) -#ifndef __NR_quotactl -#define __NR_quotactl 131 -#endif -#ifndef __NR_setresuid -#define __NR_setresuid 164 -#define __NR_getresuid 165 -#define __NR_setresgid 170 -#define __NR_getresgid 171 -#endif -#ifndef __NR_rt_sigaction -#define __NR_rt_sigreturn 173 -#define __NR_rt_sigaction 174 -#define __NR_rt_sigprocmask 175 -#define __NR_rt_sigpending 176 -#define __NR_rt_sigsuspend 179 -#endif -#ifndef __NR_pread64 -#define __NR_pread64 180 -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 181 -#endif -#ifndef __NR_ugetrlimit -#define __NR_ugetrlimit 191 -#endif -#ifndef __NR_stat64 -#define __NR_stat64 195 -#endif -#ifndef __NR_fstat64 -#define __NR_fstat64 197 -#endif -#ifndef __NR_setresuid32 -#define __NR_setresuid32 208 -#define __NR_getresuid32 209 -#define __NR_setresgid32 210 -#define __NR_getresgid32 211 -#endif -#ifndef __NR_setfsuid32 -#define __NR_setfsuid32 215 -#define __NR_setfsgid32 216 -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 220 -#endif -#ifndef __NR_gettid -#define __NR_gettid 224 -#endif -#ifndef __NR_readahead -#define __NR_readahead 225 -#endif -#ifndef __NR_setxattr -#define __NR_setxattr 226 -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr 227 -#endif -#ifndef __NR_getxattr -#define __NR_getxattr 229 -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr 230 -#endif -#ifndef __NR_listxattr -#define __NR_listxattr 232 -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr 233 -#endif -#ifndef __NR_tkill -#define __NR_tkill 238 -#endif -#ifndef __NR_futex -#define __NR_futex 240 -#endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity 241 -#define __NR_sched_getaffinity 242 -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address 258 -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime 265 -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres 266 -#endif -#ifndef __NR_statfs64 -#define __NR_statfs64 268 -#endif -#ifndef __NR_fstatfs64 -#define __NR_fstatfs64 269 -#endif -#ifndef __NR_fadvise64_64 -#define __NR_fadvise64_64 272 -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set 289 -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get 290 -#endif -#ifndef __NR_openat -#define __NR_openat 295 -#endif -#ifndef __NR_fstatat64 -#define __NR_fstatat64 300 -#endif -#ifndef __NR_unlinkat -#define __NR_unlinkat 301 -#endif -#ifndef __NR_move_pages -#define __NR_move_pages 317 -#endif -#ifndef __NR_getcpu -#define __NR_getcpu 318 -#endif -#ifndef __NR_fallocate -#define __NR_fallocate 324 -#endif -#ifndef __NR_getrandom -#define __NR_getrandom 355 -#endif -/* End of i386 definitions */ -#elif defined(__ARM_ARCH_3__) || defined(__ARM_EABI__) -#ifndef __NR_setresuid -#define __NR_setresuid (__NR_SYSCALL_BASE + 164) -#define __NR_getresuid (__NR_SYSCALL_BASE + 165) -#define __NR_setresgid (__NR_SYSCALL_BASE + 170) -#define __NR_getresgid (__NR_SYSCALL_BASE + 171) -#endif -#ifndef __NR_rt_sigaction -#define __NR_rt_sigreturn (__NR_SYSCALL_BASE + 173) -#define __NR_rt_sigaction (__NR_SYSCALL_BASE + 174) -#define __NR_rt_sigprocmask (__NR_SYSCALL_BASE + 175) -#define __NR_rt_sigpending (__NR_SYSCALL_BASE + 176) -#define __NR_rt_sigsuspend (__NR_SYSCALL_BASE + 179) -#endif -#ifndef __NR_pread64 -#define __NR_pread64 (__NR_SYSCALL_BASE + 180) -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 (__NR_SYSCALL_BASE + 181) -#endif -#ifndef __NR_ugetrlimit -#define __NR_ugetrlimit (__NR_SYSCALL_BASE + 191) -#endif -#ifndef __NR_stat64 -#define __NR_stat64 (__NR_SYSCALL_BASE + 195) -#endif -#ifndef __NR_fstat64 -#define __NR_fstat64 (__NR_SYSCALL_BASE + 197) -#endif -#ifndef __NR_setresuid32 -#define __NR_setresuid32 (__NR_SYSCALL_BASE + 208) -#define __NR_getresuid32 (__NR_SYSCALL_BASE + 209) -#define __NR_setresgid32 (__NR_SYSCALL_BASE + 210) -#define __NR_getresgid32 (__NR_SYSCALL_BASE + 211) -#endif -#ifndef __NR_setfsuid32 -#define __NR_setfsuid32 (__NR_SYSCALL_BASE + 215) -#define __NR_setfsgid32 (__NR_SYSCALL_BASE + 216) -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 (__NR_SYSCALL_BASE + 217) -#endif -#ifndef __NR_gettid -#define __NR_gettid (__NR_SYSCALL_BASE + 224) -#endif -#ifndef __NR_readahead -#define __NR_readahead (__NR_SYSCALL_BASE + 225) -#endif -#ifndef __NR_setxattr -#define __NR_setxattr (__NR_SYSCALL_BASE + 226) -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr (__NR_SYSCALL_BASE + 227) -#endif -#ifndef __NR_getxattr -#define __NR_getxattr (__NR_SYSCALL_BASE + 229) -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr (__NR_SYSCALL_BASE + 230) -#endif -#ifndef __NR_listxattr -#define __NR_listxattr (__NR_SYSCALL_BASE + 232) -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr (__NR_SYSCALL_BASE + 233) -#endif -#ifndef __NR_tkill -#define __NR_tkill (__NR_SYSCALL_BASE + 238) -#endif -#ifndef __NR_futex -#define __NR_futex (__NR_SYSCALL_BASE + 240) -#endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity (__NR_SYSCALL_BASE + 241) -#define __NR_sched_getaffinity (__NR_SYSCALL_BASE + 242) -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address (__NR_SYSCALL_BASE + 256) -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime (__NR_SYSCALL_BASE + 263) -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres (__NR_SYSCALL_BASE + 264) -#endif -#ifndef __NR_statfs64 -#define __NR_statfs64 (__NR_SYSCALL_BASE + 266) -#endif -#ifndef __NR_fstatfs64 -#define __NR_fstatfs64 (__NR_SYSCALL_BASE + 267) -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set (__NR_SYSCALL_BASE + 314) -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get (__NR_SYSCALL_BASE + 315) -#endif -#ifndef __NR_fstatat64 -#define __NR_fstatat64 (__NR_SYSCALL_BASE + 327) -#endif -#ifndef __NR_move_pages -#define __NR_move_pages (__NR_SYSCALL_BASE + 344) -#endif -#ifndef __NR_getcpu -#define __NR_getcpu (__NR_SYSCALL_BASE + 345) -#endif -#ifndef __NR_getrandom -#define __NR_getrandom (__NR_SYSCALL_BASE + 384) -#endif -/* End of ARM 3/EABI definitions */ -#elif defined(__aarch64__) || defined(__riscv) || defined(__loongarch_lp64) -#ifndef __NR_setxattr -#define __NR_setxattr 5 -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr 6 -#endif -#ifndef __NR_getxattr -#define __NR_getxattr 8 -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr 9 -#endif -#ifndef __NR_listxattr -#define __NR_listxattr 11 -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr 12 -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set 30 -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get 31 -#endif -#ifndef __NR_unlinkat -#define __NR_unlinkat 35 -#endif -#ifndef __NR_fallocate -#define __NR_fallocate 47 -#endif -#ifndef __NR_openat -#define __NR_openat 56 -#endif -#ifndef __NR_quotactl -#define __NR_quotactl 60 -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 61 -#endif -#ifndef __NR_getdents -// when getdents is not available, getdents64 is used for both. -#define __NR_getdents __NR_getdents64 -#endif -#ifndef __NR_pread64 -#define __NR_pread64 67 -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 68 -#endif -#ifndef __NR_ppoll -#define __NR_ppoll 73 -#endif -#ifndef __NR_readlinkat -#define __NR_readlinkat 78 -#endif -#if !defined(__loongarch_lp64) -#ifndef __NR_newfstatat -#define __NR_newfstatat 79 -#endif -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address 96 -#endif -#ifndef __NR_futex -#define __NR_futex 98 -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime 113 -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres 114 -#endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity 122 -#define __NR_sched_getaffinity 123 -#endif -#ifndef __NR_tkill -#define __NR_tkill 130 -#endif -#ifndef __NR_setresuid -#define __NR_setresuid 147 -#define __NR_getresuid 148 -#define __NR_setresgid 149 -#define __NR_getresgid 150 -#endif -#ifndef __NR_gettid -#define __NR_gettid 178 -#endif -#ifndef __NR_readahead -#define __NR_readahead 213 -#endif -#ifndef __NR_fadvise64 -#define __NR_fadvise64 223 -#endif -#ifndef __NR_move_pages -#define __NR_move_pages 239 -#endif -#ifndef __NR_getrandom -#define __NR_getrandom 278 -#endif -#ifndef __NR_statx -#define __NR_statx 291 -#endif -#elif defined(__x86_64__) -#ifndef __NR_pread64 -#define __NR_pread64 17 -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 18 -#endif -#ifndef __NR_setresuid -#define __NR_setresuid 117 -#define __NR_getresuid 118 -#define __NR_setresgid 119 -#define __NR_getresgid 120 -#endif -#ifndef __NR_quotactl -#define __NR_quotactl 179 -#endif -#ifndef __NR_gettid -#define __NR_gettid 186 -#endif -#ifndef __NR_readahead -#define __NR_readahead 187 -#endif -#ifndef __NR_setxattr -#define __NR_setxattr 188 -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr 189 -#endif -#ifndef __NR_getxattr -#define __NR_getxattr 191 -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr 192 -#endif -#ifndef __NR_listxattr -#define __NR_listxattr 194 -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr 195 -#endif -#ifndef __NR_tkill -#define __NR_tkill 200 -#endif -#ifndef __NR_futex -#define __NR_futex 202 -#endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity 203 -#define __NR_sched_getaffinity 204 -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 217 -#endif -#ifndef __NR_getdents -// when getdents is not available, getdents64 is used for both. -#define __NR_getdents __NR_getdents64 -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address 218 -#endif -#ifndef __NR_fadvise64 -#define __NR_fadvise64 221 -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime 228 -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres 229 -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set 251 -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get 252 -#endif -#ifndef __NR_openat -#define __NR_openat 257 -#endif -#ifndef __NR_newfstatat -#define __NR_newfstatat 262 -#endif -#ifndef __NR_unlinkat -#define __NR_unlinkat 263 -#endif -#ifndef __NR_move_pages -#define __NR_move_pages 279 -#endif -#ifndef __NR_fallocate -#define __NR_fallocate 285 -#endif -#ifndef __NR_getrandom -#define __NR_getrandom 318 -#endif -/* End of x86-64 definitions */ -#elif defined(__mips__) -#if _MIPS_SIM == _MIPS_SIM_ABI32 -#ifndef __NR_setresuid -#define __NR_setresuid (__NR_Linux + 185) -#define __NR_getresuid (__NR_Linux + 186) -#define __NR_setresgid (__NR_Linux + 190) -#define __NR_getresgid (__NR_Linux + 191) -#endif -#ifndef __NR_rt_sigaction -#define __NR_rt_sigreturn (__NR_Linux + 193) -#define __NR_rt_sigaction (__NR_Linux + 194) -#define __NR_rt_sigprocmask (__NR_Linux + 195) -#define __NR_rt_sigpending (__NR_Linux + 196) -#define __NR_rt_sigsuspend (__NR_Linux + 199) -#endif -#ifndef __NR_pread64 -#define __NR_pread64 (__NR_Linux + 200) -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 (__NR_Linux + 201) -#endif -#ifndef __NR_stat64 -#define __NR_stat64 (__NR_Linux + 213) -#endif -#ifndef __NR_fstat64 -#define __NR_fstat64 (__NR_Linux + 215) -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 (__NR_Linux + 219) -#endif -#ifndef __NR_gettid -#define __NR_gettid (__NR_Linux + 222) -#endif -#ifndef __NR_readahead -#define __NR_readahead (__NR_Linux + 223) -#endif -#ifndef __NR_setxattr -#define __NR_setxattr (__NR_Linux + 224) -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr (__NR_Linux + 225) -#endif -#ifndef __NR_getxattr -#define __NR_getxattr (__NR_Linux + 227) -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr (__NR_Linux + 228) -#endif -#ifndef __NR_listxattr -#define __NR_listxattr (__NR_Linux + 230) -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr (__NR_Linux + 231) -#endif -#ifndef __NR_tkill -#define __NR_tkill (__NR_Linux + 236) -#endif -#ifndef __NR_futex -#define __NR_futex (__NR_Linux + 238) -#endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity (__NR_Linux + 239) -#define __NR_sched_getaffinity (__NR_Linux + 240) -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address (__NR_Linux + 252) -#endif -#ifndef __NR_statfs64 -#define __NR_statfs64 (__NR_Linux + 255) -#endif -#ifndef __NR_fstatfs64 -#define __NR_fstatfs64 (__NR_Linux + 256) -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime (__NR_Linux + 263) -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres (__NR_Linux + 264) -#endif -#ifndef __NR_openat -#define __NR_openat (__NR_Linux + 288) -#endif -#ifndef __NR_fstatat -#define __NR_fstatat (__NR_Linux + 293) -#endif -#ifndef __NR_unlinkat -#define __NR_unlinkat (__NR_Linux + 294) -#endif -#ifndef __NR_move_pages -#define __NR_move_pages (__NR_Linux + 308) -#endif -#ifndef __NR_getcpu -#define __NR_getcpu (__NR_Linux + 312) -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set (__NR_Linux + 314) -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get (__NR_Linux + 315) -#endif -#ifndef __NR_getrandom -#define __NR_getrandom (__NR_Linux + 353) -#endif -/* End of MIPS (old 32bit API) definitions */ -#elif _MIPS_SIM == _MIPS_SIM_ABI64 -#ifndef __NR_pread64 -#define __NR_pread64 (__NR_Linux + 16) -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 (__NR_Linux + 17) -#endif -#ifndef __NR_setresuid -#define __NR_setresuid (__NR_Linux + 115) -#define __NR_getresuid (__NR_Linux + 116) -#define __NR_setresgid (__NR_Linux + 117) -#define __NR_getresgid (__NR_Linux + 118) -#endif -#ifndef __NR_gettid -#define __NR_gettid (__NR_Linux + 178) -#endif -#ifndef __NR_readahead -#define __NR_readahead (__NR_Linux + 179) -#endif -#ifndef __NR_setxattr -#define __NR_setxattr (__NR_Linux + 180) -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr (__NR_Linux + 181) -#endif -#ifndef __NR_getxattr -#define __NR_getxattr (__NR_Linux + 183) -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr (__NR_Linux + 184) -#endif -#ifndef __NR_listxattr -#define __NR_listxattr (__NR_Linux + 186) -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr (__NR_Linux + 187) -#endif -#ifndef __NR_tkill -#define __NR_tkill (__NR_Linux + 192) -#endif -#ifndef __NR_futex -#define __NR_futex (__NR_Linux + 194) -#endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity (__NR_Linux + 195) -#define __NR_sched_getaffinity (__NR_Linux + 196) -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address (__NR_Linux + 212) -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime (__NR_Linux + 222) -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres (__NR_Linux + 223) -#endif -#ifndef __NR_openat -#define __NR_openat (__NR_Linux + 247) -#endif -#ifndef __NR_fstatat -#define __NR_fstatat (__NR_Linux + 252) -#endif -#ifndef __NR_unlinkat -#define __NR_unlinkat (__NR_Linux + 253) -#endif -#ifndef __NR_move_pages -#define __NR_move_pages (__NR_Linux + 267) -#endif -#ifndef __NR_getcpu -#define __NR_getcpu (__NR_Linux + 271) -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set (__NR_Linux + 273) -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get (__NR_Linux + 274) -#endif -#ifndef __NR_getrandom -#define __NR_getrandom (__NR_Linux + 313) -#endif -/* End of MIPS (64bit API) definitions */ -#else -#ifndef __NR_setresuid -#define __NR_setresuid (__NR_Linux + 115) -#define __NR_getresuid (__NR_Linux + 116) -#define __NR_setresgid (__NR_Linux + 117) -#define __NR_getresgid (__NR_Linux + 118) -#endif -#ifndef __NR_gettid -#define __NR_gettid (__NR_Linux + 178) -#endif -#ifndef __NR_readahead -#define __NR_readahead (__NR_Linux + 179) -#endif -#ifndef __NR_setxattr -#define __NR_setxattr (__NR_Linux + 180) -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr (__NR_Linux + 181) -#endif -#ifndef __NR_getxattr -#define __NR_getxattr (__NR_Linux + 183) -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr (__NR_Linux + 184) -#endif -#ifndef __NR_listxattr -#define __NR_listxattr (__NR_Linux + 186) -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr (__NR_Linux + 187) -#endif -#ifndef __NR_tkill -#define __NR_tkill (__NR_Linux + 192) -#endif -#ifndef __NR_futex -#define __NR_futex (__NR_Linux + 194) -#endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity (__NR_Linux + 195) -#define __NR_sched_getaffinity (__NR_Linux + 196) -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address (__NR_Linux + 213) -#endif -#ifndef __NR_statfs64 -#define __NR_statfs64 (__NR_Linux + 217) -#endif -#ifndef __NR_fstatfs64 -#define __NR_fstatfs64 (__NR_Linux + 218) -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime (__NR_Linux + 226) -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres (__NR_Linux + 227) -#endif -#ifndef __NR_openat -#define __NR_openat (__NR_Linux + 251) -#endif -#ifndef __NR_fstatat -#define __NR_fstatat (__NR_Linux + 256) -#endif -#ifndef __NR_unlinkat -#define __NR_unlinkat (__NR_Linux + 257) -#endif -#ifndef __NR_move_pages -#define __NR_move_pages (__NR_Linux + 271) -#endif -#ifndef __NR_getcpu -#define __NR_getcpu (__NR_Linux + 275) -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set (__NR_Linux + 277) -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get (__NR_Linux + 278) -#endif -/* End of MIPS (new 32bit API) definitions */ -#endif -/* End of MIPS definitions */ -#elif defined(__PPC__) -#ifndef __NR_setfsuid -#define __NR_setfsuid 138 -#define __NR_setfsgid 139 -#endif -#ifndef __NR_setresuid -#define __NR_setresuid 164 -#define __NR_getresuid 165 -#define __NR_setresgid 169 -#define __NR_getresgid 170 -#endif -#ifndef __NR_rt_sigaction -#define __NR_rt_sigreturn 172 -#define __NR_rt_sigaction 173 -#define __NR_rt_sigprocmask 174 -#define __NR_rt_sigpending 175 -#define __NR_rt_sigsuspend 178 -#endif -#ifndef __NR_pread64 -#define __NR_pread64 179 -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 180 -#endif -#ifndef __NR_ugetrlimit -#define __NR_ugetrlimit 190 -#endif -#ifndef __NR_readahead -#define __NR_readahead 191 -#endif -#ifndef __NR_stat64 -#define __NR_stat64 195 -#endif -#ifndef __NR_fstat64 -#define __NR_fstat64 197 -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 202 -#endif -#ifndef __NR_gettid -#define __NR_gettid 207 -#endif -#ifndef __NR_tkill -#define __NR_tkill 208 -#endif -#ifndef __NR_setxattr -#define __NR_setxattr 209 -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr 210 -#endif -#ifndef __NR_getxattr -#define __NR_getxattr 212 -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr 213 -#endif -#ifndef __NR_listxattr -#define __NR_listxattr 215 -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr 216 -#endif -#ifndef __NR_futex -#define __NR_futex 221 -#endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity 222 -#define __NR_sched_getaffinity 223 -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address 232 -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime 246 -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres 247 -#endif -#ifndef __NR_statfs64 -#define __NR_statfs64 252 -#endif -#ifndef __NR_fstatfs64 -#define __NR_fstatfs64 253 -#endif -#ifndef __NR_fadvise64_64 -#define __NR_fadvise64_64 254 -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set 273 -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get 274 -#endif -#ifndef __NR_openat -#define __NR_openat 286 -#endif -#ifndef __NR_fstatat64 -#define __NR_fstatat64 291 -#endif -#ifndef __NR_unlinkat -#define __NR_unlinkat 292 -#endif -#ifndef __NR_move_pages -#define __NR_move_pages 301 -#endif -#ifndef __NR_getcpu -#define __NR_getcpu 302 -#endif -/* End of powerpc definitions */ -#elif defined(__s390__) -#ifndef __NR_quotactl -#define __NR_quotactl 131 -#endif -#ifndef __NR_rt_sigreturn -#define __NR_rt_sigreturn 173 -#endif -#ifndef __NR_rt_sigaction -#define __NR_rt_sigaction 174 -#endif -#ifndef __NR_rt_sigprocmask -#define __NR_rt_sigprocmask 175 -#endif -#ifndef __NR_rt_sigpending -#define __NR_rt_sigpending 176 -#endif -#ifndef __NR_rt_sigsuspend -#define __NR_rt_sigsuspend 179 -#endif -#ifndef __NR_pread64 -#define __NR_pread64 180 -#endif -#ifndef __NR_pwrite64 -#define __NR_pwrite64 181 -#endif -#ifndef __NR_getdents64 -#define __NR_getdents64 220 -#endif -#ifndef __NR_readahead -#define __NR_readahead 222 -#endif -#ifndef __NR_setxattr -#define __NR_setxattr 224 -#endif -#ifndef __NR_lsetxattr -#define __NR_lsetxattr 225 -#endif -#ifndef __NR_getxattr -#define __NR_getxattr 227 -#endif -#ifndef __NR_lgetxattr -#define __NR_lgetxattr 228 -#endif -#ifndef __NR_listxattr -#define __NR_listxattr 230 -#endif -#ifndef __NR_llistxattr -#define __NR_llistxattr 231 -#endif -#ifndef __NR_gettid -#define __NR_gettid 236 -#endif -#ifndef __NR_tkill -#define __NR_tkill 237 -#endif -#ifndef __NR_futex -#define __NR_futex 238 -#endif -#ifndef __NR_sched_setaffinity -#define __NR_sched_setaffinity 239 -#endif -#ifndef __NR_sched_getaffinity -#define __NR_sched_getaffinity 240 -#endif -#ifndef __NR_set_tid_address -#define __NR_set_tid_address 252 -#endif -#ifndef __NR_clock_gettime -#define __NR_clock_gettime 260 -#endif -#ifndef __NR_clock_getres -#define __NR_clock_getres 261 -#endif -#ifndef __NR_statfs64 -#define __NR_statfs64 265 -#endif -#ifndef __NR_fstatfs64 -#define __NR_fstatfs64 266 -#endif -#ifndef __NR_ioprio_set -#define __NR_ioprio_set 282 -#endif -#ifndef __NR_ioprio_get -#define __NR_ioprio_get 283 -#endif -#ifndef __NR_openat -#define __NR_openat 288 -#endif -#ifndef __NR_unlinkat -#define __NR_unlinkat 294 -#endif -#ifndef __NR_move_pages -#define __NR_move_pages 310 -#endif -#ifndef __NR_getcpu -#define __NR_getcpu 311 -#endif -#ifndef __NR_fallocate -#define __NR_fallocate 314 -#endif -/* Some syscalls are named/numbered differently between s390 and s390x. */ -#ifdef __s390x__ -# ifndef __NR_getrlimit -# define __NR_getrlimit 191 -# endif -# ifndef __NR_setresuid -# define __NR_setresuid 208 -# endif -# ifndef __NR_getresuid -# define __NR_getresuid 209 -# endif -# ifndef __NR_setresgid -# define __NR_setresgid 210 -# endif -# ifndef __NR_getresgid -# define __NR_getresgid 211 -# endif -# ifndef __NR_setfsuid -# define __NR_setfsuid 215 -# endif -# ifndef __NR_setfsgid -# define __NR_setfsgid 216 -# endif -# ifndef __NR_fadvise64 -# define __NR_fadvise64 253 -# endif -# ifndef __NR_newfstatat -# define __NR_newfstatat 293 -# endif -#else /* __s390x__ */ -# ifndef __NR_getrlimit -# define __NR_getrlimit 76 -# endif -# ifndef __NR_setfsuid -# define __NR_setfsuid 138 -# endif -# ifndef __NR_setfsgid -# define __NR_setfsgid 139 -# endif -# ifndef __NR_setresuid -# define __NR_setresuid 164 -# endif -# ifndef __NR_getresuid -# define __NR_getresuid 165 -# endif -# ifndef __NR_setresgid -# define __NR_setresgid 170 -# endif -# ifndef __NR_getresgid -# define __NR_getresgid 171 -# endif -# ifndef __NR_ugetrlimit -# define __NR_ugetrlimit 191 -# endif -# ifndef __NR_mmap2 -# define __NR_mmap2 192 -# endif -# ifndef __NR_setresuid32 -# define __NR_setresuid32 208 -# endif -# ifndef __NR_getresuid32 -# define __NR_getresuid32 209 -# endif -# ifndef __NR_setresgid32 -# define __NR_setresgid32 210 -# endif -# ifndef __NR_getresgid32 -# define __NR_getresgid32 211 -# endif -# ifndef __NR_setfsuid32 -# define __NR_setfsuid32 215 -# endif -# ifndef __NR_setfsgid32 -# define __NR_setfsgid32 216 -# endif -# ifndef __NR_fadvise64_64 -# define __NR_fadvise64_64 264 -# endif -# ifndef __NR_fstatat64 -# define __NR_fstatat64 293 -# endif -#endif /* __s390__ */ -/* End of s390/s390x definitions */ -#endif - - -/* After forking, we must make sure to only call system calls. */ -#if defined(__BOUNDED_POINTERS__) - #error "Need to port invocations of syscalls for bounded ptrs" -#else - /* The core dumper and the thread lister get executed after threads - * have been suspended. As a consequence, we cannot call any functions - * that acquire locks. Unfortunately, libc wraps most system calls - * (e.g. in order to implement pthread_atfork, and to make calls - * cancellable), which means we cannot call these functions. Instead, - * we have to call syscall() directly. - */ - #undef LSS_ERRNO - #ifdef SYS_ERRNO - /* Allow the including file to override the location of errno. This can - * be useful when using clone() with the CLONE_VM option. - */ - #define LSS_ERRNO SYS_ERRNO - #else - #define LSS_ERRNO errno - #endif - - #undef LSS_INLINE - #ifdef SYS_INLINE - #define LSS_INLINE SYS_INLINE - #else - #define LSS_INLINE static inline - #endif - - /* Allow the including file to override the prefix used for all new - * system calls. By default, it will be set to "sys_". - */ - #undef LSS_NAME - #ifndef SYS_PREFIX - #define LSS_NAME(name) sys_##name - #elif defined(SYS_PREFIX) && SYS_PREFIX < 0 - #define LSS_NAME(name) name - #elif defined(SYS_PREFIX) && SYS_PREFIX == 0 - #define LSS_NAME(name) sys0_##name - #elif defined(SYS_PREFIX) && SYS_PREFIX == 1 - #define LSS_NAME(name) sys1_##name - #elif defined(SYS_PREFIX) && SYS_PREFIX == 2 - #define LSS_NAME(name) sys2_##name - #elif defined(SYS_PREFIX) && SYS_PREFIX == 3 - #define LSS_NAME(name) sys3_##name - #elif defined(SYS_PREFIX) && SYS_PREFIX == 4 - #define LSS_NAME(name) sys4_##name - #elif defined(SYS_PREFIX) && SYS_PREFIX == 5 - #define LSS_NAME(name) sys5_##name - #elif defined(SYS_PREFIX) && SYS_PREFIX == 6 - #define LSS_NAME(name) sys6_##name - #elif defined(SYS_PREFIX) && SYS_PREFIX == 7 - #define LSS_NAME(name) sys7_##name - #elif defined(SYS_PREFIX) && SYS_PREFIX == 8 - #define LSS_NAME(name) sys8_##name - #elif defined(SYS_PREFIX) && SYS_PREFIX == 9 - #define LSS_NAME(name) sys9_##name - #endif - - #undef LSS_RETURN - #if defined(__i386__) || defined(__x86_64__) || defined(__ARM_ARCH_3__) \ - || defined(__ARM_EABI__) || defined(__aarch64__) || defined(__s390__) \ - || defined(__e2k__) || defined(__riscv) || defined(__loongarch_lp64) - /* Failing system calls return a negative result in the range of - * -1..-4095. These are "errno" values with the sign inverted. - */ - #define LSS_RETURN(type, res) \ - do { \ - if ((unsigned long)(res) >= (unsigned long)(-4095)) { \ - LSS_ERRNO = (int)(-(res)); \ - res = -1; \ - } \ - return (type) (res); \ - } while (0) - #elif defined(__mips__) - /* On MIPS, failing system calls return -1, and set errno in a - * separate CPU register. - */ - #define LSS_RETURN(type, res, err) \ - do { \ - if (err) { \ - unsigned long __errnovalue = (res); \ - LSS_ERRNO = __errnovalue; \ - res = -1; \ - } \ - return (type) (res); \ - } while (0) - #elif defined(__PPC__) - /* On PPC, failing system calls return -1, and set errno in a - * separate CPU register. See linux/unistd.h. - */ - #define LSS_RETURN(type, res, err) \ - do { \ - if (err & 0x10000000 ) { \ - LSS_ERRNO = (res); \ - res = -1; \ - } \ - return (type) (res); \ - } while (0) - #endif - #if defined(__i386__) - /* In PIC mode (e.g. when building shared libraries), gcc for i386 - * reserves ebx. Unfortunately, most distribution ship with implementations - * of _syscallX() which clobber ebx. - * Also, most definitions of _syscallX() neglect to mark "memory" as being - * clobbered. This causes problems with compilers, that do a better job - * at optimizing across __asm__ calls. - * So, we just have to redefine all of the _syscallX() macros. - */ - #undef LSS_ENTRYPOINT - #ifdef SYS_SYSCALL_ENTRYPOINT - static inline void (**LSS_NAME(get_syscall_entrypoint)(void))(void) { - void (**entrypoint)(void); - asm volatile(".bss\n" - ".align 8\n" - ".globl " SYS_SYSCALL_ENTRYPOINT "\n" - ".common " SYS_SYSCALL_ENTRYPOINT ",8,8\n" - ".previous\n" - /* This logically does 'lea "SYS_SYSCALL_ENTRYPOINT", %0' */ - "call 0f\n" - "0:pop %0\n" - "add $_GLOBAL_OFFSET_TABLE_+[.-0b], %0\n" - "mov " SYS_SYSCALL_ENTRYPOINT "@GOT(%0), %0\n" - : "=r"(entrypoint)); - return entrypoint; - } - - #define LSS_ENTRYPOINT ".bss\n" \ - ".align 8\n" \ - ".globl " SYS_SYSCALL_ENTRYPOINT "\n" \ - ".common " SYS_SYSCALL_ENTRYPOINT ",8,8\n" \ - ".previous\n" \ - /* Check the SYS_SYSCALL_ENTRYPOINT vector */ \ - "push %%eax\n" \ - "call 10000f\n" \ - "10000:pop %%eax\n" \ - "add $_GLOBAL_OFFSET_TABLE_+[.-10000b], %%eax\n" \ - "mov " SYS_SYSCALL_ENTRYPOINT \ - "@GOT(%%eax), %%eax\n" \ - "mov 0(%%eax), %%eax\n" \ - "test %%eax, %%eax\n" \ - "jz 10002f\n" \ - "push %%eax\n" \ - "call 10001f\n" \ - "10001:pop %%eax\n" \ - "add $(10003f-10001b), %%eax\n" \ - "xchg 4(%%esp), %%eax\n" \ - "ret\n" \ - "10002:pop %%eax\n" \ - "int $0x80\n" \ - "10003:\n" - #else - #define LSS_ENTRYPOINT "int $0x80\n" - #endif - #undef LSS_BODY - #define LSS_BODY(type,args...) \ - long __res; \ - __asm__ __volatile__("push %%ebx\n" \ - "movl %2,%%ebx\n" \ - LSS_ENTRYPOINT \ - "pop %%ebx" \ - args \ - : "memory"); \ - LSS_RETURN(type,__res) - #undef _syscall0 - #define _syscall0(type,name) \ - type LSS_NAME(name)(void) { \ - long __res; \ - __asm__ volatile(LSS_ENTRYPOINT \ - : "=a" (__res) \ - : "0" (__NR_##name) \ - : "memory"); \ - LSS_RETURN(type,__res); \ - } - #undef _syscall1 - #define _syscall1(type,name,type1,arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_BODY(type, \ - : "=a" (__res) \ - : "0" (__NR_##name), "ri" ((long)(arg1))); \ - } - #undef _syscall2 - #define _syscall2(type,name,type1,arg1,type2,arg2) \ - type LSS_NAME(name)(type1 arg1,type2 arg2) { \ - LSS_BODY(type, \ - : "=a" (__res) \ - : "0" (__NR_##name),"ri" ((long)(arg1)), "c" ((long)(arg2))); \ - } - #undef _syscall3 - #define _syscall3(type,name,type1,arg1,type2,arg2,type3,arg3) \ - type LSS_NAME(name)(type1 arg1,type2 arg2,type3 arg3) { \ - LSS_BODY(type, \ - : "=a" (__res) \ - : "0" (__NR_##name), "ri" ((long)(arg1)), "c" ((long)(arg2)), \ - "d" ((long)(arg3))); \ - } - #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_BODY(type, \ - : "=a" (__res) \ - : "0" (__NR_##name), "ri" ((long)(arg1)), "c" ((long)(arg2)), \ - "d" ((long)(arg3)),"S" ((long)(arg4))); \ - } - #undef _syscall5 - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - long __res; \ - __asm__ __volatile__("push %%ebx\n" \ - "movl %2,%%ebx\n" \ - "movl %1,%%eax\n" \ - LSS_ENTRYPOINT \ - "pop %%ebx" \ - : "=a" (__res) \ - : "i" (__NR_##name), "ri" ((long)(arg1)), \ - "c" ((long)(arg2)), "d" ((long)(arg3)), \ - "S" ((long)(arg4)), "D" ((long)(arg5)) \ - : "memory"); \ - LSS_RETURN(type,__res); \ - } - #undef _syscall6 - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - long __res; \ - struct { long __a1; long __a6; } __s = { (long)arg1, (long) arg6 }; \ - __asm__ __volatile__("push %%ebp\n" \ - "push %%ebx\n" \ - "movl 4(%2),%%ebp\n" \ - "movl 0(%2), %%ebx\n" \ - "movl %1,%%eax\n" \ - LSS_ENTRYPOINT \ - "pop %%ebx\n" \ - "pop %%ebp" \ - : "=a" (__res) \ - : "i" (__NR_##name), "0" ((long)(&__s)), \ - "c" ((long)(arg2)), "d" ((long)(arg3)), \ - "S" ((long)(arg4)), "D" ((long)(arg5)) \ - : "memory"); \ - LSS_RETURN(type,__res); \ - } - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - long __res; - __asm__ __volatile__(/* if (fn == NULL) - * return -EINVAL; - */ - "movl %3,%%ecx\n" - "jecxz 1f\n" - - /* if (child_stack == NULL) - * return -EINVAL; - */ - "movl %4,%%ecx\n" - "jecxz 1f\n" - - /* Set up alignment of the child stack: - * child_stack = (child_stack & ~0xF) - 20; - */ - "andl $-16,%%ecx\n" - "subl $20,%%ecx\n" - - /* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - "movl %6,%%eax\n" - "movl %%eax,4(%%ecx)\n" - "movl %3,%%eax\n" - "movl %%eax,(%%ecx)\n" - - /* %eax = syscall(%eax = __NR_clone, - * %ebx = flags, - * %ecx = child_stack, - * %edx = parent_tidptr, - * %esi = newtls, - * %edi = child_tidptr) - * Also, make sure that %ebx gets preserved as it is - * used in PIC mode. - */ - "movl %8,%%esi\n" - "movl %7,%%edx\n" - "movl %5,%%eax\n" - "movl %9,%%edi\n" - "pushl %%ebx\n" - "movl %%eax,%%ebx\n" - "movl %2,%%eax\n" - LSS_ENTRYPOINT - - /* In the parent: restore %ebx - * In the child: move "fn" into %ebx - */ - "popl %%ebx\n" - - /* if (%eax != 0) - * return %eax; - */ - "test %%eax,%%eax\n" - "jnz 1f\n" - - /* In the child, now. Terminate frame pointer chain. - */ - "movl $0,%%ebp\n" - - /* Call "fn". "arg" is already on the stack. - */ - "call *%%ebx\n" - - /* Call _exit(%ebx). Unfortunately older versions - * of gcc restrict the number of arguments that can - * be passed to asm(). So, we need to hard-code the - * system call number. - */ - "movl %%eax,%%ebx\n" - "movl $1,%%eax\n" - LSS_ENTRYPOINT - - /* Return to parent. - */ - "1:\n" - : "=a" (__res) - : "0"(-EINVAL), "i"(__NR_clone), - "m"(fn), "m"(child_stack), "m"(flags), "m"(arg), - "m"(parent_tidptr), "m"(newtls), "m"(child_tidptr) - : "memory", "ecx", "edx", "esi", "edi"); - LSS_RETURN(int, __res); - } - - LSS_INLINE _syscall1(int, set_thread_area, void *, u) - LSS_INLINE _syscall1(int, get_thread_area, void *, u) - - LSS_INLINE void (*LSS_NAME(restore_rt)(void))(void) { - /* On i386, the kernel does not know how to return from a signal - * handler. Instead, it relies on user space to provide a - * restorer function that calls the {rt_,}sigreturn() system call. - * Unfortunately, we cannot just reference the glibc version of this - * function, as glibc goes out of its way to make it inaccessible. - */ - void (*res)(void); - __asm__ __volatile__("call 2f\n" - "0:.align 16\n" - "1:movl %1,%%eax\n" - LSS_ENTRYPOINT - "2:popl %0\n" - "addl $(1b-0b),%0\n" - : "=a" (res) - : "i" (__NR_rt_sigreturn)); - return res; - } - LSS_INLINE void (*LSS_NAME(restore)(void))(void) { - /* On i386, the kernel does not know how to return from a signal - * handler. Instead, it relies on user space to provide a - * restorer function that calls the {rt_,}sigreturn() system call. - * Unfortunately, we cannot just reference the glibc version of this - * function, as glibc goes out of its way to make it inaccessible. - */ - void (*res)(void); - __asm__ __volatile__("call 2f\n" - "0:.align 16\n" - "1:pop %%eax\n" - "movl %1,%%eax\n" - LSS_ENTRYPOINT - "2:popl %0\n" - "addl $(1b-0b),%0\n" - : "=a" (res) - : "i" (__NR_sigreturn)); - return res; - } - #elif defined(__x86_64__) - /* There are no known problems with any of the _syscallX() macros - * currently shipping for x86_64, but we still need to be able to define - * our own version so that we can override the location of the errno - * location (e.g. when using the clone() system call with the CLONE_VM - * option). - */ - #undef LSS_ENTRYPOINT - #ifdef SYS_SYSCALL_ENTRYPOINT - static inline void (**LSS_NAME(get_syscall_entrypoint)(void))(void) { - void (**entrypoint)(void); - asm volatile(".bss\n" - ".align 8\n" - ".globl " SYS_SYSCALL_ENTRYPOINT "\n" - ".common " SYS_SYSCALL_ENTRYPOINT ",8,8\n" - ".previous\n" - "mov " SYS_SYSCALL_ENTRYPOINT "@GOTPCREL(%%rip), %0\n" - : "=r"(entrypoint)); - return entrypoint; - } - - #define LSS_ENTRYPOINT \ - ".bss\n" \ - ".align 8\n" \ - ".globl " SYS_SYSCALL_ENTRYPOINT "\n" \ - ".common " SYS_SYSCALL_ENTRYPOINT ",8,8\n" \ - ".previous\n" \ - "mov " SYS_SYSCALL_ENTRYPOINT "@GOTPCREL(%%rip), %%rcx\n" \ - "mov 0(%%rcx), %%rcx\n" \ - "test %%rcx, %%rcx\n" \ - "jz 10001f\n" \ - "call *%%rcx\n" \ - "jmp 10002f\n" \ - "10001:syscall\n" \ - "10002:\n" - - #else - #define LSS_ENTRYPOINT "syscall\n" - #endif - - /* The x32 ABI has 32 bit longs, but the syscall interface is 64 bit. - * We need to explicitly cast to an unsigned 64 bit type to avoid implicit - * sign extension. We can't cast pointers directly because those are - * 32 bits, and gcc will dump ugly warnings about casting from a pointer - * to an integer of a different size. - */ - #undef LSS_SYSCALL_ARG - #define LSS_SYSCALL_ARG(a) ((uint64_t)(uintptr_t)(a)) - #undef _LSS_RETURN - #define _LSS_RETURN(type, res, cast) \ - do { \ - if ((uint64_t)(res) >= (uint64_t)(-4095)) { \ - LSS_ERRNO = (int)(-(res)); \ - res = -1; \ - } \ - return (type)(cast)(res); \ - } while (0) - #undef LSS_RETURN - #define LSS_RETURN(type, res) _LSS_RETURN(type, res, uintptr_t) - - #undef _LSS_BODY - #define _LSS_BODY(nr, type, name, cast, ...) \ - long long __res; \ - __asm__ __volatile__(LSS_BODY_ASM##nr LSS_ENTRYPOINT \ - : "=a" (__res) \ - : "0" (__NR_##name) LSS_BODY_ARG##nr(__VA_ARGS__) \ - : LSS_BODY_CLOBBER##nr "r11", "rcx", "memory"); \ - _LSS_RETURN(type, __res, cast) - #undef LSS_BODY - #define LSS_BODY(nr, type, name, args...) \ - _LSS_BODY(nr, type, name, uintptr_t, ## args) - - #undef LSS_BODY_ASM0 - #undef LSS_BODY_ASM1 - #undef LSS_BODY_ASM2 - #undef LSS_BODY_ASM3 - #undef LSS_BODY_ASM4 - #undef LSS_BODY_ASM5 - #undef LSS_BODY_ASM6 - #define LSS_BODY_ASM0 - #define LSS_BODY_ASM1 LSS_BODY_ASM0 - #define LSS_BODY_ASM2 LSS_BODY_ASM1 - #define LSS_BODY_ASM3 LSS_BODY_ASM2 - #define LSS_BODY_ASM4 LSS_BODY_ASM3 "movq %5,%%r10;" - #define LSS_BODY_ASM5 LSS_BODY_ASM4 "movq %6,%%r8;" - #define LSS_BODY_ASM6 LSS_BODY_ASM5 "movq %7,%%r9;" - - #undef LSS_BODY_CLOBBER0 - #undef LSS_BODY_CLOBBER1 - #undef LSS_BODY_CLOBBER2 - #undef LSS_BODY_CLOBBER3 - #undef LSS_BODY_CLOBBER4 - #undef LSS_BODY_CLOBBER5 - #undef LSS_BODY_CLOBBER6 - #define LSS_BODY_CLOBBER0 - #define LSS_BODY_CLOBBER1 LSS_BODY_CLOBBER0 - #define LSS_BODY_CLOBBER2 LSS_BODY_CLOBBER1 - #define LSS_BODY_CLOBBER3 LSS_BODY_CLOBBER2 - #define LSS_BODY_CLOBBER4 LSS_BODY_CLOBBER3 "r10", - #define LSS_BODY_CLOBBER5 LSS_BODY_CLOBBER4 "r8", - #define LSS_BODY_CLOBBER6 LSS_BODY_CLOBBER5 "r9", - - #undef LSS_BODY_ARG0 - #undef LSS_BODY_ARG1 - #undef LSS_BODY_ARG2 - #undef LSS_BODY_ARG3 - #undef LSS_BODY_ARG4 - #undef LSS_BODY_ARG5 - #undef LSS_BODY_ARG6 - #define LSS_BODY_ARG0() - #define LSS_BODY_ARG1(arg1) \ - LSS_BODY_ARG0(), "D" (arg1) - #define LSS_BODY_ARG2(arg1, arg2) \ - LSS_BODY_ARG1(arg1), "S" (arg2) - #define LSS_BODY_ARG3(arg1, arg2, arg3) \ - LSS_BODY_ARG2(arg1, arg2), "d" (arg3) - #define LSS_BODY_ARG4(arg1, arg2, arg3, arg4) \ - LSS_BODY_ARG3(arg1, arg2, arg3), "r" (arg4) - #define LSS_BODY_ARG5(arg1, arg2, arg3, arg4, arg5) \ - LSS_BODY_ARG4(arg1, arg2, arg3, arg4), "r" (arg5) - #define LSS_BODY_ARG6(arg1, arg2, arg3, arg4, arg5, arg6) \ - LSS_BODY_ARG5(arg1, arg2, arg3, arg4, arg5), "r" (arg6) - - #undef _syscall0 - #define _syscall0(type,name) \ - type LSS_NAME(name)(void) { \ - LSS_BODY(0, type, name); \ - } - #undef _syscall1 - #define _syscall1(type,name,type1,arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_BODY(1, type, name, LSS_SYSCALL_ARG(arg1)); \ - } - #undef _syscall2 - #define _syscall2(type,name,type1,arg1,type2,arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_BODY(2, type, name, LSS_SYSCALL_ARG(arg1), LSS_SYSCALL_ARG(arg2));\ - } - #undef _syscall3 - #define _syscall3(type,name,type1,arg1,type2,arg2,type3,arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_BODY(3, type, name, LSS_SYSCALL_ARG(arg1), LSS_SYSCALL_ARG(arg2), \ - LSS_SYSCALL_ARG(arg3)); \ - } - #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_BODY(4, type, name, LSS_SYSCALL_ARG(arg1), LSS_SYSCALL_ARG(arg2), \ - LSS_SYSCALL_ARG(arg3), LSS_SYSCALL_ARG(arg4));\ - } - #undef _syscall5 - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_BODY(5, type, name, LSS_SYSCALL_ARG(arg1), LSS_SYSCALL_ARG(arg2), \ - LSS_SYSCALL_ARG(arg3), LSS_SYSCALL_ARG(arg4), \ - LSS_SYSCALL_ARG(arg5)); \ - } - #undef _syscall6 - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_BODY(6, type, name, LSS_SYSCALL_ARG(arg1), LSS_SYSCALL_ARG(arg2), \ - LSS_SYSCALL_ARG(arg3), LSS_SYSCALL_ARG(arg4), \ - LSS_SYSCALL_ARG(arg5), LSS_SYSCALL_ARG(arg6));\ - } - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - long long __res; - { - __asm__ __volatile__(/* if (fn == NULL) - * return -EINVAL; - */ - "testq %4,%4\n" - "jz 1f\n" - - /* if (child_stack == NULL) - * return -EINVAL; - */ - "testq %5,%5\n" - "jz 1f\n" - - /* childstack -= 2*sizeof(void *); - */ - "subq $16,%5\n" - - /* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - "movq %7,8(%5)\n" - "movq %4,0(%5)\n" - - /* %rax = syscall(%rax = __NR_clone, - * %rdi = flags, - * %rsi = child_stack, - * %rdx = parent_tidptr, - * %r8 = new_tls, - * %r10 = child_tidptr) - */ - "movq %2,%%rax\n" - "movq %9,%%r8\n" - "movq %10,%%r10\n" - LSS_ENTRYPOINT - - /* if (%rax != 0) - * return; - */ - "testq %%rax,%%rax\n" - "jnz 1f\n" - - /* In the child. Terminate frame pointer chain. - */ - "xorq %%rbp,%%rbp\n" - - /* Call "fn(arg)". - */ - "popq %%rax\n" - "popq %%rdi\n" - "call *%%rax\n" - - /* Call _exit(%ebx). - */ - "movq %%rax,%%rdi\n" - "movq %3,%%rax\n" - LSS_ENTRYPOINT - - /* Return to parent. - */ - "1:\n" - : "=a" (__res) - : "0"(-EINVAL), "i"(__NR_clone), "i"(__NR_exit), - "r"(LSS_SYSCALL_ARG(fn)), - "S"(LSS_SYSCALL_ARG(child_stack)), - "D"(LSS_SYSCALL_ARG(flags)), - "r"(LSS_SYSCALL_ARG(arg)), - "d"(LSS_SYSCALL_ARG(parent_tidptr)), - "r"(LSS_SYSCALL_ARG(newtls)), - "r"(LSS_SYSCALL_ARG(child_tidptr)) - : "memory", "r8", "r10", "r11", "rcx"); - } - LSS_RETURN(int, __res); - } - LSS_INLINE _syscall2(int, arch_prctl, int, c, void *, a) - - LSS_INLINE void (*LSS_NAME(restore_rt)(void))(void) { - /* On x86-64, the kernel does not know how to return from - * a signal handler. Instead, it relies on user space to provide a - * restorer function that calls the rt_sigreturn() system call. - * Unfortunately, we cannot just reference the glibc version of this - * function, as glibc goes out of its way to make it inaccessible. - */ - long long res; - __asm__ __volatile__("jmp 2f\n" - ".align 16\n" - "1:movq %1,%%rax\n" - LSS_ENTRYPOINT - "2:leaq 1b(%%rip),%0\n" - : "=r" (res) - : "i" (__NR_rt_sigreturn)); - return (void (*)(void))(uintptr_t)res; - } - #elif defined(__ARM_ARCH_3__) - /* Most definitions of _syscallX() neglect to mark "memory" as being - * clobbered. This causes problems with compilers, that do a better job - * at optimizing across __asm__ calls. - * So, we just have to redefine all of the _syscallX() macros. - */ - #undef LSS_REG - #define LSS_REG(r,a) register long __r##r __asm__("r"#r) = (long)a - #undef LSS_BODY - #define LSS_BODY(type,name,args...) \ - register long __res_r0 __asm__("r0"); \ - long __res; \ - __asm__ __volatile__ (__syscall(name) \ - : "=r"(__res_r0) : args : "lr", "memory"); \ - __res = __res_r0; \ - LSS_RETURN(type, __res) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)(void) { \ - LSS_BODY(type, name); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_REG(0, arg1); LSS_BODY(type, name, "r"(__r0)); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1)); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2)); \ - } - #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3)); \ - } - #undef _syscall5 - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ - "r"(__r4)); \ - } - #undef _syscall6 - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); LSS_REG(5, arg6); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ - "r"(__r4), "r"(__r5)); \ - } - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - long __res; - { - register int __flags __asm__("r0") = flags; - register void *__stack __asm__("r1") = child_stack; - register void *__ptid __asm__("r2") = parent_tidptr; - register void *__tls __asm__("r3") = newtls; - register int *__ctid __asm__("r4") = child_tidptr; - __asm__ __volatile__(/* if (fn == NULL || child_stack == NULL) - * return -EINVAL; - */ - "cmp %2,#0\n" - "cmpne %3,#0\n" - "moveq %0,%1\n" - "beq 1f\n" - - /* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - "str %5,[%3,#-4]!\n" - "str %2,[%3,#-4]!\n" - - /* %r0 = syscall(%r0 = flags, - * %r1 = child_stack, - * %r2 = parent_tidptr, - * %r3 = newtls, - * %r4 = child_tidptr) - */ - __syscall(clone)"\n" - - /* if (%r0 != 0) - * return %r0; - */ - "movs %0,r0\n" - "bne 1f\n" - - /* In the child, now. Call "fn(arg)". - */ - "ldr r0,[sp, #4]\n" - "mov lr,pc\n" - "ldr pc,[sp]\n" - - /* Call _exit(%r0). - */ - __syscall(exit)"\n" - "1:\n" - : "=r" (__res) - : "i"(-EINVAL), - "r"(fn), "r"(__stack), "r"(__flags), "r"(arg), - "r"(__ptid), "r"(__tls), "r"(__ctid) - : "cc", "lr", "memory"); - } - LSS_RETURN(int, __res); - } - #elif defined(__ARM_EABI__) - /* Most definitions of _syscallX() neglect to mark "memory" as being - * clobbered. This causes problems with compilers, that do a better job - * at optimizing across __asm__ calls. - * So, we just have to redefine all fo the _syscallX() macros. - */ - #undef LSS_REG - #define LSS_REG(r,a) register long __r##r __asm__("r"#r) = (long)a - #undef LSS_BODY - #define LSS_BODY(type,name,args...) \ - register long __res_r0 __asm__("r0"); \ - long __res; \ - __asm__ __volatile__ ("push {r7}\n" \ - "mov r7, %1\n" \ - "swi 0x0\n" \ - "pop {r7}\n" \ - : "=r"(__res_r0) \ - : "i"(__NR_##name) , ## args \ - : "lr", "memory"); \ - __res = __res_r0; \ - LSS_RETURN(type, __res) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)(void) { \ - LSS_BODY(type, name); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_REG(0, arg1); LSS_BODY(type, name, "r"(__r0)); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1)); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2)); \ - } - #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3)); \ - } - #undef _syscall5 - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ - "r"(__r4)); \ - } - #undef _syscall6 - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); LSS_REG(5, arg6); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ - "r"(__r4), "r"(__r5)); \ - } - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - long __res; - if (fn == NULL || child_stack == NULL) { - __res = -EINVAL; - LSS_RETURN(int, __res); - } - - /* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - { - uintptr_t* cstack = (uintptr_t*)child_stack - 2; - cstack[0] = (uintptr_t)fn; - cstack[1] = (uintptr_t)arg; - child_stack = cstack; - } - { - register int __flags __asm__("r0") = flags; - register void *__stack __asm__("r1") = child_stack; - register void *__ptid __asm__("r2") = parent_tidptr; - register void *__tls __asm__("r3") = newtls; - register int *__ctid __asm__("r4") = child_tidptr; - __asm__ __volatile__( -#ifdef __thumb2__ - "push {r7}\n" -#endif - /* %r0 = syscall(%r0 = flags, - * %r1 = child_stack, - * %r2 = parent_tidptr, - * %r3 = newtls, - * %r4 = child_tidptr) - */ - "mov r7, %6\n" - "swi 0x0\n" - - /* if (%r0 != 0) - * return %r0; - */ - "cmp r0, #0\n" - "bne 1f\n" - - /* In the child, now. Call "fn(arg)". - */ - "ldr r0,[sp, #4]\n" - - "ldr lr,[sp]\n" - "blx lr\n" - - /* Call _exit(%r0). - */ - "mov r7, %7\n" - "swi 0x0\n" - /* Unreachable */ - "bkpt #0\n" - "1:\n" -#ifdef __thumb2__ - "pop {r7}\n" -#endif - "movs %0,r0\n" - : "=r"(__res) - : "r"(__stack), "r"(__flags), "r"(__ptid), "r"(__tls), "r"(__ctid), - "i"(__NR_clone), "i"(__NR_exit) - : "cc", "lr", "memory" -#ifndef __thumb2__ - , "r7" -#endif - ); - } - LSS_RETURN(int, __res); - } - #elif defined(__aarch64__) - /* Most definitions of _syscallX() neglect to mark "memory" as being - * clobbered. This causes problems with compilers, that do a better job - * at optimizing across __asm__ calls. - * So, we just have to redefine all of the _syscallX() macros. - */ - #undef LSS_REG - #define LSS_REG(r,a) register int64_t __r##r __asm__("x"#r) = (int64_t)a - #undef LSS_BODY - #define LSS_BODY(type,name,args...) \ - register int64_t __res_x0 __asm__("x0"); \ - int64_t __res; \ - __asm__ __volatile__ ("mov x8, %1\n" \ - "svc 0x0\n" \ - : "=r"(__res_x0) \ - : "i"(__NR_##name) , ## args \ - : "x8", "memory"); \ - __res = __res_x0; \ - LSS_RETURN(type, __res) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)(void) { \ - LSS_BODY(type, name); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_REG(0, arg1); LSS_BODY(type, name, "r"(__r0)); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1)); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2)); \ - } - #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3)); \ - } - #undef _syscall5 - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ - "r"(__r4)); \ - } - #undef _syscall6 - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); LSS_REG(5, arg6); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ - "r"(__r4), "r"(__r5)); \ - } - - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - int64_t __res; - { - register uint64_t __flags __asm__("x0") = (uint64_t)flags; - register void *__stack __asm__("x1") = child_stack; - register void *__ptid __asm__("x2") = parent_tidptr; - register void *__tls __asm__("x3") = newtls; - register int *__ctid __asm__("x4") = child_tidptr; - __asm__ __volatile__(/* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - "stp %1, %4, [%2, #-16]!\n" - - /* %x0 = syscall(%x0 = flags, - * %x1 = child_stack, - * %x2 = parent_tidptr, - * %x3 = newtls, - * %x4 = child_tidptr) - */ - "mov x8, %8\n" - "svc 0x0\n" - - /* if (%r0 != 0) - * return %r0; - */ - "mov %0, x0\n" - "cbnz x0, 1f\n" - - /* In the child, now. Call "fn(arg)". - */ - "ldp x1, x0, [sp], #16\n" - "blr x1\n" - - /* Call _exit(%r0). - */ - "mov x8, %9\n" - "svc 0x0\n" - "1:\n" - : "=r" (__res) - : "r"(fn), "r"(__stack), "r"(__flags), "r"(arg), - "r"(__ptid), "r"(__tls), "r"(__ctid), - "i"(__NR_clone), "i"(__NR_exit) - : "cc", "x8", "memory"); - } - LSS_RETURN(int, __res); - } - LSS_INLINE void (*LSS_NAME(restore_rt)(void))(void) { - /* On aarch64, the kernel does not know how to return from - * a signal handler. Instead, it relies on user space to provide a - * restorer function that calls the rt_sigreturn() system call. - * Unfortunately, we cannot just reference the glibc version of this - * function, as glibc goes out of its way to make it inaccessible. - * - * This is simular to __kernel_rt_sigreturn(). - */ - long long res; - __asm__ __volatile__("b 2f\n" - "1:\n" - /* NOP required by some unwinder. For details. - * see aarch64's vdso/sigreturn.S in the kernel. - */ - "nop\n" - /* Some system softwares recognize this instruction - * sequence to unwind from * signal handlers. Do not - * modify the next two instructions. - */ - "mov x8, %1\n" - "svc 0x0\n" - "2:\n" - "adr %0, 1b\n" - : "=r" (res) - : "i" (__NR_rt_sigreturn)); - return (void (*)(void))(uintptr_t)res; - } - #elif defined(__mips__) - #undef LSS_REG - #define LSS_REG(r,a) register unsigned long __r##r __asm__("$"#r) = \ - (unsigned long)(a) - #undef LSS_BODY - #undef LSS_SYSCALL_CLOBBERS - #if _MIPS_SIM == _MIPS_SIM_ABI32 - #define LSS_SYSCALL_CLOBBERS "$1", "$3", "$8", "$9", "$10", \ - "$11", "$12", "$13", "$14", "$15", \ - "$24", "$25", "hi", "lo", "memory" - #else - #define LSS_SYSCALL_CLOBBERS "$1", "$3", "$10", "$11", "$12", \ - "$13", "$14", "$15", "$24", "$25", \ - "hi", "lo", "memory" - #endif - #define LSS_BODY(type,name,r7,...) \ - register unsigned long __v0 __asm__("$2") = __NR_##name; \ - __asm__ __volatile__ ("syscall\n" \ - : "=r"(__v0), r7 (__r7) \ - : "0"(__v0), ##__VA_ARGS__ \ - : LSS_SYSCALL_CLOBBERS); \ - LSS_RETURN(type, __v0, __r7) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)(void) { \ - register unsigned long __r7 __asm__("$7"); \ - LSS_BODY(type, name, "=r"); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - register unsigned long __r7 __asm__("$7"); \ - LSS_REG(4, arg1); LSS_BODY(type, name, "=r", "r"(__r4)); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - register unsigned long __r7 __asm__("$7"); \ - LSS_REG(4, arg1); LSS_REG(5, arg2); \ - LSS_BODY(type, name, "=r", "r"(__r4), "r"(__r5)); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - register unsigned long __r7 __asm__("$7"); \ - LSS_REG(4, arg1); LSS_REG(5, arg2); LSS_REG(6, arg3); \ - LSS_BODY(type, name, "=r", "r"(__r4), "r"(__r5), "r"(__r6)); \ - } - #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_REG(4, arg1); LSS_REG(5, arg2); LSS_REG(6, arg3); \ - LSS_REG(7, arg4); \ - LSS_BODY(type, name, "+r", "r"(__r4), "r"(__r5), "r"(__r6)); \ - } - #undef _syscall5 - #if _MIPS_SIM == _MIPS_SIM_ABI32 - /* The old 32bit MIPS system call API passes the fifth and sixth argument - * on the stack, whereas the new APIs use registers "r8" and "r9". - */ - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_REG(4, arg1); LSS_REG(5, arg2); LSS_REG(6, arg3); \ - LSS_REG(7, arg4); \ - register unsigned long __v0 __asm__("$2") = __NR_##name; \ - __asm__ __volatile__ (".set noreorder\n" \ - "subu $29, 32\n" \ - "sw %5, 16($29)\n" \ - "syscall\n" \ - "addiu $29, 32\n" \ - ".set reorder\n" \ - : "+r"(__v0), "+r" (__r7) \ - : "r"(__r4), "r"(__r5), \ - "r"(__r6), "r" ((unsigned long)arg5) \ - : "$8", "$9", "$10", "$11", "$12", \ - "$13", "$14", "$15", "$24", "$25", \ - "memory"); \ - LSS_RETURN(type, __v0, __r7); \ - } - #else - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_REG(4, arg1); LSS_REG(5, arg2); LSS_REG(6, arg3); \ - LSS_REG(7, arg4); LSS_REG(8, arg5); \ - LSS_BODY(type, name, "+r", "r"(__r4), "r"(__r5), "r"(__r6), \ - "r"(__r8)); \ - } - #endif - #undef _syscall6 - #if _MIPS_SIM == _MIPS_SIM_ABI32 - /* The old 32bit MIPS system call API passes the fifth and sixth argument - * on the stack, whereas the new APIs use registers "r8" and "r9". - */ - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_REG(4, arg1); LSS_REG(5, arg2); LSS_REG(6, arg3); \ - LSS_REG(7, arg4); \ - register unsigned long __v0 __asm__("$2") = __NR_##name; \ - __asm__ __volatile__ (".set noreorder\n" \ - "subu $29, 32\n" \ - "sw %5, 16($29)\n" \ - "sw %6, 20($29)\n" \ - "syscall\n" \ - "addiu $29, 32\n" \ - ".set reorder\n" \ - : "+r"(__v0), "+r" (__r7) \ - : "r"(__r4), "r"(__r5), \ - "r"(__r6), "r" ((unsigned long)arg5), \ - "r" ((unsigned long)arg6) \ - : "$8", "$9", "$10", "$11", "$12", \ - "$13", "$14", "$15", "$24", "$25", \ - "memory"); \ - LSS_RETURN(type, __v0, __r7); \ - } - #else - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5,type6 arg6) { \ - LSS_REG(4, arg1); LSS_REG(5, arg2); LSS_REG(6, arg3); \ - LSS_REG(7, arg4); LSS_REG(8, arg5); LSS_REG(9, arg6); \ - LSS_BODY(type, name, "+r", "r"(__r4), "r"(__r5), "r"(__r6), \ - "r"(__r8), "r"(__r9)); \ - } - #endif - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - register unsigned long __v0 __asm__("$2") = -EINVAL; - register unsigned long __r7 __asm__("$7") = (unsigned long)newtls; - { - register int __flags __asm__("$4") = flags; - register void *__stack __asm__("$5") = child_stack; - register void *__ptid __asm__("$6") = parent_tidptr; - register int *__ctid __asm__("$8") = child_tidptr; - __asm__ __volatile__( - #if _MIPS_SIM == _MIPS_SIM_ABI32 && _MIPS_SZPTR == 32 - "subu $29,24\n" - #elif _MIPS_SIM == _MIPS_SIM_NABI32 - "sub $29,16\n" - #else - "dsubu $29,16\n" - #endif - - /* if (fn == NULL || child_stack == NULL) - * return -EINVAL; - */ - "beqz %4,1f\n" - "beqz %5,1f\n" - - /* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - #if _MIPS_SIM == _MIPS_SIM_ABI32 && _MIPS_SZPTR == 32 - "subu %5,32\n" - "sw %4,0(%5)\n" - "sw %7,4(%5)\n" - #elif _MIPS_SIM == _MIPS_SIM_NABI32 - "sub %5,32\n" - "sw %4,0(%5)\n" - "sw %7,8(%5)\n" - #else - "dsubu %5,32\n" - "sd %4,0(%5)\n" - "sd %7,8(%5)\n" - #endif - - /* $7 = syscall($4 = flags, - * $5 = child_stack, - * $6 = parent_tidptr, - * $7 = newtls, - * $8 = child_tidptr) - */ - "li $2,%2\n" - "syscall\n" - - /* if ($7 != 0) - * return $2; - */ - "bnez $7,1f\n" - "bnez $2,1f\n" - - /* In the child, now. Call "fn(arg)". - */ - #if _MIPS_SIM == _MIPS_SIM_ABI32 && _MIPS_SZPTR == 32 - "lw $25,0($29)\n" - "lw $4,4($29)\n" - #elif _MIPS_SIM == _MIPS_SIM_NABI32 - "lw $25,0($29)\n" - "lw $4,8($29)\n" - #else - "ld $25,0($29)\n" - "ld $4,8($29)\n" - #endif - "jalr $25\n" - - /* Call _exit($2) - */ - "move $4,$2\n" - "li $2,%3\n" - "syscall\n" - - "1:\n" - #if _MIPS_SIM == _MIPS_SIM_ABI32 && _MIPS_SZPTR == 32 - "addu $29, 24\n" - #elif _MIPS_SIM == _MIPS_SIM_NABI32 - "add $29, 16\n" - #else - "daddu $29,16\n" - #endif - : "+r" (__v0), "+r" (__r7) - : "i"(__NR_clone), "i"(__NR_exit), "r"(fn), - "r"(__stack), "r"(__flags), "r"(arg), - "r"(__ptid), "r"(__ctid) - : "$9", "$10", "$11", "$12", "$13", "$14", "$15", - "$24", "$25", "memory"); - } - LSS_RETURN(int, __v0, __r7); - } - #elif defined (__PPC__) - #undef LSS_LOADARGS_0 - #define LSS_LOADARGS_0(name, dummy...) \ - __sc_0 = __NR_##name - #undef LSS_LOADARGS_1 - #define LSS_LOADARGS_1(name, arg1) \ - LSS_LOADARGS_0(name); \ - __sc_3 = (unsigned long) (arg1) - #undef LSS_LOADARGS_2 - #define LSS_LOADARGS_2(name, arg1, arg2) \ - LSS_LOADARGS_1(name, arg1); \ - __sc_4 = (unsigned long) (arg2) - #undef LSS_LOADARGS_3 - #define LSS_LOADARGS_3(name, arg1, arg2, arg3) \ - LSS_LOADARGS_2(name, arg1, arg2); \ - __sc_5 = (unsigned long) (arg3) - #undef LSS_LOADARGS_4 - #define LSS_LOADARGS_4(name, arg1, arg2, arg3, arg4) \ - LSS_LOADARGS_3(name, arg1, arg2, arg3); \ - __sc_6 = (unsigned long) (arg4) - #undef LSS_LOADARGS_5 - #define LSS_LOADARGS_5(name, arg1, arg2, arg3, arg4, arg5) \ - LSS_LOADARGS_4(name, arg1, arg2, arg3, arg4); \ - __sc_7 = (unsigned long) (arg5) - #undef LSS_LOADARGS_6 - #define LSS_LOADARGS_6(name, arg1, arg2, arg3, arg4, arg5, arg6) \ - LSS_LOADARGS_5(name, arg1, arg2, arg3, arg4, arg5); \ - __sc_8 = (unsigned long) (arg6) - #undef LSS_ASMINPUT_0 - #define LSS_ASMINPUT_0 "0" (__sc_0) - #undef LSS_ASMINPUT_1 - #define LSS_ASMINPUT_1 LSS_ASMINPUT_0, "1" (__sc_3) - #undef LSS_ASMINPUT_2 - #define LSS_ASMINPUT_2 LSS_ASMINPUT_1, "2" (__sc_4) - #undef LSS_ASMINPUT_3 - #define LSS_ASMINPUT_3 LSS_ASMINPUT_2, "3" (__sc_5) - #undef LSS_ASMINPUT_4 - #define LSS_ASMINPUT_4 LSS_ASMINPUT_3, "4" (__sc_6) - #undef LSS_ASMINPUT_5 - #define LSS_ASMINPUT_5 LSS_ASMINPUT_4, "5" (__sc_7) - #undef LSS_ASMINPUT_6 - #define LSS_ASMINPUT_6 LSS_ASMINPUT_5, "6" (__sc_8) - #undef LSS_BODY - #define LSS_BODY(nr, type, name, args...) \ - long __sc_ret, __sc_err; \ - { \ - register unsigned long __sc_0 __asm__ ("r0"); \ - register unsigned long __sc_3 __asm__ ("r3"); \ - register unsigned long __sc_4 __asm__ ("r4"); \ - register unsigned long __sc_5 __asm__ ("r5"); \ - register unsigned long __sc_6 __asm__ ("r6"); \ - register unsigned long __sc_7 __asm__ ("r7"); \ - register unsigned long __sc_8 __asm__ ("r8"); \ - \ - LSS_LOADARGS_##nr(name, args); \ - __asm__ __volatile__ \ - ("sc\n\t" \ - "mfcr %0" \ - : "=&r" (__sc_0), \ - "=&r" (__sc_3), "=&r" (__sc_4), \ - "=&r" (__sc_5), "=&r" (__sc_6), \ - "=&r" (__sc_7), "=&r" (__sc_8) \ - : LSS_ASMINPUT_##nr \ - : "cr0", "ctr", "memory", \ - "r9", "r10", "r11", "r12"); \ - __sc_ret = __sc_3; \ - __sc_err = __sc_0; \ - } \ - LSS_RETURN(type, __sc_ret, __sc_err) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)(void) { \ - LSS_BODY(0, type, name); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_BODY(1, type, name, arg1); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_BODY(2, type, name, arg1, arg2); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_BODY(3, type, name, arg1, arg2, arg3); \ - } - #undef _syscall4 - #define _syscall4(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_BODY(4, type, name, arg1, arg2, arg3, arg4); \ - } - #undef _syscall5 - #define _syscall5(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4, type5, arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_BODY(5, type, name, arg1, arg2, arg3, arg4, arg5); \ - } - #undef _syscall6 - #define _syscall6(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4, type5, arg5, type6, arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_BODY(6, type, name, arg1, arg2, arg3, arg4, arg5, arg6); \ - } - /* clone function adapted from glibc 2.3.6 clone.S */ - /* TODO(csilvers): consider wrapping some args up in a struct, like we - * do for i386's _syscall6, so we can compile successfully on gcc 2.95 - */ - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - long __ret, __err; - { - register int (*__fn)(void *) __asm__ ("r8") = fn; - register void *__cstack __asm__ ("r4") = child_stack; - register int __flags __asm__ ("r3") = flags; - register void * __arg __asm__ ("r9") = arg; - register int * __ptidptr __asm__ ("r5") = parent_tidptr; - register void * __newtls __asm__ ("r6") = newtls; - register int * __ctidptr __asm__ ("r7") = child_tidptr; - __asm__ __volatile__( - /* check for fn == NULL - * and child_stack == NULL - */ - "cmpwi cr0, %6, 0\n\t" - "cmpwi cr1, %7, 0\n\t" - "cror cr0*4+eq, cr1*4+eq, cr0*4+eq\n\t" - "beq- cr0, 1f\n\t" - - /* set up stack frame for child */ - "clrrwi %7, %7, 4\n\t" - "li 0, 0\n\t" - "stwu 0, -16(%7)\n\t" - - /* fn, arg, child_stack are saved across the syscall: r28-30 */ - "mr 28, %6\n\t" - "mr 29, %7\n\t" - "mr 27, %9\n\t" - - /* syscall */ - "li 0, %4\n\t" - /* flags already in r3 - * child_stack already in r4 - * ptidptr already in r5 - * newtls already in r6 - * ctidptr already in r7 - */ - "sc\n\t" - - /* Test if syscall was successful */ - "cmpwi cr1, 3, 0\n\t" - "crandc cr1*4+eq, cr1*4+eq, cr0*4+so\n\t" - "bne- cr1, 1f\n\t" - - /* Do the function call */ - "mtctr 28\n\t" - "mr 3, 27\n\t" - "bctrl\n\t" - - /* Call _exit(r3) */ - "li 0, %5\n\t" - "sc\n\t" - - /* Return to parent */ - "1:\n" - "mfcr %1\n\t" - "mr %0, 3\n\t" - : "=r" (__ret), "=r" (__err) - : "0" (-1), "1" (EINVAL), - "i" (__NR_clone), "i" (__NR_exit), - "r" (__fn), "r" (__cstack), "r" (__flags), - "r" (__arg), "r" (__ptidptr), "r" (__newtls), - "r" (__ctidptr) - : "cr0", "cr1", "memory", "ctr", - "r0", "r29", "r27", "r28"); - } - LSS_RETURN(int, __ret, __err); - } - #elif defined(__s390__) - #undef LSS_REG - #define LSS_REG(r, a) register unsigned long __r##r __asm__("r"#r) = (unsigned long) a - #undef LSS_BODY - #define LSS_BODY(type, name, args...) \ - register unsigned long __nr __asm__("r1") \ - = (unsigned long)(__NR_##name); \ - register long __res_r2 __asm__("r2"); \ - long __res; \ - __asm__ __volatile__ \ - ("svc 0\n\t" \ - : "=d"(__res_r2) \ - : "d"(__nr), ## args \ - : "memory"); \ - __res = __res_r2; \ - LSS_RETURN(type, __res) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)(void) { \ - LSS_BODY(type, name); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_REG(2, arg1); \ - LSS_BODY(type, name, "0"(__r2)); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_REG(2, arg1); LSS_REG(3, arg2); \ - LSS_BODY(type, name, "0"(__r2), "d"(__r3)); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_REG(2, arg1); LSS_REG(3, arg2); LSS_REG(4, arg3); \ - LSS_BODY(type, name, "0"(__r2), "d"(__r3), "d"(__r4)); \ - } - #undef _syscall4 - #define _syscall4(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, \ - type4 arg4) { \ - LSS_REG(2, arg1); LSS_REG(3, arg2); LSS_REG(4, arg3); \ - LSS_REG(5, arg4); \ - LSS_BODY(type, name, "0"(__r2), "d"(__r3), "d"(__r4), \ - "d"(__r5)); \ - } - #undef _syscall5 - #define _syscall5(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4, type5, arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, \ - type4 arg4, type5 arg5) { \ - LSS_REG(2, arg1); LSS_REG(3, arg2); LSS_REG(4, arg3); \ - LSS_REG(5, arg4); LSS_REG(6, arg5); \ - LSS_BODY(type, name, "0"(__r2), "d"(__r3), "d"(__r4), \ - "d"(__r5), "d"(__r6)); \ - } - #undef _syscall6 - #define _syscall6(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4, type5, arg5, type6, arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, \ - type4 arg4, type5 arg5, type6 arg6) { \ - LSS_REG(2, arg1); LSS_REG(3, arg2); LSS_REG(4, arg3); \ - LSS_REG(5, arg4); LSS_REG(6, arg5); LSS_REG(7, arg6); \ - LSS_BODY(type, name, "0"(__r2), "d"(__r3), "d"(__r4), \ - "d"(__r5), "d"(__r6), "d"(__r7)); \ - } - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - long __ret; - { - register int (*__fn)(void *) __asm__ ("r1") = fn; - register void *__cstack __asm__ ("r2") = child_stack; - register int __flags __asm__ ("r3") = flags; - register void *__arg __asm__ ("r0") = arg; - register int *__ptidptr __asm__ ("r4") = parent_tidptr; - register void *__newtls __asm__ ("r6") = newtls; - register int *__ctidptr __asm__ ("r5") = child_tidptr; - __asm__ __volatile__ ( - #ifndef __s390x__ - /* arg already in r0 */ - "ltr %4, %4\n\t" /* check fn, which is already in r1 */ - "jz 1f\n\t" /* NULL function pointer, return -EINVAL */ - "ltr %5, %5\n\t" /* check child_stack, which is already in r2 */ - "jz 1f\n\t" /* NULL stack pointer, return -EINVAL */ - /* flags already in r3 */ - /* parent_tidptr already in r4 */ - /* child_tidptr already in r5 */ - /* newtls already in r6 */ - "svc %2\n\t" /* invoke clone syscall */ - "ltr %0,%%r2\n\t" /* load return code into __ret and test */ - "jnz 1f\n\t" /* return to parent if non-zero */ - /* start child thread */ - "lr %%r2, %7\n\t" /* set first parameter to void *arg */ - "ahi %%r15, -96\n\t" /* make room on the stack for the save area */ - "xc 0(4,%%r15), 0(%%r15)\n\t" - "basr %%r14, %4\n\t" /* jump to fn */ - "svc %3\n" /* invoke exit syscall */ - "1:\n" - #else - /* arg already in r0 */ - "ltgr %4, %4\n\t" /* check fn, which is already in r1 */ - "jz 1f\n\t" /* NULL function pointer, return -EINVAL */ - "ltgr %5, %5\n\t" /* check child_stack, which is already in r2 */ - "jz 1f\n\t" /* NULL stack pointer, return -EINVAL */ - /* flags already in r3 */ - /* parent_tidptr already in r4 */ - /* child_tidptr already in r5 */ - /* newtls already in r6 */ - "svc %2\n\t" /* invoke clone syscall */ - "ltgr %0, %%r2\n\t" /* load return code into __ret and test */ - "jnz 1f\n\t" /* return to parent if non-zero */ - /* start child thread */ - "lgr %%r2, %7\n\t" /* set first parameter to void *arg */ - "aghi %%r15, -160\n\t" /* make room on the stack for the save area */ - "xc 0(8,%%r15), 0(%%r15)\n\t" - "basr %%r14, %4\n\t" /* jump to fn */ - "svc %3\n" /* invoke exit syscall */ - "1:\n" - #endif - : "=r" (__ret) - : "0" (-EINVAL), "i" (__NR_clone), "i" (__NR_exit), - "d" (__fn), "d" (__cstack), "d" (__flags), "d" (__arg), - "d" (__ptidptr), "d" (__newtls), "d" (__ctidptr) - : "cc", "r14", "memory" - ); - } - LSS_RETURN(int, __ret); - } - #elif defined(__riscv) && __riscv_xlen == 64 - #undef LSS_REG - #define LSS_REG(r,a) register int64_t __r##r __asm__("a"#r) = (int64_t)a - #undef LSS_BODY - #define LSS_BODY(type,name,args...) \ - register int64_t __res_a0 __asm__("a0"); \ - register int64_t __a7 __asm__("a7") = __NR_##name; \ - int64_t __res; \ - __asm__ __volatile__ ("scall\n" \ - : "=r"(__res_a0) \ - : "r"(__a7) , ## args \ - : "memory"); \ - __res = __res_a0; \ - LSS_RETURN(type, __res) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)(void) { \ - LSS_BODY(type, name); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_REG(0, arg1); LSS_BODY(type, name, "r"(__r0)); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1)); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2)); \ - } - #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3)); \ - } - #undef _syscall5 - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ - "r"(__r4)); \ - } - #undef _syscall6 - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); LSS_REG(5, arg6); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ - "r"(__r4), "r"(__r5)); \ - } - - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - int64_t __res; - { - register int64_t __res_a0 __asm__("a0"); - register uint64_t __flags __asm__("a0") = (uint64_t)flags; - register void *__stack __asm__("a1") = child_stack; - register void *__ptid __asm__("a2") = parent_tidptr; - register void *__tls __asm__("a3") = newtls; - register int *__ctid __asm__("a4") = child_tidptr; - __asm__ __volatile__(/* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - "addi %2,%2,-16\n" - "sd %1, 0(%2)\n" - "sd %4, 8(%2)\n" - - /* %a0 = syscall(%a0 = flags, - * %a1 = child_stack, - * %a2 = parent_tidptr, - * %a3 = newtls, - * %a4 = child_tidptr) - */ - "li a7, %8\n" - "scall\n" - - /* if (%a0 != 0) - * return %a0; - */ - "bnez %0, 1f\n" - - /* In the child, now. Call "fn(arg)". - */ - "ld a1, 0(sp)\n" - "ld a0, 8(sp)\n" - "jalr a1\n" - - /* Call _exit(%a0). - */ - "li a7, %9\n" - "scall\n" - "1:\n" - : "=r" (__res_a0) - : "r"(fn), "r"(__stack), "r"(__flags), "r"(arg), - "r"(__ptid), "r"(__tls), "r"(__ctid), - "i"(__NR_clone), "i"(__NR_exit) - : "cc", "memory"); - __res = __res_a0; - } - LSS_RETURN(int, __res); - } - #elif defined(__e2k__) - - #undef _LSS_BODY - #define _LSS_BODY(nr, type, name, ...) \ - register unsigned long long __res; \ - __asm__ __volatile__ \ - ( \ - "{\n\t" \ - " sdisp %%ctpr1, 0x3\n\t" \ - " addd, s 0x0, %[sys_num], %%b[0]\n\t" \ - LSS_BODY_ASM##nr \ - "}\n\t" \ - "{\n\t" \ - " call %%ctpr1, wbs = %#\n\t" \ - "}\n\t" \ - "{\n\t" \ - " addd, s 0x0, %%b[0], %[res]\n\t" \ - "}\n\t" \ - : [res] "=r" (__res) \ - : \ - LSS_BODY_ARG##nr(__VA_ARGS__) \ - [sys_num] "ri" (__NR_##name) \ - : "ctpr1", "ctpr2", "ctpr3", \ - "b[0]", "b[1]", "b[2]", "b[3]", \ - "b[4]", "b[5]", "b[6]", "b[7]" \ - ); \ - LSS_RETURN(type, __res); - - #undef LSS_BODY - #define LSS_BODY(nr, type, name, args...) \ - _LSS_BODY(nr, type, name, ## args) - - #undef LSS_BODY_ASM0 - #undef LSS_BODY_ASM1 - #undef LSS_BODY_ASM2 - #undef LSS_BODY_ASM3 - #undef LSS_BODY_ASM4 - #undef LSS_BODY_ASM5 - #undef LSS_BODY_ASM6 - - #define LSS_BODY_ASM0 - #define LSS_BODY_ASM1 LSS_BODY_ASM0 \ - " addd, s 0x0, %[arg1], %%b[1]\n\t" - #define LSS_BODY_ASM2 LSS_BODY_ASM1 \ - " addd, s 0x0, %[arg2], %%b[2]\n\t" - #define LSS_BODY_ASM3 LSS_BODY_ASM2 \ - " addd, s 0x0, %[arg3], %%b[3]\n\t" - #define LSS_BODY_ASM4 LSS_BODY_ASM3 \ - " addd, s 0x0, %[arg4], %%b[4]\n\t" - #define LSS_BODY_ASM5 LSS_BODY_ASM4 \ - " addd, s 0x0, %[arg5], %%b[5]\n\t" - #define LSS_BODY_ASM6 LSS_BODY_ASM5 \ - "}\n\t" \ - "{\n\t" \ - " addd, s 0x0, %[arg6], %%b[6]\n\t" - - #undef LSS_SYSCALL_ARG - #define LSS_SYSCALL_ARG(a) ((unsigned long long)(uintptr_t)(a)) - - #undef LSS_BODY_ARG0 - #undef LSS_BODY_ARG1 - #undef LSS_BODY_ARG2 - #undef LSS_BODY_ARG3 - #undef LSS_BODY_ARG4 - #undef LSS_BODY_ARG5 - #undef LSS_BODY_ARG6 - - #define LSS_BODY_ARG0() - #define LSS_BODY_ARG1(_arg1) \ - [arg1] "ri" LSS_SYSCALL_ARG(_arg1), - #define LSS_BODY_ARG2(_arg1, _arg2) \ - LSS_BODY_ARG1(_arg1) \ - [arg2] "ri" LSS_SYSCALL_ARG(_arg2), - #define LSS_BODY_ARG3(_arg1, _arg2, _arg3) \ - LSS_BODY_ARG2(_arg1, _arg2) \ - [arg3] "ri" LSS_SYSCALL_ARG(_arg3), - #define LSS_BODY_ARG4(_arg1, _arg2, _arg3, _arg4) \ - LSS_BODY_ARG3(_arg1, _arg2, _arg3) \ - [arg4] "ri" LSS_SYSCALL_ARG(_arg4), - #define LSS_BODY_ARG5(_arg1, _arg2, _arg3, _arg4, _arg5) \ - LSS_BODY_ARG4(_arg1, _arg2, _arg3, _arg4) \ - [arg5] "ri" LSS_SYSCALL_ARG(_arg5), - #define LSS_BODY_ARG6(_arg1, _arg2, _arg3, _arg4, _arg5, _arg6) \ - LSS_BODY_ARG5(_arg1, _arg2, _arg3, _arg4, _arg5) \ - [arg6] "ri" LSS_SYSCALL_ARG(_arg6), - - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)(void) { \ - LSS_BODY(0, type, name); \ - } - - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_BODY(1, type, name, arg1) \ - } - - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_BODY(2, type, name, arg1, arg2) \ - } - - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_BODY(3, type, name, arg1, arg2, arg3) \ - } - - #undef _syscall4 - #define _syscall4(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_BODY(4, type, name, arg1, arg2, arg3, arg4) \ - } - - #undef _syscall5 - #define _syscall5(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4, type5, arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_BODY(5, type, name, arg1, arg2, arg3, arg4, arg5) \ - } - - #undef _syscall6 - #define _syscall6(type, name, type1, arg1, type2, arg2, type3, arg3, \ - type4, arg4, type5, arg5, type6, arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_BODY(6, type, name, arg1, arg2, arg3, arg4, arg5, arg6) \ - } - - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - unsigned long long __res; - - __asm__ __volatile__ ( - "{\n\t" - " addd,s 0x0, %[nr_clone], %%b[0]\n\t" - " addd,s 0x0, %[flags], %%db[1]\n\t" - " addd,s 0x0, %[child_stack], %%db[2]\n\t" - " addd,s 0x0, %[parent_tidptr], %%db[3]\n\t" - " addd,s 0x0, %[child_tidptr], %%db[4]\n\t" - " addd,s 0x0, %[newtls], %%db[5]\n\t" - "}\n\t" - /* if (fn == NULL) - * return -EINVAL; - */ - - "{\n\t" - " disp %%ctpr1, .L1\n\t" - "}\n\t" - "{\n\t" - " cmpesb,s 0x0, %[fn], %%pred0\n\t" - "}\n\t" - "{\n\t" - " ct %%ctpr1 ? %%pred0\n\t" - "}\n\t" - - /* if (child_stack == NULL) - * return -EINVAL; - */ - "{\n\t" - " cmpesb,s 0x0, %%db[2], %%pred0\n\t" - "}\n\t" - "{\n\t" - " ct %%ctpr1 ? %%pred0\n\t" - "}\n\t" - - /* b[0] = syscall(%b[0] = __NR_clone, - * %db[1] = flags, - * %db[2] = child_stack, - * %db[3] = parent_tidptr, - * %db[4] = child_tidptr, - * %db[5] = newtls) - */ - "{\n\t" - " sdisp %%ctpr1, 0x3\n\t" - "}\n\t" - "{\n\t" - " call %%ctpr1, wbs = %#\n\t" - "}\n\t" - - /* if (%[b0] != 0) - * return %b[0]; - */ - "{\n\t" - " disp %%ctpr1, .L2\n\t" - " cmpesb,s 0x0, %%b[0], %%pred0\n\t" - "}\n\t" - "{\n\t" - " ct %%ctpr1 ? ~%%pred0\n\t" - "}\n\t" - /* In the child, now. Call "fn(arg)". - */ - - "{\n\t" - " movtd,s %[fn], %%ctpr1\n\t" - "}\n\t" - "{\n\t" - " addd,s 0x0, %[arg], %%db[0]\n\t" - "}\n\t" - "{\n\t" - " call %%ctpr1, wbs = %#\n\t" - "}\n\t" - /* Call _exit(%b[0]). - */ - - "{\n\t" - " sdisp %%ctpr1, 0x3\n\t" - " addd,s 0x0, %%b[0], %%b[1]\n\t" - "}\n\t" - "{\n\t" - " addd,s 0x0, %[nr_exit], %%b[0]\n\t" - "}\n\t" - "{\n\t" - " call %%ctpr1, wbs = %#\n\t" - "}\n\t" - "{\n\t" - " disp %%ctpr1, .L2\n\t" - " adds,s 0x0, 0x0, %%b[0]\n\t" - "}\n\t" - "{\n\t" - " ct %%ctpr1\n\t" - "}\n\t" - ".L1:\n\t" - "{\n\t" - " addd,s 0x0, %[einval], %%b[0]\n\t" - "}\n\t" - ".L2:\n\t" - "{\n\t" - " addd,s 0x0, %%b[0], %[res]\n\t" - "}\n\t" - : [res] "=r" LSS_SYSCALL_ARG(__res) - : [nr_clone] "ri" LSS_SYSCALL_ARG(__NR_clone) - [arg] "ri" LSS_SYSCALL_ARG(arg) - [nr_exit] "ri" LSS_SYSCALL_ARG(__NR_exit) - [flags] "ri" LSS_SYSCALL_ARG(flags) - [child_stack] "ri" LSS_SYSCALL_ARG(child_stack) - [parent_tidptr] "ri" - LSS_SYSCALL_ARG(parent_tidptr) - [newtls] "ri" LSS_SYSCALL_ARG(newtls) - [child_tidptr] "ri" - LSS_SYSCALL_ARG(child_tidptr) - [fn] "ri" LSS_SYSCALL_ARG(fn) - [einval] "ri" LSS_SYSCALL_ARG(-EINVAL) - : "ctpr1", "b[0]", "b[1]", "b[2]", "b[3]", - "b[4]", "b[5]", "pred0"); - LSS_RETURN(int, __res); - } - #elif defined(__loongarch_lp64) - /* Most definitions of _syscallX() neglect to mark "memory" as being - * clobbered. This causes problems with compilers, that do a better job - * at optimizing across __asm__ calls. - * So, we just have to redefine all of the _syscallX() macros. - */ - #undef LSS_REG - #define LSS_REG(ar,a) register int64_t __r##ar __asm__("a"#ar) = (int64_t)a - /* syscall is like subroutine calls, all caller-saved registers may be - * clobbered, we should add them to the |Clobbers| list. - * a0 is not included because it's in the output list. - */ - #define LSS_SYSCALL_CLOBBERS "t0", "t1", "t2", "t3", "t4", "t5", "t6", \ - "t7", "t8", "memory" - #undef LSS_BODY - #define LSS_BODY(type,name,args...) \ - register int64_t __res_a0 __asm__("a0"); \ - register int64_t __a7 __asm__("a7") = __NR_##name; \ - int64_t __res; \ - __asm__ __volatile__ ("syscall 0x0\n" \ - : "=r"(__res_a0) \ - : "r"(__a7), ## args \ - : LSS_SYSCALL_CLOBBERS); \ - __res = __res_a0; \ - LSS_RETURN(type, __res) - #undef _syscall0 - #define _syscall0(type, name) \ - type LSS_NAME(name)(void) { \ - LSS_BODY(type, name); \ - } - #undef _syscall1 - #define _syscall1(type, name, type1, arg1) \ - type LSS_NAME(name)(type1 arg1) { \ - LSS_REG(0, arg1); LSS_BODY(type, name, "r"(__r0)); \ - } - #undef _syscall2 - #define _syscall2(type, name, type1, arg1, type2, arg2) \ - type LSS_NAME(name)(type1 arg1, type2 arg2) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1)); \ - } - #undef _syscall3 - #define _syscall3(type, name, type1, arg1, type2, arg2, type3, arg3) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2)); \ - } - #undef _syscall4 - #define _syscall4(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3)); \ - } - #undef _syscall5 - #define _syscall5(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ - "r"(__r4)); \ - } - #undef _syscall6 - #define _syscall6(type,name,type1,arg1,type2,arg2,type3,arg3,type4,arg4, \ - type5,arg5,type6,arg6) \ - type LSS_NAME(name)(type1 arg1, type2 arg2, type3 arg3, type4 arg4, \ - type5 arg5, type6 arg6) { \ - LSS_REG(0, arg1); LSS_REG(1, arg2); LSS_REG(2, arg3); \ - LSS_REG(3, arg4); LSS_REG(4, arg5); LSS_REG(5, arg6); \ - LSS_BODY(type, name, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), \ - "r"(__r4), "r"(__r5)); \ - } - - LSS_INLINE int LSS_NAME(clone)(int (*fn)(void *), void *child_stack, - int flags, void *arg, int *parent_tidptr, - void *newtls, int *child_tidptr) { - int64_t __res; - { - register int64_t __res_a0 __asm__("a0"); - register uint64_t __flags __asm__("a0") = flags; - register void *__stack __asm__("a1") = child_stack; - register void *__ptid __asm__("a2") = parent_tidptr; - register void *__tls __asm__("a3") = newtls; - register int *__ctid __asm__("a4") = child_tidptr; - __asm__ __volatile__(/* Push "arg" and "fn" onto the stack that will be - * used by the child. - */ - "addi.d %2, %2, -16\n" - "st.d %1, %2, 8\n" - "st.d %4, %2, 0\n" - - /* %a0 = syscall(%a0 = flags, - * %a1 = child_stack, - * %a2 = parent_tidptr, - * %a3 = newtls, - * %a4 = child_tidptr) - */ - "li.d $a7, %8\n" - "syscall 0x0\n" - - /* if (%a0 != 0) - * return %a0; - */ - "bnez $a0, 1f\n" - - /* In the child, now. Call "fn(arg)". - */ - "ld.d $a0, $sp, 0\n" - "ld.d $a1, $sp, 8\n" - "addi.d $sp, $sp, 16\n" - "jirl $ra, $a1, 0\n" - - /* Call _exit(%a0). - */ - "li.d $a7, %9\n" - "syscall 0x0\n" - "1:\n" - : "=r" (__res_a0) - : "r"(fn), "r"(__stack), "r"(__flags), "r"(arg), - "r"(__ptid), "r"(__tls), "r"(__ctid), - "i"(__NR_clone), "i"(__NR_exit) - : "a7", LSS_SYSCALL_CLOBBERS); - __res = __res_a0; - } - LSS_RETURN(int, __res); - } - - #endif - #define __NR__exit __NR_exit - #define __NR__gettid __NR_gettid - #define __NR__mremap __NR_mremap - LSS_INLINE _syscall1(void *, brk, void *, e) - LSS_INLINE _syscall1(int, chdir, const char *,p) - LSS_INLINE _syscall1(int, close, int, f) - LSS_INLINE _syscall2(int, clock_getres, int, c, - struct kernel_timespec*, t) - LSS_INLINE _syscall2(int, clock_gettime, int, c, - struct kernel_timespec*, t) - LSS_INLINE _syscall1(int, dup, int, f) - #if defined(__NR_dup2) - // dup2 is polyfilled below when not available. - LSS_INLINE _syscall2(int, dup2, int, s, - int, d) - #endif - #if defined(__NR_dup3) - LSS_INLINE _syscall3(int, dup3, int, s, int, d, int, f) - #endif - LSS_INLINE _syscall3(int, execve, const char*, f, - const char*const*,a,const char*const*, e) - LSS_INLINE _syscall1(int, _exit, int, e) - LSS_INLINE _syscall1(int, exit_group, int, e) - LSS_INLINE _syscall3(int, fcntl, int, f, - int, c, long, a) - #if defined(__NR_fork) - // fork is polyfilled below when not available. - LSS_INLINE _syscall0(pid_t, fork) - #endif - #if defined(__NR_fstat) - LSS_INLINE _syscall2(int, fstat, int, f, - struct kernel_stat*, b) - #endif - LSS_INLINE _syscall2(int, fstatfs, int, f, - struct kernel_statfs*, b) - #if defined(__x86_64__) - /* Need to make sure off_t isn't truncated to 32-bits under x32. */ - LSS_INLINE int LSS_NAME(ftruncate)(int f, off_t l) { - LSS_BODY(2, int, ftruncate, LSS_SYSCALL_ARG(f), (uint64_t)(l)); - } - #else - LSS_INLINE _syscall2(int, ftruncate, int, f, - off_t, l) - #endif - LSS_INLINE _syscall6(int, futex, int*, u, - int, o, int, v, - struct kernel_timespec*, t, - int*, u2, int, v2) - LSS_INLINE _syscall3(int, getdents, int, f, - struct kernel_dirent*, d, int, c) - LSS_INLINE _syscall3(int, getdents64, int, f, - struct kernel_dirent64*, d, int, c) - LSS_INLINE _syscall0(gid_t, getegid) - LSS_INLINE _syscall0(uid_t, geteuid) - LSS_INLINE _syscall2(int, getitimer, int, w, - struct kernel_itimerval*, c) - #if defined(__NR_getpgrp) - LSS_INLINE _syscall0(pid_t, getpgrp) - #endif - LSS_INLINE _syscall0(pid_t, getpid) - LSS_INLINE _syscall0(pid_t, getppid) - LSS_INLINE _syscall2(int, getpriority, int, a, - int, b) - LSS_INLINE _syscall3(int, getresgid, gid_t *, r, - gid_t *, e, gid_t *, s) - LSS_INLINE _syscall3(int, getresuid, uid_t *, r, - uid_t *, e, uid_t *, s) - #if defined(__NR_getrlimit) - LSS_INLINE _syscall2(int, getrlimit, int, r, - struct kernel_rlimit*, l) - #endif - LSS_INLINE _syscall1(pid_t, getsid, pid_t, p) - LSS_INLINE _syscall0(pid_t, _gettid) - LSS_INLINE _syscall2(pid_t, gettimeofday, struct kernel_timeval*, t, - void*, tz) - LSS_INLINE _syscall5(int, setxattr, const char *,p, - const char *, n, const void *,v, - size_t, s, int, f) - LSS_INLINE _syscall5(int, lsetxattr, const char *,p, - const char *, n, const void *,v, - size_t, s, int, f) - LSS_INLINE _syscall4(ssize_t, getxattr, const char *,p, - const char *, n, void *, v, size_t, s) - LSS_INLINE _syscall4(ssize_t, lgetxattr, const char *,p, - const char *, n, void *, v, size_t, s) - LSS_INLINE _syscall3(ssize_t, listxattr, const char *,p, - char *, l, size_t, s) - LSS_INLINE _syscall3(ssize_t, llistxattr, const char *,p, - char *, l, size_t, s) - LSS_INLINE _syscall3(int, ioctl, int, d, - int, r, void *, a) - LSS_INLINE _syscall2(int, ioprio_get, int, which, - int, who) - LSS_INLINE _syscall3(int, ioprio_set, int, which, - int, who, int, ioprio) - LSS_INLINE _syscall2(int, kill, pid_t, p, - int, s) - #if defined(__x86_64__) - /* Need to make sure off_t isn't truncated to 32-bits under x32. */ - LSS_INLINE off_t LSS_NAME(lseek)(int f, off_t o, int w) { - _LSS_BODY(3, off_t, lseek, off_t, LSS_SYSCALL_ARG(f), (uint64_t)(o), - LSS_SYSCALL_ARG(w)); - } - #else - LSS_INLINE _syscall3(off_t, lseek, int, f, - off_t, o, int, w) - #endif - LSS_INLINE _syscall2(int, munmap, void*, s, - size_t, l) - LSS_INLINE _syscall6(long, move_pages, pid_t, p, - unsigned long, n, void **,g, int *, d, - int *, s, int, f) - LSS_INLINE _syscall3(int, mprotect, const void *,a, - size_t, l, int, p) - LSS_INLINE _syscall5(void*, _mremap, void*, o, - size_t, os, size_t, ns, - unsigned long, f, void *, a) - #if defined(__NR_open) - // open is polyfilled below when not available. - LSS_INLINE _syscall3(int, open, const char*, p, - int, f, int, m) - #endif - #if defined(__NR_poll) - // poll is polyfilled below when not available. - LSS_INLINE _syscall3(int, poll, struct kernel_pollfd*, u, - unsigned int, n, int, t) - #endif - #if defined(__NR_ppoll) - LSS_INLINE _syscall5(int, ppoll, struct kernel_pollfd *, u, - unsigned int, n, const struct kernel_timespec *, t, - const struct kernel_sigset_t *, sigmask, size_t, s) - #endif - LSS_INLINE _syscall5(int, prctl, int, option, - unsigned long, arg2, - unsigned long, arg3, - unsigned long, arg4, - unsigned long, arg5) - LSS_INLINE _syscall4(long, ptrace, int, r, - pid_t, p, void *, a, void *, d) - #if defined(__NR_quotactl) - // Defined on x86_64 / i386 only - LSS_INLINE _syscall4(int, quotactl, int, cmd, const char *, special, - int, id, caddr_t, addr) - #endif - LSS_INLINE _syscall3(ssize_t, read, int, f, - void *, b, size_t, c) - #if defined(__NR_readlink) - // readlink is polyfilled below when not available. - LSS_INLINE _syscall3(int, readlink, const char*, p, - char*, b, size_t, s) - #endif - #if defined(__NR_readlinkat) - LSS_INLINE _syscall4(int, readlinkat, int, d, const char *, p, char *, b, - size_t, s) - #endif - LSS_INLINE _syscall4(int, rt_sigaction, int, s, - const struct kernel_sigaction*, a, - struct kernel_sigaction*, o, size_t, c) - LSS_INLINE _syscall2(int, rt_sigpending, struct kernel_sigset_t *, s, - size_t, c) - LSS_INLINE _syscall4(int, rt_sigprocmask, int, h, - const struct kernel_sigset_t*, s, - struct kernel_sigset_t*, o, size_t, c) - LSS_INLINE _syscall2(int, rt_sigsuspend, - const struct kernel_sigset_t*, s, size_t, c) - LSS_INLINE _syscall4(int, rt_sigtimedwait, const struct kernel_sigset_t*, s, - siginfo_t*, i, const struct timespec*, t, size_t, c) - LSS_INLINE _syscall3(int, sched_getaffinity,pid_t, p, - unsigned int, l, unsigned long *, m) - LSS_INLINE _syscall3(int, sched_setaffinity,pid_t, p, - unsigned int, l, unsigned long *, m) - LSS_INLINE _syscall0(int, sched_yield) - LSS_INLINE _syscall1(long, set_tid_address, int *, t) - LSS_INLINE _syscall1(int, setfsgid, gid_t, g) - LSS_INLINE _syscall1(int, setfsuid, uid_t, u) - LSS_INLINE _syscall1(int, setuid, uid_t, u) - LSS_INLINE _syscall1(int, setgid, gid_t, g) - LSS_INLINE _syscall3(int, setitimer, int, w, - const struct kernel_itimerval*, n, - struct kernel_itimerval*, o) - LSS_INLINE _syscall2(int, setpgid, pid_t, p, - pid_t, g) - LSS_INLINE _syscall3(int, setpriority, int, a, - int, b, int, p) - LSS_INLINE _syscall3(int, setresgid, gid_t, r, - gid_t, e, gid_t, s) - LSS_INLINE _syscall3(int, setresuid, uid_t, r, - uid_t, e, uid_t, s) - #if defined(__NR_setrlimit) - LSS_INLINE _syscall2(int, setrlimit, int, r, - const struct kernel_rlimit*, l) - #endif - LSS_INLINE _syscall0(pid_t, setsid) - LSS_INLINE _syscall2(int, sigaltstack, const stack_t*, s, - const stack_t*, o) - #if defined(__NR_sigreturn) - LSS_INLINE _syscall1(int, sigreturn, unsigned long, u) - #endif - #if defined(__NR_stat) - // stat and lstat are polyfilled below when not available. - LSS_INLINE _syscall2(int, stat, const char*, f, - struct kernel_stat*, b) - #endif - #if defined(__NR_lstat) - LSS_INLINE _syscall2(int, lstat, const char*, f, - struct kernel_stat*, b) - #endif - LSS_INLINE _syscall2(int, statfs, const char*, f, - struct kernel_statfs*, b) - LSS_INLINE _syscall3(int, tgkill, pid_t, p, - pid_t, t, int, s) - LSS_INLINE _syscall2(int, tkill, pid_t, p, - int, s) - #if defined(__NR_unlink) - // unlink is polyfilled below when not available. - LSS_INLINE _syscall1(int, unlink, const char*, f) - #endif - LSS_INLINE _syscall3(ssize_t, write, int, f, - const void *, b, size_t, c) - LSS_INLINE _syscall3(ssize_t, writev, int, f, - const struct kernel_iovec*, v, size_t, c) - #if defined(__NR_getcpu) - LSS_INLINE _syscall3(long, getcpu, unsigned *, cpu, - unsigned *, node, void *, unused) - #endif - #if defined(__NR_fadvise64) - #if defined(__x86_64__) - /* Need to make sure loff_t isn't truncated to 32-bits under x32. */ - LSS_INLINE int LSS_NAME(fadvise64)(int fd, loff_t offset, loff_t len, - int advice) { - LSS_BODY(4, int, fadvise64, LSS_SYSCALL_ARG(fd), (uint64_t)(offset), - (uint64_t)(len), LSS_SYSCALL_ARG(advice)); - } - #else - LSS_INLINE _syscall4(int, fadvise64, - int, fd, loff_t, offset, loff_t, len, int, advice) - #endif - #elif defined(__i386__) - #define __NR__fadvise64_64 __NR_fadvise64_64 - LSS_INLINE _syscall6(int, _fadvise64_64, int, fd, - unsigned, offset_lo, unsigned, offset_hi, - unsigned, len_lo, unsigned, len_hi, - int, advice) - - LSS_INLINE int LSS_NAME(fadvise64)(int fd, loff_t offset, - loff_t len, int advice) { - return LSS_NAME(_fadvise64_64)(fd, - (unsigned)offset, (unsigned)(offset >>32), - (unsigned)len, (unsigned)(len >> 32), - advice); - } - - #elif defined(__s390__) && !defined(__s390x__) - #define __NR__fadvise64_64 __NR_fadvise64_64 - struct kernel_fadvise64_64_args { - int fd; - long long offset; - long long len; - int advice; - }; - - LSS_INLINE _syscall1(int, _fadvise64_64, - struct kernel_fadvise64_64_args *args) - - LSS_INLINE int LSS_NAME(fadvise64)(int fd, loff_t offset, - loff_t len, int advice) { - struct kernel_fadvise64_64_args args = { fd, offset, len, advice }; - return LSS_NAME(_fadvise64_64)(&args); - } - #endif - #if defined(__NR_fallocate) - #if defined(__x86_64__) - /* Need to make sure loff_t isn't truncated to 32-bits under x32. */ - LSS_INLINE int LSS_NAME(fallocate)(int f, int mode, loff_t offset, - loff_t len) { - LSS_BODY(4, int, fallocate, LSS_SYSCALL_ARG(f), LSS_SYSCALL_ARG(mode), - (uint64_t)(offset), (uint64_t)(len)); - } - #elif (defined(__i386__) || (defined(__s390__) && !defined(__s390x__)) \ - || defined(__ARM_ARCH_3__) || defined(__ARM_EABI__) \ - || (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI32) \ - || defined(__PPC__)) - #define __NR__fallocate __NR_fallocate - LSS_INLINE _syscall6(int, _fallocate, int, fd, - int, mode, - unsigned, offset_lo, unsigned, offset_hi, - unsigned, len_lo, unsigned, len_hi) - - LSS_INLINE int LSS_NAME(fallocate)(int fd, int mode, - loff_t offset, loff_t len) { - union { loff_t off; unsigned w[2]; } o = { offset }, l = { len }; - return LSS_NAME(_fallocate)(fd, mode, o.w[0], o.w[1], l.w[0], l.w[1]); - } - #else - LSS_INLINE _syscall4(int, fallocate, - int, f, int, mode, loff_t, offset, loff_t, len) - #endif - #endif - #if defined(__NR_getrandom) - LSS_INLINE _syscall3(ssize_t, getrandom, void*, buffer, size_t, length, - unsigned int, flags) - #endif - #if defined(__NR_newfstatat) - LSS_INLINE _syscall4(int, newfstatat, int, d, - const char *, p, - struct kernel_stat*, b, int, f) - #endif - #if defined(__NR_statx) - LSS_INLINE _syscall5(int, statx, int, d, - const char *, p, - int, f, int, m, - struct kernel_statx*, b) - #endif - #if defined(__x86_64__) || defined(__s390x__) - LSS_INLINE int LSS_NAME(getresgid32)(gid_t *rgid, - gid_t *egid, - gid_t *sgid) { - return LSS_NAME(getresgid)(rgid, egid, sgid); - } - - LSS_INLINE int LSS_NAME(getresuid32)(uid_t *ruid, - uid_t *euid, - uid_t *suid) { - return LSS_NAME(getresuid)(ruid, euid, suid); - } - - LSS_INLINE int LSS_NAME(setfsgid32)(gid_t gid) { - return LSS_NAME(setfsgid)(gid); - } - - LSS_INLINE int LSS_NAME(setfsuid32)(uid_t uid) { - return LSS_NAME(setfsuid)(uid); - } - - LSS_INLINE int LSS_NAME(setresgid32)(gid_t rgid, gid_t egid, gid_t sgid) { - return LSS_NAME(setresgid)(rgid, egid, sgid); - } - - LSS_INLINE int LSS_NAME(setresuid32)(uid_t ruid, uid_t euid, uid_t suid) { - return LSS_NAME(setresuid)(ruid, euid, suid); - } - - LSS_INLINE int LSS_NAME(sigaction)(int signum, - const struct kernel_sigaction *act, - struct kernel_sigaction *oldact) { - #if defined(__x86_64__) - /* On x86_64, the kernel requires us to always set our own - * SA_RESTORER in order to be able to return from a signal handler. - * This function must have a "magic" signature that the "gdb" - * (and maybe the kernel?) can recognize. - */ - if (act != NULL && !(act->sa_flags & SA_RESTORER)) { - struct kernel_sigaction a = *act; - a.sa_flags |= SA_RESTORER; - a.sa_restorer = LSS_NAME(restore_rt)(); - return LSS_NAME(rt_sigaction)(signum, &a, oldact, - (KERNEL_NSIG+7)/8); - } else - #endif - return LSS_NAME(rt_sigaction)(signum, act, oldact, - (KERNEL_NSIG+7)/8); - } - LSS_INLINE int LSS_NAME(sigpending)(struct kernel_sigset_t *set) { - return LSS_NAME(rt_sigpending)(set, (KERNEL_NSIG+7)/8); - } - LSS_INLINE int LSS_NAME(sigsuspend)(const struct kernel_sigset_t *set) { - return LSS_NAME(rt_sigsuspend)(set, (KERNEL_NSIG+7)/8); - } - #endif - #if defined(__aarch64__) - LSS_INLINE int LSS_NAME(sigaction)(int signum, - const struct kernel_sigaction *act, - struct kernel_sigaction *oldact) { - /* On aarch64, the kernel requires us to always set our own - * SA_RESTORER in order to be able to return from a signal handler. - * This function must have a known "magic" instruction sequence - * that system softwares like a stack unwinder can recognize. - */ - if (act != NULL && !(act->sa_flags & SA_RESTORER)) { - struct kernel_sigaction a = *act; - a.sa_flags |= SA_RESTORER; - a.sa_restorer = LSS_NAME(restore_rt)(); - return LSS_NAME(rt_sigaction)(signum, &a, oldact, - (KERNEL_NSIG+7)/8); - } else - return LSS_NAME(rt_sigaction)(signum, act, oldact, - (KERNEL_NSIG+7)/8); - } - #endif - #if defined(__NR_rt_sigprocmask) - LSS_INLINE int LSS_NAME(sigprocmask)(int how, - const struct kernel_sigset_t *set, - struct kernel_sigset_t *oldset) { - return LSS_NAME(rt_sigprocmask)(how, set, oldset, (KERNEL_NSIG+7)/8); - } - #endif - #if defined(__NR_rt_sigtimedwait) - LSS_INLINE int LSS_NAME(sigtimedwait)(const struct kernel_sigset_t *set, - siginfo_t *info, - const struct timespec *timeout) { - return LSS_NAME(rt_sigtimedwait)(set, info, timeout, (KERNEL_NSIG+7)/8); - } - #endif - #if defined(__NR_wait4) - LSS_INLINE _syscall4(pid_t, wait4, pid_t, p, - int*, s, int, o, - struct kernel_rusage*, r) - #endif - #if defined(__NR_openat) - LSS_INLINE _syscall4(int, openat, int, d, const char *, p, int, f, int, m) - #endif - #if defined(__NR_unlinkat) - LSS_INLINE _syscall3(int, unlinkat, int, d, const char *, p, int, f) - #endif - #if defined(__i386__) || defined(__ARM_ARCH_3__) || defined(__ARM_EABI__) || \ - (defined(__s390__) && !defined(__s390x__)) - #define __NR__getresgid32 __NR_getresgid32 - #define __NR__getresuid32 __NR_getresuid32 - #define __NR__setfsgid32 __NR_setfsgid32 - #define __NR__setfsuid32 __NR_setfsuid32 - #define __NR__setresgid32 __NR_setresgid32 - #define __NR__setresuid32 __NR_setresuid32 -#if defined(__ARM_EABI__) - LSS_INLINE _syscall2(int, ugetrlimit, int, r, - struct kernel_rlimit*, l) -#endif - LSS_INLINE _syscall3(int, _getresgid32, gid_t *, r, - gid_t *, e, gid_t *, s) - LSS_INLINE _syscall3(int, _getresuid32, uid_t *, r, - uid_t *, e, uid_t *, s) - LSS_INLINE _syscall1(int, _setfsgid32, gid_t, f) - LSS_INLINE _syscall1(int, _setfsuid32, uid_t, f) - LSS_INLINE _syscall3(int, _setresgid32, gid_t, r, - gid_t, e, gid_t, s) - LSS_INLINE _syscall3(int, _setresuid32, uid_t, r, - uid_t, e, uid_t, s) - - LSS_INLINE int LSS_NAME(getresgid32)(gid_t *rgid, - gid_t *egid, - gid_t *sgid) { - int rc; - if ((rc = LSS_NAME(_getresgid32)(rgid, egid, sgid)) < 0 && - LSS_ERRNO == ENOSYS) { - if ((rgid == NULL) || (egid == NULL) || (sgid == NULL)) { - return EFAULT; - } - // Clear the high bits first, since getresgid only sets 16 bits - *rgid = *egid = *sgid = 0; - rc = LSS_NAME(getresgid)(rgid, egid, sgid); - } - return rc; - } - - LSS_INLINE int LSS_NAME(getresuid32)(uid_t *ruid, - uid_t *euid, - uid_t *suid) { - int rc; - if ((rc = LSS_NAME(_getresuid32)(ruid, euid, suid)) < 0 && - LSS_ERRNO == ENOSYS) { - if ((ruid == NULL) || (euid == NULL) || (suid == NULL)) { - return EFAULT; - } - // Clear the high bits first, since getresuid only sets 16 bits - *ruid = *euid = *suid = 0; - rc = LSS_NAME(getresuid)(ruid, euid, suid); - } - return rc; - } - - LSS_INLINE int LSS_NAME(setfsgid32)(gid_t gid) { - int rc; - if ((rc = LSS_NAME(_setfsgid32)(gid)) < 0 && - LSS_ERRNO == ENOSYS) { - if ((unsigned int)gid & ~0xFFFFu) { - rc = EINVAL; - } else { - rc = LSS_NAME(setfsgid)(gid); - } - } - return rc; - } - - LSS_INLINE int LSS_NAME(setfsuid32)(uid_t uid) { - int rc; - if ((rc = LSS_NAME(_setfsuid32)(uid)) < 0 && - LSS_ERRNO == ENOSYS) { - if ((unsigned int)uid & ~0xFFFFu) { - rc = EINVAL; - } else { - rc = LSS_NAME(setfsuid)(uid); - } - } - return rc; - } - - LSS_INLINE int LSS_NAME(setresgid32)(gid_t rgid, gid_t egid, gid_t sgid) { - int rc; - if ((rc = LSS_NAME(_setresgid32)(rgid, egid, sgid)) < 0 && - LSS_ERRNO == ENOSYS) { - if ((unsigned int)rgid & ~0xFFFFu || - (unsigned int)egid & ~0xFFFFu || - (unsigned int)sgid & ~0xFFFFu) { - rc = EINVAL; - } else { - rc = LSS_NAME(setresgid)(rgid, egid, sgid); - } - } - return rc; - } - - LSS_INLINE int LSS_NAME(setresuid32)(uid_t ruid, uid_t euid, uid_t suid) { - int rc; - if ((rc = LSS_NAME(_setresuid32)(ruid, euid, suid)) < 0 && - LSS_ERRNO == ENOSYS) { - if ((unsigned int)ruid & ~0xFFFFu || - (unsigned int)euid & ~0xFFFFu || - (unsigned int)suid & ~0xFFFFu) { - rc = EINVAL; - } else { - rc = LSS_NAME(setresuid)(ruid, euid, suid); - } - } - return rc; - } - #endif - LSS_INLINE int LSS_NAME(sigemptyset)(struct kernel_sigset_t *set) { - memset(&set->sig, 0, sizeof(set->sig)); - return 0; - } - - LSS_INLINE int LSS_NAME(sigfillset)(struct kernel_sigset_t *set) { - memset(&set->sig, -1, sizeof(set->sig)); - return 0; - } - - LSS_INLINE int LSS_NAME(sigaddset)(struct kernel_sigset_t *set, - int signum) { - if (signum < 1 || (size_t)signum > (8*sizeof(set->sig))) { - LSS_ERRNO = EINVAL; - return -1; - } else { - set->sig[(size_t)(signum - 1)/(8*sizeof(set->sig[0]))] - |= 1UL << ((size_t)(signum - 1) % (8*sizeof(set->sig[0]))); - return 0; - } - } - - LSS_INLINE int LSS_NAME(sigdelset)(struct kernel_sigset_t *set, - int signum) { - if (signum < 1 || (size_t)signum > (8*sizeof(set->sig))) { - LSS_ERRNO = EINVAL; - return -1; - } else { - set->sig[(size_t)(signum - 1)/(8*sizeof(set->sig[0]))] - &= ~(1UL << ((size_t)(signum - 1) % (8*sizeof(set->sig[0])))); - return 0; - } - } - - LSS_INLINE int LSS_NAME(sigismember)(struct kernel_sigset_t *set, - int signum) { - if (signum < 1 || (size_t)signum > (8*sizeof(set->sig))) { - LSS_ERRNO = EINVAL; - return -1; - } else { - return !!(set->sig[(size_t)(signum - 1)/(8*sizeof(set->sig[0]))] & - (1UL << ((size_t)(signum - 1) % (8*sizeof(set->sig[0]))))); - } - } - #if defined(__i386__) || \ - defined(__ARM_ARCH_3__) || defined(__ARM_EABI__) || \ - (defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI32) || \ - defined(__PPC__) || \ - (defined(__s390__) && !defined(__s390x__)) || defined(__e2k__) - #define __NR__sigaction __NR_sigaction - #define __NR__sigpending __NR_sigpending - #define __NR__sigsuspend __NR_sigsuspend - #define __NR__socketcall __NR_socketcall - LSS_INLINE _syscall2(int, fstat64, int, f, - struct kernel_stat64 *, b) - LSS_INLINE _syscall5(int, _llseek, uint, fd, - unsigned long, hi, unsigned long, lo, - loff_t *, res, uint, wh) -#if defined(__s390__) && !defined(__s390x__) - /* On s390, mmap2() arguments are passed in memory. */ - LSS_INLINE void* LSS_NAME(_mmap2)(void *s, size_t l, int p, int f, int d, - off_t o) { - unsigned long buf[6] = { (unsigned long) s, (unsigned long) l, - (unsigned long) p, (unsigned long) f, - (unsigned long) d, (unsigned long) o }; - LSS_REG(2, buf); - LSS_BODY(void*, mmap2, "0"(__r2)); - } -#elif defined(__NR_mmap2) - #define __NR__mmap2 __NR_mmap2 - LSS_INLINE _syscall6(void*, _mmap2, void*, s, - size_t, l, int, p, - int, f, int, d, - off_t, o) -#endif - LSS_INLINE _syscall3(int, _sigaction, int, s, - const struct kernel_old_sigaction*, a, - struct kernel_old_sigaction*, o) - LSS_INLINE _syscall1(int, _sigpending, unsigned long*, s) - #ifdef __PPC__ - LSS_INLINE _syscall1(int, _sigsuspend, unsigned long, s) - #else - LSS_INLINE _syscall3(int, _sigsuspend, const void*, a, - int, b, - unsigned long, s) - #endif - LSS_INLINE _syscall2(int, stat64, const char *, p, - struct kernel_stat64 *, b) - - LSS_INLINE int LSS_NAME(sigaction)(int signum, - const struct kernel_sigaction *act, - struct kernel_sigaction *oldact) { - int old_errno = LSS_ERRNO; - int rc; - struct kernel_sigaction a; - if (act != NULL) { - a = *act; - #ifdef __i386__ - /* On i386, the kernel requires us to always set our own - * SA_RESTORER when using realtime signals. Otherwise, it does not - * know how to return from a signal handler. This function must have - * a "magic" signature that the "gdb" (and maybe the kernel?) can - * recognize. - * Apparently, a SA_RESTORER is implicitly set by the kernel, when - * using non-realtime signals. - * - * TODO: Test whether ARM needs a restorer - */ - if (!(a.sa_flags & SA_RESTORER)) { - a.sa_flags |= SA_RESTORER; - a.sa_restorer = (a.sa_flags & SA_SIGINFO) - ? LSS_NAME(restore_rt)() : LSS_NAME(restore)(); - } - #endif - } - rc = LSS_NAME(rt_sigaction)(signum, act ? &a : act, oldact, - (KERNEL_NSIG+7)/8); - if (rc < 0 && LSS_ERRNO == ENOSYS) { - struct kernel_old_sigaction oa, ooa, *ptr_a = &oa, *ptr_oa = &ooa; - if (!act) { - ptr_a = NULL; - } else { - oa.sa_handler_ = act->sa_handler_; - memcpy(&oa.sa_mask, &act->sa_mask, sizeof(oa.sa_mask)); - #ifndef __mips__ - oa.sa_restorer = act->sa_restorer; - #endif - oa.sa_flags = act->sa_flags; - } - if (!oldact) { - ptr_oa = NULL; - } - LSS_ERRNO = old_errno; - rc = LSS_NAME(_sigaction)(signum, ptr_a, ptr_oa); - if (rc == 0 && oldact) { - if (act) { - memcpy(oldact, act, sizeof(*act)); - } else { - memset(oldact, 0, sizeof(*oldact)); - } - oldact->sa_handler_ = ptr_oa->sa_handler_; - oldact->sa_flags = ptr_oa->sa_flags; - memcpy(&oldact->sa_mask, &ptr_oa->sa_mask, sizeof(ptr_oa->sa_mask)); - #ifndef __mips__ - oldact->sa_restorer = ptr_oa->sa_restorer; - #endif - } - } - return rc; - } - - LSS_INLINE int LSS_NAME(sigpending)(struct kernel_sigset_t *set) { - int old_errno = LSS_ERRNO; - int rc = LSS_NAME(rt_sigpending)(set, (KERNEL_NSIG+7)/8); - if (rc < 0 && LSS_ERRNO == ENOSYS) { - LSS_ERRNO = old_errno; - LSS_NAME(sigemptyset)(set); - rc = LSS_NAME(_sigpending)(&set->sig[0]); - } - return rc; - } - - LSS_INLINE int LSS_NAME(sigsuspend)(const struct kernel_sigset_t *set) { - int olderrno = LSS_ERRNO; - int rc = LSS_NAME(rt_sigsuspend)(set, (KERNEL_NSIG+7)/8); - if (rc < 0 && LSS_ERRNO == ENOSYS) { - LSS_ERRNO = olderrno; - rc = LSS_NAME(_sigsuspend)( - #ifndef __PPC__ - set, 0, - #endif - set->sig[0]); - } - return rc; - } - #endif - #if defined(__s390x__) - /* On s390x, mmap() arguments are passed in memory. */ - LSS_INLINE void* LSS_NAME(mmap)(void *s, size_t l, int p, int f, int d, - int64_t o) { - unsigned long buf[6] = { (unsigned long) s, (unsigned long) l, - (unsigned long) p, (unsigned long) f, - (unsigned long) d, (unsigned long) o }; - LSS_REG(2, buf); - LSS_BODY(void*, mmap, "0"(__r2)); - } - #elif defined(__x86_64__) - /* Need to make sure __off64_t isn't truncated to 32-bits under x32. */ - LSS_INLINE void* LSS_NAME(mmap)(void *s, size_t l, int p, int f, int d, - int64_t o) { - LSS_BODY(6, void*, mmap, LSS_SYSCALL_ARG(s), LSS_SYSCALL_ARG(l), - LSS_SYSCALL_ARG(p), LSS_SYSCALL_ARG(f), - LSS_SYSCALL_ARG(d), (uint64_t)(o)); - } - #elif defined(__NR_mmap2) - /* On these architectures, implement mmap() with mmap2(). */ - LSS_INLINE void* LSS_NAME(mmap)(void *s, size_t l, int p, int f, int d, - int64_t o) { - if (o % 4096) { - LSS_ERRNO = EINVAL; - return (void *) -1; - } - return LSS_NAME(_mmap2)(s, l, p, f, d, (o / 4096)); - } - #else - /* Remaining 64-bit architectures. */ - LSS_INLINE _syscall6(void*, mmap, void*, addr, size_t, length, int, prot, - int, flags, int, fd, int64_t, offset) - #endif - #if defined(__PPC__) - #undef LSS_SC_LOADARGS_0 - #define LSS_SC_LOADARGS_0(dummy...) - #undef LSS_SC_LOADARGS_1 - #define LSS_SC_LOADARGS_1(arg1) \ - __sc_4 = (unsigned long) (arg1) - #undef LSS_SC_LOADARGS_2 - #define LSS_SC_LOADARGS_2(arg1, arg2) \ - LSS_SC_LOADARGS_1(arg1); \ - __sc_5 = (unsigned long) (arg2) - #undef LSS_SC_LOADARGS_3 - #define LSS_SC_LOADARGS_3(arg1, arg2, arg3) \ - LSS_SC_LOADARGS_2(arg1, arg2); \ - __sc_6 = (unsigned long) (arg3) - #undef LSS_SC_LOADARGS_4 - #define LSS_SC_LOADARGS_4(arg1, arg2, arg3, arg4) \ - LSS_SC_LOADARGS_3(arg1, arg2, arg3); \ - __sc_7 = (unsigned long) (arg4) - #undef LSS_SC_LOADARGS_5 - #define LSS_SC_LOADARGS_5(arg1, arg2, arg3, arg4, arg5) \ - LSS_SC_LOADARGS_4(arg1, arg2, arg3, arg4); \ - __sc_8 = (unsigned long) (arg5) - #undef LSS_SC_BODY - #define LSS_SC_BODY(nr, type, opt, args...) \ - long __sc_ret, __sc_err; \ - { \ - register unsigned long __sc_0 __asm__ ("r0") = __NR_socketcall; \ - register unsigned long __sc_3 __asm__ ("r3") = opt; \ - register unsigned long __sc_4 __asm__ ("r4"); \ - register unsigned long __sc_5 __asm__ ("r5"); \ - register unsigned long __sc_6 __asm__ ("r6"); \ - register unsigned long __sc_7 __asm__ ("r7"); \ - register unsigned long __sc_8 __asm__ ("r8"); \ - LSS_SC_LOADARGS_##nr(args); \ - __asm__ __volatile__ \ - ("stwu 1, -48(1)\n\t" \ - "stw 4, 20(1)\n\t" \ - "stw 5, 24(1)\n\t" \ - "stw 6, 28(1)\n\t" \ - "stw 7, 32(1)\n\t" \ - "stw 8, 36(1)\n\t" \ - "addi 4, 1, 20\n\t" \ - "sc\n\t" \ - "mfcr %0" \ - : "=&r" (__sc_0), \ - "=&r" (__sc_3), "=&r" (__sc_4), \ - "=&r" (__sc_5), "=&r" (__sc_6), \ - "=&r" (__sc_7), "=&r" (__sc_8) \ - : LSS_ASMINPUT_##nr \ - : "cr0", "ctr", "memory"); \ - __sc_ret = __sc_3; \ - __sc_err = __sc_0; \ - } \ - LSS_RETURN(type, __sc_ret, __sc_err) - - LSS_INLINE ssize_t LSS_NAME(recvmsg)(int s,struct kernel_msghdr *msg, - int flags){ - LSS_SC_BODY(3, ssize_t, 17, s, msg, flags); - } - - LSS_INLINE ssize_t LSS_NAME(sendmsg)(int s, - const struct kernel_msghdr *msg, - int flags) { - LSS_SC_BODY(3, ssize_t, 16, s, msg, flags); - } - - // TODO(csilvers): why is this ifdef'ed out? -#if 0 - LSS_INLINE ssize_t LSS_NAME(sendto)(int s, const void *buf, size_t len, - int flags, - const struct kernel_sockaddr *to, - unsigned int tolen) { - LSS_BODY(6, ssize_t, 11, s, buf, len, flags, to, tolen); - } -#endif - - LSS_INLINE int LSS_NAME(shutdown)(int s, int how) { - LSS_SC_BODY(2, int, 13, s, how); - } - - LSS_INLINE int LSS_NAME(socket)(int domain, int type, int protocol) { - LSS_SC_BODY(3, int, 1, domain, type, protocol); - } - - LSS_INLINE int LSS_NAME(socketpair)(int d, int type, int protocol, - int sv[2]) { - LSS_SC_BODY(4, int, 8, d, type, protocol, sv); - } - #endif - #if defined(__NR_recvmsg) - LSS_INLINE _syscall3(ssize_t, recvmsg, int, s, struct kernel_msghdr*, msg, - int, flags) - #endif - #if defined(__NR_sendmsg) - LSS_INLINE _syscall3(ssize_t, sendmsg, int, s, const struct kernel_msghdr*, - msg, int, flags) - #endif - #if defined(__NR_sendto) - LSS_INLINE _syscall6(ssize_t, sendto, int, s, const void*, buf, size_t,len, - int, flags, const struct kernel_sockaddr*, to, - unsigned int, tolen) - #endif - #if defined(__NR_shutdown) - LSS_INLINE _syscall2(int, shutdown, int, s, int, how) - #endif - #if defined(__NR_socket) - LSS_INLINE _syscall3(int, socket, int, domain, int, type, int, protocol) - #endif - #if defined(__NR_socketpair) - LSS_INLINE _syscall4(int, socketpair, int, d, int, type, int, protocol, - int*, sv) - #endif - - #if defined(__NR_socketcall) - LSS_INLINE _syscall2(int, _socketcall, int, c, - va_list, a) - LSS_INLINE int LSS_NAME(socketcall)(int op, ...) { - int rc; - va_list ap; - va_start(ap, op); - rc = LSS_NAME(_socketcall)(op, ap); - va_end(ap); - return rc; - } - - # if !defined(__NR_recvmsg) - LSS_INLINE ssize_t LSS_NAME(recvmsg)(int s,struct kernel_msghdr *msg, - int flags){ - return (ssize_t)LSS_NAME(socketcall)(17, s, msg, flags); - } - # endif - # if !defined(__NR_sendmsg) - LSS_INLINE ssize_t LSS_NAME(sendmsg)(int s, - const struct kernel_msghdr *msg, - int flags) { - return (ssize_t)LSS_NAME(socketcall)(16, s, msg, flags); - } - # endif - # if !defined(__NR_sendto) - LSS_INLINE ssize_t LSS_NAME(sendto)(int s, const void *buf, size_t len, - int flags, - const struct kernel_sockaddr *to, - unsigned int tolen) { - return (ssize_t)LSS_NAME(socketcall)(11, s, buf, len, flags, to, tolen); - } - # endif - # if !defined(__NR_shutdown) - LSS_INLINE int LSS_NAME(shutdown)(int s, int how) { - return LSS_NAME(socketcall)(13, s, how); - } - # endif - # if !defined(__NR_socket) - LSS_INLINE int LSS_NAME(socket)(int domain, int type, int protocol) { - return LSS_NAME(socketcall)(1, domain, type, protocol); - } - # endif - # if !defined(__NR_socketpair) - LSS_INLINE int LSS_NAME(socketpair)(int d, int type, int protocol, - int sv[2]) { - return LSS_NAME(socketcall)(8, d, type, protocol, sv); - } - # endif - #endif - #if defined(__NR_fstatat64) - LSS_INLINE _syscall4(int, fstatat64, int, d, - const char *, p, - struct kernel_stat64 *, b, int, f) - #endif - #if defined(__NR_waitpid) - // waitpid is polyfilled below when not available. - LSS_INLINE _syscall3(pid_t, waitpid, pid_t, p, - int*, s, int, o) - #endif - #if defined(__mips__) - /* sys_pipe() on MIPS has non-standard calling conventions, as it returns - * both file handles through CPU registers. - */ - LSS_INLINE int LSS_NAME(pipe)(int *p) { - register unsigned long __v0 __asm__("$2") = __NR_pipe; - register unsigned long __v1 __asm__("$3"); - register unsigned long __r7 __asm__("$7"); - __asm__ __volatile__ ("syscall\n" - : "=r"(__v0), "=r"(__v1), "=r" (__r7) - : "0"(__v0) - : "$8", "$9", "$10", "$11", "$12", - "$13", "$14", "$15", "$24", "$25", "memory"); - if (__r7) { - unsigned long __errnovalue = __v0; - LSS_ERRNO = __errnovalue; - return -1; - } else { - p[0] = __v0; - p[1] = __v1; - return 0; - } - } - #elif defined(__NR_pipe) - // pipe is polyfilled below when not available. - LSS_INLINE _syscall1(int, pipe, int *, p) - #endif - #if defined(__NR_pipe2) - LSS_INLINE _syscall2(int, pipe2, int *, pipefd, int, flags) - #endif - /* TODO(csilvers): see if ppc can/should support this as well */ - #if defined(__i386__) || defined(__ARM_ARCH_3__) || \ - defined(__ARM_EABI__) || \ - (defined(__mips__) && _MIPS_SIM != _MIPS_SIM_ABI64) || \ - (defined(__s390__) && !defined(__s390x__)) - #define __NR__statfs64 __NR_statfs64 - #define __NR__fstatfs64 __NR_fstatfs64 - LSS_INLINE _syscall3(int, _statfs64, const char*, p, - size_t, s,struct kernel_statfs64*, b) - LSS_INLINE _syscall3(int, _fstatfs64, int, f, - size_t, s,struct kernel_statfs64*, b) - LSS_INLINE int LSS_NAME(statfs64)(const char *p, - struct kernel_statfs64 *b) { - return LSS_NAME(_statfs64)(p, sizeof(*b), b); - } - LSS_INLINE int LSS_NAME(fstatfs64)(int f,struct kernel_statfs64 *b) { - return LSS_NAME(_fstatfs64)(f, sizeof(*b), b); - } - #endif - - LSS_INLINE int LSS_NAME(execv)(const char *path, const char *const argv[]) { - extern char **environ; - return LSS_NAME(execve)(path, argv, (const char *const *)environ); - } - - LSS_INLINE pid_t LSS_NAME(gettid)(void) { - pid_t tid = LSS_NAME(_gettid)(); - if (tid != -1) { - return tid; - } - return LSS_NAME(getpid)(); - } - - LSS_INLINE void *LSS_NAME(mremap)(void *old_address, size_t old_size, - size_t new_size, int flags, ...) { - va_list ap; - void *new_address, *rc; - va_start(ap, flags); - new_address = va_arg(ap, void *); - rc = LSS_NAME(_mremap)(old_address, old_size, new_size, - (unsigned long)flags, new_address); - va_end(ap); - return rc; - } - - LSS_INLINE long LSS_NAME(ptrace_detach)(pid_t pid) { - /* PTRACE_DETACH can sometimes forget to wake up the tracee and it - * then sends job control signals to the real parent, rather than to - * the tracer. We reduce the risk of this happening by starting a - * whole new time slice, and then quickly sending a SIGCONT signal - * right after detaching from the tracee. - * - * We use tkill to ensure that we only issue a wakeup for the thread being - * detached. Large multi threaded apps can take a long time in the kernel - * processing SIGCONT. - */ - long rc; - int err; - LSS_NAME(sched_yield)(); - rc = LSS_NAME(ptrace)(PTRACE_DETACH, pid, (void *)0, (void *)0); - err = LSS_ERRNO; - LSS_NAME(tkill)(pid, SIGCONT); - /* Old systems don't have tkill */ - if (LSS_ERRNO == ENOSYS) - LSS_NAME(kill)(pid, SIGCONT); - LSS_ERRNO = err; - return rc; - } - - LSS_INLINE int LSS_NAME(raise)(int sig) { - return LSS_NAME(kill)(LSS_NAME(getpid)(), sig); - } - - LSS_INLINE int LSS_NAME(setpgrp)(void) { - return LSS_NAME(setpgid)(0, 0); - } - - #if defined(__x86_64__) - /* Need to make sure loff_t isn't truncated to 32-bits under x32. */ - LSS_INLINE ssize_t LSS_NAME(pread64)(int f, void *b, size_t c, loff_t o) { - LSS_BODY(4, ssize_t, pread64, LSS_SYSCALL_ARG(f), LSS_SYSCALL_ARG(b), - LSS_SYSCALL_ARG(c), (uint64_t)(o)); - } - - LSS_INLINE ssize_t LSS_NAME(pwrite64)(int f, const void *b, size_t c, - loff_t o) { - LSS_BODY(4, ssize_t, pwrite64, LSS_SYSCALL_ARG(f), LSS_SYSCALL_ARG(b), - LSS_SYSCALL_ARG(c), (uint64_t)(o)); - } - - LSS_INLINE int LSS_NAME(readahead)(int f, loff_t o, size_t c) { - LSS_BODY(3, int, readahead, LSS_SYSCALL_ARG(f), (uint64_t)(o), - LSS_SYSCALL_ARG(c)); - } - #elif defined(__mips__) && _MIPS_SIM == _MIPS_SIM_ABI64 - LSS_INLINE _syscall4(ssize_t, pread64, int, f, - void *, b, size_t, c, - loff_t, o) - LSS_INLINE _syscall4(ssize_t, pwrite64, int, f, - const void *, b, size_t, c, - loff_t, o) - LSS_INLINE _syscall3(int, readahead, int, f, - loff_t, o, unsigned, c) - #else - #define __NR__pread64 __NR_pread64 - #define __NR__pwrite64 __NR_pwrite64 - #define __NR__readahead __NR_readahead - #if defined(__ARM_EABI__) || defined(__mips__) - /* On ARM and MIPS, a 64-bit parameter has to be in an even-odd register - * pair. Hence these calls ignore their fourth argument (r3) so that their - * fifth and sixth make such a pair (r4,r5). - */ - #define LSS_LLARG_PAD 0, - LSS_INLINE _syscall6(ssize_t, _pread64, int, f, - void *, b, size_t, c, - unsigned, skip, unsigned, o1, unsigned, o2) - LSS_INLINE _syscall6(ssize_t, _pwrite64, int, f, - const void *, b, size_t, c, - unsigned, skip, unsigned, o1, unsigned, o2) - LSS_INLINE _syscall5(int, _readahead, int, f, - unsigned, skip, - unsigned, o1, unsigned, o2, size_t, c) - #else - #define LSS_LLARG_PAD - LSS_INLINE _syscall5(ssize_t, _pread64, int, f, - void *, b, size_t, c, unsigned, o1, - unsigned, o2) - LSS_INLINE _syscall5(ssize_t, _pwrite64, int, f, - const void *, b, size_t, c, unsigned, o1, - unsigned, o2) - LSS_INLINE _syscall4(int, _readahead, int, f, - unsigned, o1, unsigned, o2, size_t, c) - #endif - /* We force 64bit-wide parameters onto the stack, then access each - * 32-bit component individually. This guarantees that we build the - * correct parameters independent of the native byte-order of the - * underlying architecture. - */ - LSS_INLINE ssize_t LSS_NAME(pread64)(int fd, void *buf, size_t count, - loff_t off) { - union { loff_t off; unsigned arg[2]; } o = { off }; - return LSS_NAME(_pread64)(fd, buf, count, - LSS_LLARG_PAD o.arg[0], o.arg[1]); - } - LSS_INLINE ssize_t LSS_NAME(pwrite64)(int fd, const void *buf, - size_t count, loff_t off) { - union { loff_t off; unsigned arg[2]; } o = { off }; - return LSS_NAME(_pwrite64)(fd, buf, count, - LSS_LLARG_PAD o.arg[0], o.arg[1]); - } - LSS_INLINE int LSS_NAME(readahead)(int fd, loff_t off, size_t count) { - union { loff_t off; unsigned arg[2]; } o = { off }; - return LSS_NAME(_readahead)(fd, LSS_LLARG_PAD o.arg[0], o.arg[1], count); - } - #endif -#endif - -/* - * Polyfills for deprecated syscalls. - */ - -#if !defined(__NR_dup2) - LSS_INLINE int LSS_NAME(dup2)(int s, int d) { - return LSS_NAME(dup3)(s, d, 0); - } -#endif - -#if !defined(__NR_open) - LSS_INLINE int LSS_NAME(open)(const char *pathname, int flags, int mode) { - return LSS_NAME(openat)(AT_FDCWD, pathname, flags, mode); - } -#endif - -#if !defined(__NR_unlink) - LSS_INLINE int LSS_NAME(unlink)(const char *pathname) { - return LSS_NAME(unlinkat)(AT_FDCWD, pathname, 0); - } -#endif - -#if !defined(__NR_readlink) - LSS_INLINE int LSS_NAME(readlink)(const char *pathname, char *buffer, - size_t size) { - return LSS_NAME(readlinkat)(AT_FDCWD, pathname, buffer, size); - } -#endif - -#if !defined(__NR_pipe) - LSS_INLINE int LSS_NAME(pipe)(int *pipefd) { - return LSS_NAME(pipe2)(pipefd, 0); - } -#endif - -#if !defined(__NR_poll) - LSS_INLINE int LSS_NAME(poll)(struct kernel_pollfd *fds, unsigned int nfds, - int timeout) { - struct kernel_timespec timeout_ts; - struct kernel_timespec *timeout_ts_p = NULL; - - if (timeout >= 0) { - timeout_ts.tv_sec = timeout / 1000; - timeout_ts.tv_nsec = (timeout % 1000) * 1000000; - timeout_ts_p = &timeout_ts; - } - return LSS_NAME(ppoll)(fds, nfds, timeout_ts_p, NULL, 0); - } -#endif - -#if defined(__NR_statx) - /* copy the contents of kernel_statx to the kernel_stat structure. */ - LSS_INLINE void LSS_NAME(cp_stat_statx)(struct kernel_stat *to, - struct kernel_statx *from) { - memset(to, 0, sizeof(struct kernel_stat)); - to->st_dev = (kernel_dev_t)((from->stx_dev_minor & 0xff) | - ((from->stx_dev_major & 0xfff) << 8) | - ((from->stx_dev_minor & ~0xffu) << 12)); - to->st_rdev = (kernel_dev_t)((from->stx_rdev_minor & 0xff) | - ((from->stx_rdev_major & 0xfff) << 8) | - ((from->stx_rdev_minor & ~0xffu) << 12)); - to->st_ino = (kernel_ino_t)from->stx_ino; - to->st_mode = (kernel_mode_t)from->stx_mode; - to->st_nlink = (kernel_nlink_t)from->stx_nlink; - to->st_uid = (kernel_uid_t)from->stx_uid; - to->st_gid = (kernel_gid_t)from->stx_gid; - to->st_atime_ = (kernel_time_t)(from->stx_atime.tv_sec); - to->st_atime_nsec_ = from->stx_atime.tv_nsec; - to->st_mtime_ = (kernel_time_t)(from->stx_mtime.tv_sec); - to->st_mtime_nsec_ = from->stx_mtime.tv_nsec; - to->st_ctime_ = (kernel_time_t)(from->stx_ctime.tv_sec); - to->st_ctime_nsec_ = from->stx_ctime.tv_nsec; - to->st_size = (kernel_off_t)(from->stx_size); - to->st_blocks = (kernel_blkcnt_t)(from->stx_blocks); - to->st_blksize = (kernel_blksize_t)from->stx_blksize; - } -#endif - -#if !defined(__NR_fstat) - LSS_INLINE int LSS_NAME(fstat)(int fd, - struct kernel_stat *buf) { - #if defined(__NR_newfstatat) - return LSS_NAME(newfstatat)(fd, "", buf, AT_EMPTY_PATH); - #elif defined(__NR_statx) - struct kernel_statx stx; - int flags = AT_NO_AUTOMOUNT | AT_EMPTY_PATH; - int mask = STATX_BASIC_STATS; - int res = LSS_NAME(statx)(fd, "", flags, mask, &stx); - LSS_NAME(cp_stat_statx)(buf, &stx); - return res; - #endif - } -#endif - -#if !defined(__NR_stat) - LSS_INLINE int LSS_NAME(stat)(const char *pathname, - struct kernel_stat *buf) { - #if defined(__NR_newfstatat) - return LSS_NAME(newfstatat)(AT_FDCWD, pathname, buf, 0); - #elif defined(__NR_statx) - struct kernel_statx stx; - int flags = AT_NO_AUTOMOUNT | AT_STATX_SYNC_AS_STAT; - int mask = STATX_BASIC_STATS; - int res = LSS_NAME(statx)(AT_FDCWD, pathname, flags, mask, &stx); - LSS_NAME(cp_stat_statx)(buf, &stx); - return res; - #endif - } -#endif - -#if !defined(__NR_lstat) - LSS_INLINE int LSS_NAME(lstat)(const char *pathname, - struct kernel_stat *buf) { - #if defined(__NR_newfstatat) - return LSS_NAME(newfstatat)(AT_FDCWD, pathname, buf, AT_SYMLINK_NOFOLLOW); - #elif defined(__NR_statx) - struct kernel_statx stx; - int flags = AT_NO_AUTOMOUNT | AT_SYMLINK_NOFOLLOW; - int mask = STATX_BASIC_STATS; - int res = LSS_NAME(statx)(AT_FDCWD, pathname, flags, mask, &stx); - LSS_NAME(cp_stat_statx)(buf, &stx); - return res; - #endif - } -#endif - -#if !defined(__NR_waitpid) - LSS_INLINE pid_t LSS_NAME(waitpid)(pid_t pid, int *status, int options) { - return LSS_NAME(wait4)(pid, status, options, 0); - } -#endif - -#if !defined(__NR_fork) -// TODO: define this in an arch-independant way instead of inlining the clone -// syscall body. - -# if defined(__aarch64__) || defined(__riscv) || defined(__loongarch_lp64) - LSS_INLINE pid_t LSS_NAME(fork)(void) { - // No fork syscall on aarch64 - implement by means of the clone syscall. - // Note that this does not reset glibc's cached view of the PID/TID, so - // some glibc interfaces might go wrong in the forked subprocess. - int flags = SIGCHLD; - void *child_stack = NULL; - void *parent_tidptr = NULL; - void *newtls = NULL; - void *child_tidptr = NULL; - - LSS_REG(0, flags); - LSS_REG(1, child_stack); - LSS_REG(2, parent_tidptr); - LSS_REG(3, newtls); - LSS_REG(4, child_tidptr); - LSS_BODY(pid_t, clone, "r"(__r0), "r"(__r1), "r"(__r2), "r"(__r3), - "r"(__r4)); - } -# elif defined(__x86_64__) - LSS_INLINE pid_t LSS_NAME(fork)(void) { - // Android disallows the fork syscall on x86_64 - implement by means of the - // clone syscall as above for aarch64. - int flags = SIGCHLD; - void *child_stack = NULL; - void *parent_tidptr = NULL; - void *newtls = NULL; - void *child_tidptr = NULL; - - LSS_BODY(5, pid_t, clone, LSS_SYSCALL_ARG(flags), - LSS_SYSCALL_ARG(child_stack), LSS_SYSCALL_ARG(parent_tidptr), - LSS_SYSCALL_ARG(newtls), LSS_SYSCALL_ARG(child_tidptr)); - } -# else -# error missing fork polyfill for this architecture -# endif -#endif - -/* These restore the original values of these macros saved by the - * corresponding #pragma push_macro near the top of this file. */ -#pragma pop_macro("stat64") -#pragma pop_macro("fstat64") -#pragma pop_macro("lstat64") -#pragma pop_macro("pread64") -#pragma pop_macro("pwrite64") -#pragma pop_macro("getdents64") - -#if defined(__cplusplus) && !defined(SYS_CPLUSPLUS) -} -#endif - -#endif -#endif diff --git a/native/src/crt0/malloc.c b/native/src/crt0/malloc.c deleted file mode 100644 index 14d766157..000000000 --- a/native/src/crt0/malloc.c +++ /dev/null @@ -1,16 +0,0 @@ -// Configure dlmalloc - -#define USE_LOCKS 0 -#define INSECURE 1 -#define HAVE_MORECORE 0 -#define NO_MALLINFO 1 -#define NO_MALLOC_STATS 1 -#define LACKS_TIME_H 1 -#define malloc_getpagesize 4096 - -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wunused-but-set-variable" - -#include "dlmalloc/malloc.c" - -#pragma clang diagnostic pop diff --git a/native/src/crt0/mem.c b/native/src/crt0/mem.c deleted file mode 100644 index cc17d5f7c..000000000 --- a/native/src/crt0/mem.c +++ /dev/null @@ -1,361 +0,0 @@ -#include -#include - -void *memset(void *dst, int ch, size_t n) { - uint8_t *d = dst; - uint8_t c = ch; - while (n--) - (*d++) = c; - return dst; -} - -void *memmove(void *dst, const void *src, size_t n) { - return memcpy(dst, src, n); -} - -void *memcpy(void *dst, const void *src, size_t n) { - uint8_t *d = dst; - const uint8_t *s = src; - while (n--) - *d++ = *s++; - return dst; -} - -int memcmp(const void *lhs, const void *rhs, size_t n) { - const uint8_t *l = lhs; - const uint8_t *r = rhs; - while (n--) { - if (*l != *r) { - return *l - *r; - } else { - l++; - r++; - } - } - return 0; -} - -void *memchr(const void *ptr, int ch, size_t n) { - const uint8_t *p = ptr; - uint8_t c = ch; - while (n--) { - if (*p != c) - ++p; - else - return (void *) p; - } - return NULL; -} - -char *strchr(const char *s, int ch) { - char c = ch; - while (*s != c) - if (!*s++) - return NULL; - return (char *) s; -} - -int strcmp(const char *lhs, const char *rhs) { - while (*lhs && (*lhs == *rhs)) { - ++lhs; - ++rhs; - } - return *(uint8_t *)lhs - *(uint8_t *)rhs; -} - -size_t strlen(const char *str) { - size_t l = 0; - while (str[l]) - ++l; - return l; -} - -char *strcpy(char *restrict dest, const char *restrict src) { - char *ret = dest; - while ((*dest++ = *src++)) {} - return ret; -} - -char *strdup(const char *str) { - size_t siz; - char *copy; - siz = strlen(str) + 1; - if ((copy = malloc(siz)) == NULL) - return NULL; - memcpy(copy, str, siz); - return copy; -} - -// memmem source: bionic/libc/upstream-openbsd/lib/libc/string/memmem.c - -static char *twobyte_memmem(const unsigned char *h, size_t k, const unsigned char *n) { - uint16_t nw = n[0]<<8 | n[1], hw = h[0]<<8 | h[1]; - for (h+=2, k-=2; k; k--, hw = hw<<8 | *h++) - if (hw == nw) return (char *)h-2; - return hw == nw ? (char *)h-2 : 0; -} - -static char *threebyte_memmem(const unsigned char *h, size_t k, const unsigned char *n) { - uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8; - uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8; - for (h+=3, k-=3; k; k--, hw = (hw|*h++)<<8) - if (hw == nw) return (char *)h-3; - return hw == nw ? (char *)h-3 : 0; -} - -static char *fourbyte_memmem(const unsigned char *h, size_t k, const unsigned char *n) { - uint32_t nw = n[0]<<24 | n[1]<<16 | n[2]<<8 | n[3]; - uint32_t hw = h[0]<<24 | h[1]<<16 | h[2]<<8 | h[3]; - for (h+=4, k-=4; k; k--, hw = hw<<8 | *h++) - if (hw == nw) return (char *)h-4; - return hw == nw ? (char *)h-4 : 0; -} - -#define MAX(a,b) ((a)>(b)?(a):(b)) -#define MIN(a,b) ((a)<(b)?(a):(b)) - -#define BITOP(a,b,op) \ - ((a)[(size_t)(b)/(8*sizeof *(a))] op (size_t)1<<((size_t)(b)%(8*sizeof *(a)))) - -/* - * Maxime Crochemore and Dominique Perrin, Two-way string-matching, - * Journal of the ACM, 38(3):651-675, July 1991. - * - */ -static char *twoway_memmem( - const unsigned char *h, const unsigned char *z, const unsigned char *n, size_t l) { - size_t i, ip, jp, k, p, ms, p0, mem, mem0; - size_t byteset[32 / sizeof(size_t)] = { 0 }; - size_t shift[256]; - - /* Computing length of needle and fill shift table */ - for (i=0; i n[jp+k]) { - jp += k; - k = 1; - p = jp - ip; - } else { - ip = jp++; - k = p = 1; - } - } - ms = ip; - p0 = p; - - /* And with the opposite comparison */ - ip = -1; jp = 0; k = p = 1; - while (jp+k ms+1) ms = ip; - else p = p0; - - /* Periodic needle? */ - if (memcmp(n, n+p, ms+1)) { - mem0 = 0; - p = MAX(ms, l-ms-1) + 1; - } else mem0 = l-p; - mem = 0; - - /* Search loop */ - for (;;) { - /* If remainder of haystack is shorter than needle, done */ - if (z-h < l) return 0; - - /* Check last byte first; advance by shift on mismatch */ - if (BITOP(byteset, h[l-1], &)) { - k = l-shift[h[l-1]]; - if (k) { - if (k < mem) k = mem; - h += k; - mem = 0; - continue; - } - } else { - h += l; - mem = 0; - continue; - } - - /* Compare right half */ - for (k=MAX(ms+1,mem); kmem && n[k-1] == h[k-1]; k--); - if (k <= mem) return (char *)h; - h += p; - mem = mem0; - } -} - -void *memmem(const void *h0, size_t k, const void *n0, size_t l) { - const unsigned char *h = h0, *n = n0; - - /* Return immediately on empty needle */ - if (!l) return (void *)h; - - /* Return immediately when needle is longer than haystack */ - if (k -#include -#include -#include -#include -#include - -// Source: bionic/libc/upstream-openbsd/lib/libc/stdlib/getenv.c -static char *__findenv(const char *name, int len, int *offset) { - int i; - const char *np; - char **p, *cp; - - if (name == NULL || environ == NULL) - return (NULL); - for (p = environ + *offset; (cp = *p) != NULL; ++p) { - for (np = name, i = len; i && *cp; i--) - if (*cp++ != *np++) - break; - if (i == 0 && *cp++ == '=') { - *offset = p - environ; - return (cp); - } - } - return (NULL); -} - -// Source: bionic/libc/upstream-openbsd/lib/libc/stdlib/setenv.c -int setenv(const char *name, const char *value, int rewrite) { - static char **lastenv; - - char *C, **P; - const char *np; - int l_value, offset = 0; - - if (!name || !*name) { - errno = EINVAL; - return (-1); - } - for (np = name; *np && *np != '='; ++np) - ; - if (*np) { - errno = EINVAL; - return (-1); /* has `=' in name */ - } - - l_value = strlen(value); - if ((C = __findenv(name, (int)(np - name), &offset)) != NULL) { - int tmpoff = offset + 1; - if (!rewrite) - return (0); -#if 0 /* XXX - existing entry may not be writable */ - if (strlen(C) >= l_value) { /* old larger; copy over */ - while ((*C++ = *value++)) - ; - return (0); - } -#endif - /* could be set multiple times */ - while (__findenv(name, (int)(np - name), &tmpoff)) { - for (P = &environ[tmpoff];; ++P) - if (!(*P = *(P + 1))) - break; - } - } else { /* create new slot */ - size_t cnt = 0; - - if (environ != NULL) { - for (P = environ; *P != NULL; P++) - ; - cnt = P - environ; - } - size_t new_size; - if (__builtin_mul_overflow(cnt + 2, sizeof(char *), &new_size)) { - errno = ENOMEM; - return (-1); - } - P = realloc(lastenv, new_size); - if (!P) - return (-1); - if (lastenv != environ && environ != NULL) - memcpy(P, environ, cnt * sizeof(char *)); - lastenv = environ = P; - offset = cnt; - environ[cnt + 1] = NULL; - } - if (!(environ[offset] = /* name + `=' + value */ - malloc((int)(np - name) + l_value + 2))) - return (-1); - for (C = environ[offset]; (*C = *name++) && *C != '='; ++C) - ; - for (*C++ = '='; (*C++ = *value++); ) - ; - return (0); -} - -// Source: bionic/libc/bionic/libgen.cpp -static int __basename_r(const char *path, char* buffer, size_t buffer_size) { - const char *startp = NULL; - const char *endp = NULL; - int len; - int result; - - // Empty or NULL string gets treated as ".". - if (path == NULL || *path == '\0') { - startp = "."; - len = 1; - goto Exit; - } - - // Strip trailing slashes. - endp = path + strlen(path) - 1; - while (endp > path && *endp == '/') { - endp--; - } - - // All slashes becomes "/". - if (endp == path && *endp == '/') { - startp = "/"; - len = 1; - goto Exit; - } - - // Find the start of the base. - startp = endp; - while (startp > path && *(startp - 1) != '/') { - startp--; - } - - len = endp - startp +1; - -Exit: - result = len; - if (buffer == NULL) { - return result; - } - if (len > (int) buffer_size - 1) { - len = buffer_size - 1; - result = -1; - errno = ERANGE; - } - - if (len >= 0) { - memcpy(buffer, startp, len); - buffer[len] = 0; - } - return result; -} - -char *basename(const char *path) { - static char buf[4069]; - int rc = __basename_r(path, buf, sizeof(buf)); - return (rc < 0) ? NULL : buf; -} - -// Simply just abort when abort_message is called -void __wrap_abort_message(const char* format, ...) { - abort(); -} - -// Don't care about C++ global destructors -int __cxa_atexit(void (*func) (void *), void * arg, void * dso_handle) { - return 0; -} - -// Emulate pthread functions - -static pthread_key_t g_counter = 0; -static void **g_key_values = NULL; - -int pthread_key_create(pthread_key_t *key_ptr, void (*dtor)(void*)) { - *key_ptr = g_counter++; - g_key_values = realloc(g_key_values, g_counter * sizeof(void*)); - return 0; -} - -int pthread_key_delete(pthread_key_t key) { - if (key < g_counter) { - g_key_values[key] = NULL; - } - return 0; -} - -void *pthread_getspecific(pthread_key_t key) { - return key < g_counter ? g_key_values[key] : NULL; -} - -int pthread_setspecific(pthread_key_t key, const void *value) { - if (key < g_counter) { - g_key_values[key] = (void *) value; - } - return 0; -} - -// Workaround LTO bug: https://github.com/llvm/llvm-project/issues/61101 -#if defined(__i386__) -extern long *_GLOBAL_OFFSET_TABLE_; -long unused() { - return *_GLOBAL_OFFSET_TABLE_; -} -#endif diff --git a/native/src/crt0/nolibc.c b/native/src/crt0/nolibc.c deleted file mode 100644 index 9df3cbfa8..000000000 --- a/native/src/crt0/nolibc.c +++ /dev/null @@ -1,15 +0,0 @@ -#include "nolibc/crt.h" -#include "nolibc/arch.h" - -// errno - -static int g_errno = 0; - -int *__errno(void) { - return &g_errno; -} - -long __set_errno_internal(int n) { - g_errno = n; - return -1; -} diff --git a/native/src/crt0/nolibc/arch-aarch64.h b/native/src/crt0/nolibc/arch-aarch64.h deleted file mode 100644 index 73ff481fb..000000000 --- a/native/src/crt0/nolibc/arch-aarch64.h +++ /dev/null @@ -1,159 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ -/* - * AARCH64 specific definitions for NOLIBC - * Copyright (C) 2017-2022 Willy Tarreau - */ - -#ifndef _NOLIBC_ARCH_AARCH64_H -#define _NOLIBC_ARCH_AARCH64_H - -/* Syscalls for AARCH64 : - * - registers are 64-bit - * - stack is 16-byte aligned - * - syscall number is passed in x8 - * - arguments are in x0, x1, x2, x3, x4, x5 - * - the system call is performed by calling svc 0 - * - syscall return comes in x0. - * - the arguments are cast to long and assigned into the target registers - * which are then simply passed as registers to the asm code, so that we - * don't have to experience issues with register constraints. - * - * On aarch64, select() is not implemented so we have to use pselect6(). - */ -#define __ARCH_WANT_SYS_PSELECT6 - -#define my_syscall0(num) \ -({ \ - register long _num __asm__ ("x8") = (num); \ - register long _arg1 __asm__ ("x0"); \ - \ - __asm__ volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall1(num, arg1) \ -({ \ - register long _num __asm__ ("x8") = (num); \ - register long _arg1 __asm__ ("x0") = (long)(arg1); \ - \ - __asm__ volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_arg1), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall2(num, arg1, arg2) \ -({ \ - register long _num __asm__ ("x8") = (num); \ - register long _arg1 __asm__ ("x0") = (long)(arg1); \ - register long _arg2 __asm__ ("x1") = (long)(arg2); \ - \ - __asm__ volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_arg1), "r"(_arg2), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall3(num, arg1, arg2, arg3) \ -({ \ - register long _num __asm__ ("x8") = (num); \ - register long _arg1 __asm__ ("x0") = (long)(arg1); \ - register long _arg2 __asm__ ("x1") = (long)(arg2); \ - register long _arg3 __asm__ ("x2") = (long)(arg3); \ - \ - __asm__ volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall4(num, arg1, arg2, arg3, arg4) \ -({ \ - register long _num __asm__ ("x8") = (num); \ - register long _arg1 __asm__ ("x0") = (long)(arg1); \ - register long _arg2 __asm__ ("x1") = (long)(arg2); \ - register long _arg3 __asm__ ("x2") = (long)(arg3); \ - register long _arg4 __asm__ ("x3") = (long)(arg4); \ - \ - __asm__ volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall5(num, arg1, arg2, arg3, arg4, arg5) \ -({ \ - register long _num __asm__ ("x8") = (num); \ - register long _arg1 __asm__ ("x0") = (long)(arg1); \ - register long _arg2 __asm__ ("x1") = (long)(arg2); \ - register long _arg3 __asm__ ("x2") = (long)(arg3); \ - register long _arg4 __asm__ ("x3") = (long)(arg4); \ - register long _arg5 __asm__ ("x4") = (long)(arg5); \ - \ - __asm__ volatile ( \ - "svc #0\n" \ - : "=r" (_arg1) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall6(num, arg1, arg2, arg3, arg4, arg5, arg6) \ -({ \ - register long _num __asm__ ("x8") = (num); \ - register long _arg1 __asm__ ("x0") = (long)(arg1); \ - register long _arg2 __asm__ ("x1") = (long)(arg2); \ - register long _arg3 __asm__ ("x2") = (long)(arg3); \ - register long _arg4 __asm__ ("x3") = (long)(arg4); \ - register long _arg5 __asm__ ("x4") = (long)(arg5); \ - register long _arg6 __asm__ ("x5") = (long)(arg6); \ - \ - __asm__ volatile ( \ - "svc #0\n" \ - : "=r" (_arg1) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "r"(_arg6), "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#ifdef _NOLIBC_CRT_H - -/* startup code */ -void __attribute__((noreturn)) _start(void) -{ - __asm__ volatile ( - "mov x0, sp\n" /* save stack pointer to x0, as arg1 of _start_c */ - "and sp, x0, -16\n" /* sp must be 16-byte aligned in the callee */ - "bl _start_c\n" /* transfer to c runtime */ - ); - __builtin_unreachable(); -} - -#endif /* _NOLIBC_CRT_H */ - -#endif /* _NOLIBC_ARCH_AARCH64_H */ diff --git a/native/src/crt0/nolibc/arch-arm.h b/native/src/crt0/nolibc/arch-arm.h deleted file mode 100644 index 82a1a628b..000000000 --- a/native/src/crt0/nolibc/arch-arm.h +++ /dev/null @@ -1,200 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ -/* - * ARM specific definitions for NOLIBC - * Copyright (C) 2017-2022 Willy Tarreau - */ - -#ifndef _NOLIBC_ARCH_ARM_H -#define _NOLIBC_ARCH_ARM_H - -/* Syscalls for ARM in ARM or Thumb modes : - * - registers are 32-bit - * - stack is 8-byte aligned - * ( http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka4127.html) - * - syscall number is passed in r7 - * - arguments are in r0, r1, r2, r3, r4, r5 - * - the system call is performed by calling svc #0 - * - syscall return comes in r0. - * - only lr is clobbered. - * - the arguments are cast to long and assigned into the target registers - * which are then simply passed as registers to the asm code, so that we - * don't have to experience issues with register constraints. - * - the syscall number is always specified last in order to allow to force - * some registers before (gcc refuses a %-register at the last position). - * - in thumb mode without -fomit-frame-pointer, r7 is also used to store the - * frame pointer, and we cannot directly assign it as a register variable, - * nor can we clobber it. Instead we assign the r6 register and swap it - * with r7 before calling svc, and r6 is marked as clobbered. - * We're just using any regular register which we assign to r7 after saving - * it. - * - * Also, ARM supports the old_select syscall if newselect is not available - */ -#define __ARCH_WANT_SYS_OLD_SELECT - -#if (defined(__THUMBEB__) || defined(__THUMBEL__)) && \ - !defined(NOLIBC_OMIT_FRAME_POINTER) -/* swap r6,r7 needed in Thumb mode since we can't use nor clobber r7 */ -#define _NOLIBC_SYSCALL_REG "r6" -#define _NOLIBC_THUMB_SET_R7 "eor r7, r6\neor r6, r7\neor r7, r6\n" -#define _NOLIBC_THUMB_RESTORE_R7 "mov r7, r6\n" - -#else /* we're in ARM mode */ -/* in Arm mode we can directly use r7 */ -#define _NOLIBC_SYSCALL_REG "r7" -#define _NOLIBC_THUMB_SET_R7 "" -#define _NOLIBC_THUMB_RESTORE_R7 "" - -#endif /* end THUMB */ - -#define my_syscall0(num) \ -({ \ - register long _num __asm__(_NOLIBC_SYSCALL_REG) = (num); \ - register long _arg1 __asm__ ("r0"); \ - \ - __asm__ volatile ( \ - _NOLIBC_THUMB_SET_R7 \ - "svc #0\n" \ - _NOLIBC_THUMB_RESTORE_R7 \ - : "=r"(_arg1), "=r"(_num) \ - : "r"(_arg1), \ - "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -#define my_syscall1(num, arg1) \ -({ \ - register long _num __asm__(_NOLIBC_SYSCALL_REG) = (num); \ - register long _arg1 __asm__ ("r0") = (long)(arg1); \ - \ - __asm__ volatile ( \ - _NOLIBC_THUMB_SET_R7 \ - "svc #0\n" \ - _NOLIBC_THUMB_RESTORE_R7 \ - : "=r"(_arg1), "=r" (_num) \ - : "r"(_arg1), \ - "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -#define my_syscall2(num, arg1, arg2) \ -({ \ - register long _num __asm__(_NOLIBC_SYSCALL_REG) = (num); \ - register long _arg1 __asm__ ("r0") = (long)(arg1); \ - register long _arg2 __asm__ ("r1") = (long)(arg2); \ - \ - __asm__ volatile ( \ - _NOLIBC_THUMB_SET_R7 \ - "svc #0\n" \ - _NOLIBC_THUMB_RESTORE_R7 \ - : "=r"(_arg1), "=r" (_num) \ - : "r"(_arg1), "r"(_arg2), \ - "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -#define my_syscall3(num, arg1, arg2, arg3) \ -({ \ - register long _num __asm__(_NOLIBC_SYSCALL_REG) = (num); \ - register long _arg1 __asm__ ("r0") = (long)(arg1); \ - register long _arg2 __asm__ ("r1") = (long)(arg2); \ - register long _arg3 __asm__ ("r2") = (long)(arg3); \ - \ - __asm__ volatile ( \ - _NOLIBC_THUMB_SET_R7 \ - "svc #0\n" \ - _NOLIBC_THUMB_RESTORE_R7 \ - : "=r"(_arg1), "=r" (_num) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), \ - "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -#define my_syscall4(num, arg1, arg2, arg3, arg4) \ -({ \ - register long _num __asm__(_NOLIBC_SYSCALL_REG) = (num); \ - register long _arg1 __asm__ ("r0") = (long)(arg1); \ - register long _arg2 __asm__ ("r1") = (long)(arg2); \ - register long _arg3 __asm__ ("r2") = (long)(arg3); \ - register long _arg4 __asm__ ("r3") = (long)(arg4); \ - \ - __asm__ volatile ( \ - _NOLIBC_THUMB_SET_R7 \ - "svc #0\n" \ - _NOLIBC_THUMB_RESTORE_R7 \ - : "=r"(_arg1), "=r" (_num) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), \ - "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -#define my_syscall5(num, arg1, arg2, arg3, arg4, arg5) \ -({ \ - register long _num __asm__(_NOLIBC_SYSCALL_REG) = (num); \ - register long _arg1 __asm__ ("r0") = (long)(arg1); \ - register long _arg2 __asm__ ("r1") = (long)(arg2); \ - register long _arg3 __asm__ ("r2") = (long)(arg3); \ - register long _arg4 __asm__ ("r3") = (long)(arg4); \ - register long _arg5 __asm__ ("r4") = (long)(arg5); \ - \ - __asm__ volatile ( \ - _NOLIBC_THUMB_SET_R7 \ - "svc #0\n" \ - _NOLIBC_THUMB_RESTORE_R7 \ - : "=r"(_arg1), "=r" (_num) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -#define my_syscall6(num, arg1, arg2, arg3, arg4, arg5, arg6) \ -({ \ - register long _num __asm__(_NOLIBC_SYSCALL_REG) = (num); \ - register long _arg1 __asm__ ("r0") = (long)(arg1); \ - register long _arg2 __asm__ ("r1") = (long)(arg2); \ - register long _arg3 __asm__ ("r2") = (long)(arg3); \ - register long _arg4 __asm__ ("r3") = (long)(arg4); \ - register long _arg5 __asm__ ("r4") = (long)(arg5); \ - register long _arg6 __asm__ ("r5") = (long)(arg6); \ - \ - __asm__ volatile ( \ - _NOLIBC_THUMB_SET_R7 \ - "svc #0\n" \ - _NOLIBC_THUMB_RESTORE_R7 \ - : "=r"(_arg1), "=r" (_num) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "r"(_arg6), "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -#ifdef _NOLIBC_CRT_H - -/* startup code */ -void __attribute__((noreturn)) _start(void) -{ - __asm__ volatile ( - "mov r0, sp\n" /* save stack pointer to %r0, as arg1 of _start_c */ - "and ip, r0, #-8\n" /* sp must be 8-byte aligned in the callee */ - "mov sp, ip\n" - "bl _start_c\n" /* transfer to c runtime */ - ); - __builtin_unreachable(); -} - -#endif /* _NOLIBC_CRT_H */ - -#endif /* _NOLIBC_ARCH_ARM_H */ diff --git a/native/src/crt0/nolibc/arch-i386.h b/native/src/crt0/nolibc/arch-i386.h deleted file mode 100644 index 4870ed29f..000000000 --- a/native/src/crt0/nolibc/arch-i386.h +++ /dev/null @@ -1,179 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ -/* - * i386 specific definitions for NOLIBC - * Copyright (C) 2017-2022 Willy Tarreau - */ - -#ifndef _NOLIBC_ARCH_I386_H -#define _NOLIBC_ARCH_I386_H - -/* Syscalls for i386 : - * - mostly similar to x86_64 - * - registers are 32-bit - * - syscall number is passed in eax - * - arguments are in ebx, ecx, edx, esi, edi, ebp respectively - * - all registers are preserved (except eax of course) - * - the system call is performed by calling int $0x80 - * - syscall return comes in eax - * - the arguments are cast to long and assigned into the target registers - * which are then simply passed as registers to the asm code, so that we - * don't have to experience issues with register constraints. - * - the syscall number is always specified last in order to allow to force - * some registers before (gcc refuses a %-register at the last position). - * - * Also, i386 supports the old_select syscall if newselect is not available - */ -#define __ARCH_WANT_SYS_OLD_SELECT - -#define my_syscall0(num) \ -({ \ - long _ret; \ - register long _num __asm__ ("eax") = (num); \ - \ - __asm__ volatile ( \ - "int $0x80\n" \ - : "=a" (_ret) \ - : "0"(_num) \ - : "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall1(num, arg1) \ -({ \ - long _ret; \ - register long _num __asm__ ("eax") = (num); \ - register long _arg1 __asm__ ("ebx") = (long)(arg1); \ - \ - __asm__ volatile ( \ - "int $0x80\n" \ - : "=a" (_ret) \ - : "r"(_arg1), \ - "0"(_num) \ - : "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall2(num, arg1, arg2) \ -({ \ - long _ret; \ - register long _num __asm__ ("eax") = (num); \ - register long _arg1 __asm__ ("ebx") = (long)(arg1); \ - register long _arg2 __asm__ ("ecx") = (long)(arg2); \ - \ - __asm__ volatile ( \ - "int $0x80\n" \ - : "=a" (_ret) \ - : "r"(_arg1), "r"(_arg2), \ - "0"(_num) \ - : "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall3(num, arg1, arg2, arg3) \ -({ \ - long _ret; \ - register long _num __asm__ ("eax") = (num); \ - register long _arg1 __asm__ ("ebx") = (long)(arg1); \ - register long _arg2 __asm__ ("ecx") = (long)(arg2); \ - register long _arg3 __asm__ ("edx") = (long)(arg3); \ - \ - __asm__ volatile ( \ - "int $0x80\n" \ - : "=a" (_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), \ - "0"(_num) \ - : "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall4(num, arg1, arg2, arg3, arg4) \ -({ \ - long _ret; \ - register long _num __asm__ ("eax") = (num); \ - register long _arg1 __asm__ ("ebx") = (long)(arg1); \ - register long _arg2 __asm__ ("ecx") = (long)(arg2); \ - register long _arg3 __asm__ ("edx") = (long)(arg3); \ - register long _arg4 __asm__ ("esi") = (long)(arg4); \ - \ - __asm__ volatile ( \ - "int $0x80\n" \ - : "=a" (_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), \ - "0"(_num) \ - : "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall5(num, arg1, arg2, arg3, arg4, arg5) \ -({ \ - long _ret; \ - register long _num __asm__ ("eax") = (num); \ - register long _arg1 __asm__ ("ebx") = (long)(arg1); \ - register long _arg2 __asm__ ("ecx") = (long)(arg2); \ - register long _arg3 __asm__ ("edx") = (long)(arg3); \ - register long _arg4 __asm__ ("esi") = (long)(arg4); \ - register long _arg5 __asm__ ("edi") = (long)(arg5); \ - \ - __asm__ volatile ( \ - "int $0x80\n" \ - : "=a" (_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "0"(_num) \ - : "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall6(num, arg1, arg2, arg3, arg4, arg5, arg6) \ -({ \ - long _eax = (long)(num); \ - long _arg6 = (long)(arg6); /* Always in memory */ \ - __asm__ volatile ( \ - "pushl %[_arg6]\n\t" \ - "pushl %%ebp\n\t" \ - "movl 4(%%esp),%%ebp\n\t" \ - "int $0x80\n\t" \ - "popl %%ebp\n\t" \ - "addl $4,%%esp\n\t" \ - : "+a"(_eax) /* %eax */ \ - : "b"(arg1), /* %ebx */ \ - "c"(arg2), /* %ecx */ \ - "d"(arg3), /* %edx */ \ - "S"(arg4), /* %esi */ \ - "D"(arg5), /* %edi */ \ - [_arg6]"m"(_arg6) /* memory */ \ - : "memory", "cc" \ - ); \ - _eax; \ -}) - -#ifdef _NOLIBC_CRT_H - -/* startup code */ -/* - * i386 System V ABI mandates: - * 1) last pushed argument must be 16-byte aligned. - * 2) The deepest stack frame should be set to zero - * - */ -void __attribute__((noreturn)) _start(void) -{ - __asm__ volatile ( - "xor %ebp, %ebp\n" /* zero the stack frame */ - "mov %esp, %eax\n" /* save stack pointer to %eax, as arg1 of _start_c */ - "and $-16, %esp\n" /* last pushed argument must be 16-byte aligned */ - "push %eax\n" /* push arg1 on stack to support plain stack modes too */ - "call _start_c\n" /* transfer to c runtime */ - "hlt\n" /* ensure it does not return */ - ); - __builtin_unreachable(); -} - -#endif /* _NOLIBC_CRT_H */ - -#endif /* _NOLIBC_ARCH_I386_H */ diff --git a/native/src/crt0/nolibc/arch-x86_64.h b/native/src/crt0/nolibc/arch-x86_64.h deleted file mode 100644 index eb9201082..000000000 --- a/native/src/crt0/nolibc/arch-x86_64.h +++ /dev/null @@ -1,177 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ -/* - * x86_64 specific definitions for NOLIBC - * Copyright (C) 2017-2022 Willy Tarreau - */ - -#ifndef _NOLIBC_ARCH_X86_64_H -#define _NOLIBC_ARCH_X86_64_H - -/* Syscalls for x86_64 : - * - registers are 64-bit - * - syscall number is passed in rax - * - arguments are in rdi, rsi, rdx, r10, r8, r9 respectively - * - the system call is performed by calling the syscall instruction - * - syscall return comes in rax - * - rcx and r11 are clobbered, others are preserved. - * - the arguments are cast to long and assigned into the target registers - * which are then simply passed as registers to the asm code, so that we - * don't have to experience issues with register constraints. - * - the syscall number is always specified last in order to allow to force - * some registers before (gcc refuses a %-register at the last position). - * - see also x86-64 ABI section A.2 AMD64 Linux Kernel Conventions, A.2.1 - * Calling Conventions. - * - * Link x86-64 ABI: https://gitlab.com/x86-psABIs/x86-64-ABI/-/wikis/home - * - */ - -#define my_syscall0(num) \ -({ \ - long _ret; \ - register long _num __asm__ ("rax") = (num); \ - \ - __asm__ volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall1(num, arg1) \ -({ \ - long _ret; \ - register long _num __asm__ ("rax") = (num); \ - register long _arg1 __asm__ ("rdi") = (long)(arg1); \ - \ - __asm__ volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "r"(_arg1), \ - "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall2(num, arg1, arg2) \ -({ \ - long _ret; \ - register long _num __asm__ ("rax") = (num); \ - register long _arg1 __asm__ ("rdi") = (long)(arg1); \ - register long _arg2 __asm__ ("rsi") = (long)(arg2); \ - \ - __asm__ volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "r"(_arg1), "r"(_arg2), \ - "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall3(num, arg1, arg2, arg3) \ -({ \ - long _ret; \ - register long _num __asm__ ("rax") = (num); \ - register long _arg1 __asm__ ("rdi") = (long)(arg1); \ - register long _arg2 __asm__ ("rsi") = (long)(arg2); \ - register long _arg3 __asm__ ("rdx") = (long)(arg3); \ - \ - __asm__ volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), \ - "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall4(num, arg1, arg2, arg3, arg4) \ -({ \ - long _ret; \ - register long _num __asm__ ("rax") = (num); \ - register long _arg1 __asm__ ("rdi") = (long)(arg1); \ - register long _arg2 __asm__ ("rsi") = (long)(arg2); \ - register long _arg3 __asm__ ("rdx") = (long)(arg3); \ - register long _arg4 __asm__ ("r10") = (long)(arg4); \ - \ - __asm__ volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), \ - "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall5(num, arg1, arg2, arg3, arg4, arg5) \ -({ \ - long _ret; \ - register long _num __asm__ ("rax") = (num); \ - register long _arg1 __asm__ ("rdi") = (long)(arg1); \ - register long _arg2 __asm__ ("rsi") = (long)(arg2); \ - register long _arg3 __asm__ ("rdx") = (long)(arg3); \ - register long _arg4 __asm__ ("r10") = (long)(arg4); \ - register long _arg5 __asm__ ("r8") = (long)(arg5); \ - \ - __asm__ volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall6(num, arg1, arg2, arg3, arg4, arg5, arg6) \ -({ \ - long _ret; \ - register long _num __asm__ ("rax") = (num); \ - register long _arg1 __asm__ ("rdi") = (long)(arg1); \ - register long _arg2 __asm__ ("rsi") = (long)(arg2); \ - register long _arg3 __asm__ ("rdx") = (long)(arg3); \ - register long _arg4 __asm__ ("r10") = (long)(arg4); \ - register long _arg5 __asm__ ("r8") = (long)(arg5); \ - register long _arg6 __asm__ ("r9") = (long)(arg6); \ - \ - __asm__ volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "r"(_arg6), "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#ifdef _NOLIBC_CRT_H - -/* startup code */ -/* - * x86-64 System V ABI mandates: - * 1) %rsp must be 16-byte aligned right before the function call. - * 2) The deepest stack frame should be zero (the %rbp). - * - */ -void __attribute__((noreturn)) _start(void) -{ - __asm__ volatile ( - "xor %ebp, %ebp\n" /* zero the stack frame */ - "mov %rsp, %rdi\n" /* save stack pointer to %rdi, as arg1 of _start_c */ - "and $-16, %rsp\n" /* %rsp must be 16-byte aligned before call */ - "call _start_c\n" /* transfer to c runtime */ - "hlt\n" /* ensure it does not return */ - ); - __builtin_unreachable(); -} - -#endif /* _NOLIBC_CRT_H */ - -#endif /* _NOLIBC_ARCH_X86_64_H */ diff --git a/native/src/crt0/nolibc/arch.h b/native/src/crt0/nolibc/arch.h deleted file mode 100644 index b7ee2a6c7..000000000 --- a/native/src/crt0/nolibc/arch.h +++ /dev/null @@ -1,28 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ -/* - * Copyright (C) 2017-2022 Willy Tarreau - */ - -/* Below comes the architecture-specific code. For each architecture, we have - * the syscall declarations and the _start code definition. This is the only - * global part. On all architectures the kernel puts everything in the stack - * before jumping to _start just above us, without any return address (_start - * is not a function but an entry point). So at the stack pointer we find argc. - * Then argv[] begins, and ends at the first NULL. Then we have envp which - * starts and ends with a NULL as well. So envp=argv+argc+1. - */ - -#ifndef _NOLIBC_ARCH_H -#define _NOLIBC_ARCH_H - -#if defined(__x86_64__) -#include "arch-x86_64.h" -#elif defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__) -#include "arch-i386.h" -#elif defined(__ARM_EABI__) -#include "arch-arm.h" -#elif defined(__aarch64__) -#include "arch-aarch64.h" -#endif - -#endif /* _NOLIBC_ARCH_H */ diff --git a/native/src/crt0/nolibc/crt.h b/native/src/crt0/nolibc/crt.h deleted file mode 100644 index ccbe2ea17..000000000 --- a/native/src/crt0/nolibc/crt.h +++ /dev/null @@ -1,80 +0,0 @@ -/* SPDX-License-Identifier: LGPL-2.1 OR MIT */ -/* - * C Run Time support for NOLIBC - * Copyright (C) 2023 Zhangjin Wu - */ - -#ifndef _NOLIBC_CRT_H -#define _NOLIBC_CRT_H - -char **environ; -const unsigned long *_auxv; -void _exit(int); -void __init_stdio(void); - -typedef void init_func_t(int, char*[], char*[]); -typedef void fini_func_t(void); - -extern init_func_t *__preinit_array_start[]; -extern init_func_t *__preinit_array_end[]; -extern init_func_t *__init_array_start[]; -extern init_func_t *__init_array_end[]; -extern fini_func_t *__fini_array_start[]; -extern fini_func_t *__fini_array_end[]; - -static void call_array(init_func_t **start, init_func_t **end, int argc, char *argv[], char *envp[]) { - unsigned long count = end - start; - while (count-- > 0) { - init_func_t* function = *start++; - (*function)(argc, argv, envp); - } -} - -void __attribute__((used)) _start_c(long *sp) -{ - long argc; - char **argv; - char **envp; - const unsigned long *auxv; - /* silence potential warning: conflicting types for 'main' */ - int _nolibc_main(int, char **, char **) __asm__ ("main"); - - /* - * sp : argc <-- argument count, required by main() - * argv: argv[0] <-- argument vector, required by main() - * argv[1] - * ... - * argv[argc-1] - * null - * environ: environ[0] <-- environment variables, required by main() and getenv() - * environ[1] - * ... - * null - * _auxv: _auxv[0] <-- auxiliary vector, required by getauxval() - * _auxv[1] - * ... - * null - */ - - /* assign argc and argv */ - argc = *sp; - argv = (void *)(sp + 1); - - /* find environ */ - environ = envp = argv + argc + 1; - - /* find _auxv */ - for (auxv = (void *)envp; *auxv++;) - ; - _auxv = auxv; - - /* call preinit and init */ - __init_stdio(); - call_array(__preinit_array_start, __preinit_array_end, argc, argv, envp); - call_array(__init_array_start, __init_array_end, argc, argv, envp); - - /* go to application */ - _exit(_nolibc_main(argc, argv, envp)); -} - -#endif /* _NOLIBC_CRT_H */ diff --git a/native/src/crt0/stdio.c b/native/src/crt0/stdio.c deleted file mode 100644 index 2e78aa734..000000000 --- a/native/src/crt0/stdio.c +++ /dev/null @@ -1,222 +0,0 @@ -#include -#include -#include -#include - -typedef struct file_ptr_t { - int fd; - void *cookie; - int (*read_fn)(void*, char*, int); - int (*write_fn)(void*, const char*, int); - int (*close_fn)(void*); -} file_ptr_t; - -static int fp_read_fn(void *p, char *buf, int sz) { - intptr_t fd = (intptr_t) p; - return read(fd, buf, sz); -} - -static int fp_write_fn(void *p, const char *buf, int sz) { - intptr_t fd = (intptr_t) p; - return write(fd, buf, sz); -} - -static int fp_close_fn(void *p) { - intptr_t fd = (intptr_t) p; - return close(fd); -} - -static void set_fp_fd(file_ptr_t *fp, int fd) { - fp->fd = fd; - fp->cookie = NULL; - fp->read_fn = fp_read_fn; - fp->write_fn = fp_write_fn; - fp->close_fn = fp_close_fn; -} - -static file_ptr_t __stdio_fp[3]; - -FILE* stdin = (FILE *) &__stdio_fp[0]; -FILE* stdout = (FILE *) &__stdio_fp[1]; -FILE* stderr = (FILE *) &__stdio_fp[2]; - -void __init_stdio(void) { - set_fp_fd((file_ptr_t *) stdin, 0); - set_fp_fd((file_ptr_t *) stdout, 1); - set_fp_fd((file_ptr_t *) stderr, 2); -} - -FILE *fdopen(int fd, const char *mode __attribute__((unused))) { - file_ptr_t *fp = malloc(sizeof(file_ptr_t)); - set_fp_fd(fp, fd); - return (FILE *) fp; -} - -FILE *funopen(const void* cookie, - int (*read_fn)(void*, char*, int), - int (*write_fn)(void*, const char*, int), - fpos_t (*seek_fn)(void*, fpos_t, int), - int (*close_fn)(void*)) { - file_ptr_t *fp = malloc(sizeof(file_ptr_t)); - fp->fd = -1; - fp->cookie = (void *) cookie; - fp->read_fn = read_fn; - fp->write_fn = write_fn; - fp->close_fn = close_fn; - return (FILE *) fp; -} - -#define fn_arg (fp->fd < 0 ? fp->cookie : ((void*)(intptr_t) fp->fd)) - -int fclose(FILE *stream) { - file_ptr_t *fp = (file_ptr_t *) stream; - int ret = fp->close_fn(fn_arg); - free(fp); - return ret; -} - -int fileno(FILE *stream) { - file_ptr_t *fp = (file_ptr_t *) stream; - return fp->fd; -} - -int fputc(int ch, FILE *stream) { - char c = ch; - file_ptr_t *fp = (file_ptr_t *) stream; - return fp->write_fn(fn_arg, &c, 1) >= 0 ? 0 : EOF; -} - -size_t fwrite(const void* buf, size_t size, size_t count, FILE* stream) { - file_ptr_t *fp = (file_ptr_t *) stream; - int len = size * count; - int ret = fp->write_fn(fn_arg, buf, len); - return ret == len ? count : 0; -} - -int fputs(const char* s, FILE* stream) { - file_ptr_t *fp = (file_ptr_t *) stream; - size_t length = strlen(s); - return fp->write_fn(fn_arg, s, length) == length ? 0 : EOF; -} - -int fgetc(FILE *stream) { - char ch; - file_ptr_t *fp = (file_ptr_t *) stream; - if (fp->read_fn(fn_arg, &ch, 1) == 1) { - return ch; - } - return -1; -} - -size_t fread(void *buf, size_t size, size_t count, FILE* stream) { - file_ptr_t *fp = (file_ptr_t *) stream; - int len = size * count; - int ret = fp->read_fn(fn_arg, buf, len); - return ret == len ? count : 0; -} - -void setbuf(FILE* fp, char* buf) {} - -#include "tinystdio/tinystdio.c" - -struct file_putp { - FILE *fp; - int len; -}; - -static void file_putc(void *data, char ch) { - struct file_putp *putp = data; - int r = write(fileno(putp->fp), &ch, 1); - if (r >= 0) - putp->len += r; -} - -int vfprintf(FILE *stream, const char *format, va_list arg) { - struct file_putp data; - data.fp = stream; - data.len = 0; - tfp_format(&data, &file_putc, format, arg); - return data.len; -} - -int vasprintf(char **strp, const char *fmt, va_list ap) { - int size = vsnprintf(NULL, 0, fmt, ap); - if (size >= 0) { - *strp = malloc(size + 1); - vsnprintf(*strp, size, fmt, ap); - } - return size; -} - -int vprintf(const char *fmt, va_list args) { - return vfprintf(stdout, fmt, args); -} - -int fprintf(FILE *stream, const char *fmt, ...) { - va_list args; - int ret; - - va_start(args, fmt); - ret = vfprintf(stream, fmt, args); - va_end(args); - return ret; -} - -int printf(const char *fmt, ...) { - va_list args; - int ret; - - va_start(args, fmt); - ret = vfprintf(stdout, fmt, args); - va_end(args); - return ret; -} - -int sscanf(const char *str, const char *format, ...) { - va_list ap; - int retval; - - va_start(ap, format); - retval = tfp_vsscanf(str, format, ap); - va_end(ap); - return retval; -} - -// Original source: https://github.com/freebsd/freebsd/blob/master/contrib/file/src/getline.c -// License: BSD, full copyright notice please check original source -ssize_t getdelim(char **buf, size_t *bufsiz, int delimiter, FILE *fp) { - char *ptr, *eptr; - - if (*buf == NULL || *bufsiz == 0) { - *bufsiz = BUFSIZ; - if ((*buf = (char *) malloc(*bufsiz)) == NULL) - return -1; - } - - for (ptr = *buf, eptr = *buf + *bufsiz;;) { - int c = fgetc(fp); - if (c == -1) { - return ptr == *buf ? -1 : ptr - *buf; - } - *ptr++ = c; - if (c == delimiter) { - *ptr = '\0'; - return ptr - *buf; - } - if (ptr + 2 >= eptr) { - char *nbuf; - size_t nbufsiz = *bufsiz * 2; - ssize_t d = ptr - *buf; - if ((nbuf = (char *) realloc(*buf, nbufsiz)) == NULL) - return -1; - *buf = nbuf; - *bufsiz = nbufsiz; - eptr = nbuf + nbufsiz; - ptr = nbuf + d; - } - } -} - -ssize_t getline(char **buf, size_t *bufsiz, FILE *fp) { - return getdelim(buf, bufsiz, '\n', fp); -} diff --git a/native/src/crt0/syscall.c b/native/src/crt0/syscall.c deleted file mode 100644 index 1f10a502b..000000000 --- a/native/src/crt0/syscall.c +++ /dev/null @@ -1,221 +0,0 @@ -#define SYS_INLINE - -#include "linux_syscall_support.h" - -// Some missing declarations -static inline _syscall3(int, faccessat, int, f, const char *, p, int, m) -_syscall2(int, umount2, const char *, t, int, f) -_syscall4(int, renameat, int, o, const char *, op, int, n, const char *, np) -_syscall1(mode_t, umask, mode_t, mask) -_syscall1(int, chroot, const char *, path) -_syscall2(int, nanosleep, const struct kernel_timespec *, req, struct kernel_timespec *, rem) -_syscall5(int, mount, const char *, s, const char *, t, - const char *, fs, unsigned long, f, const void *, d) -_syscall3(int, symlinkat, const char *, t, int, fd, const char *, l) -_syscall3(int, mkdirat, int, dirfd, const char *, pathname, mode_t, mode) -_syscall4(ssize_t, sendfile, int, out_fd, int, in_fd, off_t *, offset, size_t, count) -_syscall5(int, linkat, int, o, const char *, op, int, n, const char *, np, int, f) -_syscall4(int, mknodat, int, dirfd, const char *, pathname, mode_t, mode, dev_t, dev) -_syscall2(int, fchmod, int, fd, mode_t, mode) -_syscall4(int, fchmodat, int, dirfd, const char *, pathname, mode_t, mode, int, flags) -_syscall5(int, fchownat, int, dirfd, const char *, p, uid_t, owner, gid_t, group, int, flags) -_syscall3(ssize_t, readv, int, fd, const struct kernel_iovec*, v, size_t, c) - -#define SYMBOL_ALIAS(from, to) \ -__asm__(".global " #from " \n " #from " = " #to) - -#define EXPORT_SYMBOL(name) \ -SYMBOL_ALIAS(name, sys_##name) - -EXPORT_SYMBOL(_exit); -EXPORT_SYMBOL(openat); -EXPORT_SYMBOL(close); -EXPORT_SYMBOL(read); -EXPORT_SYMBOL(symlink); -EXPORT_SYMBOL(write); -EXPORT_SYMBOL(writev); -EXPORT_SYMBOL(unlink); -EXPORT_SYMBOL(mmap); -EXPORT_SYMBOL(munmap); -EXPORT_SYMBOL(mremap); -EXPORT_SYMBOL(readlink); -EXPORT_SYMBOL(unlinkat); -EXPORT_SYMBOL(getpid); -EXPORT_SYMBOL(chdir); -EXPORT_SYMBOL(umount2); -EXPORT_SYMBOL(readlinkat); -EXPORT_SYMBOL(renameat); -EXPORT_SYMBOL(umask); -EXPORT_SYMBOL(chroot); -EXPORT_SYMBOL(mount); -EXPORT_SYMBOL(symlinkat); -EXPORT_SYMBOL(stat); -EXPORT_SYMBOL(lstat); -EXPORT_SYMBOL(statfs); -EXPORT_SYMBOL(mkdirat); -EXPORT_SYMBOL(ioctl); -EXPORT_SYMBOL(fork); -EXPORT_SYMBOL(sendfile); -EXPORT_SYMBOL(ftruncate); -EXPORT_SYMBOL(linkat); -EXPORT_SYMBOL(mknodat); -EXPORT_SYMBOL(fchmod); -EXPORT_SYMBOL(fchmodat); -EXPORT_SYMBOL(fchownat); -EXPORT_SYMBOL(readv); -EXPORT_SYMBOL(lseek); -EXPORT_SYMBOL(execve); -EXPORT_SYMBOL(getdents64); - -SYMBOL_ALIAS(exit, _exit); - -#if defined(__LP64__) - -EXPORT_SYMBOL(fstat); -EXPORT_SYMBOL(newfstatat); -SYMBOL_ALIAS(fstatat, newfstatat); - -#else - -EXPORT_SYMBOL(fstat64); -EXPORT_SYMBOL(fstatat64); -SYMBOL_ALIAS(fstat, fstat64); -SYMBOL_ALIAS(fstatat, fstatat64); - -#endif - -int fchown(int fd, uid_t owner, gid_t group) { - return fchownat(fd, "", owner, group, AT_EMPTY_PATH); -} - -int lchown(const char* path, uid_t uid, gid_t gid) { - return fchownat(AT_FDCWD, path, uid, gid, AT_SYMLINK_NOFOLLOW); -} - -int chown(const char* path, uid_t uid, gid_t gid) { - return fchownat(AT_FDCWD, path, uid, gid, 0); -} - -int chmod(const char* path, mode_t mode) { - return sys_fchmodat(AT_FDCWD, path, mode, 0); -} - -int mkfifoat(int fd, const char* path, mode_t mode) { - return sys_mknodat(fd, path, (mode & ~S_IFMT) | S_IFIFO, 0); -} - -int mkfifo(const char* path, mode_t mode) { - return mkfifoat(AT_FDCWD, path, mode); -} - -int mknod(const char* path, mode_t mode, dev_t dev) { - return sys_mknodat(AT_FDCWD, path, mode, dev); -} - -int link(const char *oldpath, const char *newpath) { - return sys_linkat(AT_FDCWD, oldpath, AT_FDCWD, newpath, 0); -} - -int rmdir(const char *path) { - return sys_unlinkat(AT_FDCWD, path, AT_REMOVEDIR); -} - -int mkdir(const char *pathname, mode_t mode) { - return sys_mkdirat(AT_FDCWD, pathname, mode); -} - -int symlink(const char *target, const char *linkpath) { - return sys_symlinkat(target, AT_FDCWD, linkpath); -} - -int rename(const char *oldpath, const char *newpath) { - return sys_renameat(AT_FDCWD, oldpath, AT_FDCWD, newpath); -} - -int access(const char* path, int mode) { - return faccessat(AT_FDCWD, path, mode, 0); -} - -int remove(const char *path) { - int r = sys_unlinkat(AT_FDCWD, path, 0); - if (r < 0 && errno == EISDIR) { - r = sys_unlinkat(AT_FDCWD, path, AT_REMOVEDIR); - } - return r; -} - -// Source: bionic/libc/bionic/abort.cpp -void abort() { - // Don't block SIGABRT to give any signal handler a chance; we ignore - // any errors -- X311J doesn't allow abort to return anyway. - struct kernel_sigset_t mask; - sys_sigfillset(&mask); - sys_sigdelset(&mask, SIGABRT); - - sys_sigprocmask(SIG_SETMASK, &mask, NULL); - sys_raise(SIGABRT); - - // If SIGABRT is ignored or it's caught and the handler returns, - // remove the SIGABRT signal handler and raise SIGABRT again. - struct kernel_sigaction sa = { .sa_handler_ = SIG_DFL, .sa_flags = SA_RESTART }; - sys_sigaction(SIGABRT, &sa, NULL); - - sys_sigprocmask(SIG_SETMASK, &mask, NULL); - sys_raise(SIGABRT); - - // If we get this far, just exit. - _exit(127); -} - -// Source: bionic/libc/bionic/usleep.cpp -int usleep(useconds_t us) { - struct kernel_timespec ts; - ts.tv_sec = us / 1000000; - ts.tv_nsec = (us % 1000000) * 1000; - return sys_nanosleep(&ts, NULL); -} - -// Source: bionic/libc/bionic/faccessat.cpp -int faccessat(int dirfd, const char *pathname, int mode, int flags) { - // "The mode specifies the accessibility check(s) to be performed, - // and is either the value F_OK, or a mask consisting of the - // bitwise OR of one or more of R_OK, W_OK, and X_OK." - if ((mode != F_OK) && ((mode & ~(R_OK | W_OK | X_OK)) != 0) && - ((mode & (R_OK | W_OK | X_OK)) == 0)) { - errno = EINVAL; - return -1; - } - - if (flags != 0) { - // We deliberately don't support AT_SYMLINK_NOFOLLOW, a glibc - // only feature which is error prone and dangerous. - // More details at http://permalink.gmane.org/gmane.linux.lib.musl.general/6952 - // - // AT_EACCESS isn't supported either. Android doesn't have setuid - // programs, and never runs code with euid!=uid. - // - // We could use faccessat2(2) from Linux 5.8, but since we don't want the - // first feature and don't need the second, we just reject such requests. - errno = EINVAL; - return -1; - } - - return sys_faccessat(dirfd, pathname, mode); -} - -int open(const char *pathname, int flags, ...) { - int mode = 0; - - if (((flags & O_CREAT) == O_CREAT) || ((flags & O_TMPFILE) == O_TMPFILE)) { - va_list args; - va_start(args, flags); - mode = va_arg(args, int); - va_end(args); - } - -#if !defined(__LP64__) - flags |= O_LARGEFILE; -#endif - - return sys_openat(AT_FDCWD, pathname, flags, mode); -} diff --git a/native/src/crt0/tinystdio/tinystdio.c b/native/src/crt0/tinystdio/tinystdio.c deleted file mode 100644 index b9b1f7c46..000000000 --- a/native/src/crt0/tinystdio/tinystdio.c +++ /dev/null @@ -1,834 +0,0 @@ -/* -File: tinyprintf.c - -Copyright (C) 2004 Kustaa Nyholm - -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; either -version 2.1 of the License, or (at your option) any later version. - -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with this library; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -*/ - -#include - -#include "tinystdio.h" - - -/* - * Configuration - */ - -/* Enable long int support */ -#define PRINTF_LONG_SUPPORT - -/* Enable long long int support (implies long int support) */ -#define PRINTF_LONG_LONG_SUPPORT - -/* Enable %z (size_t) support */ -#define PRINTF_SIZE_T_SUPPORT - - -/* - * Configuration adjustments - */ -#ifdef PRINTF_SIZE_T_SUPPORT -#include -#endif - -#ifdef PRINTF_LONG_LONG_SUPPORT -# define PRINTF_LONG_SUPPORT -#endif - -/* __SIZEOF___ defined at least by gcc */ -#ifdef __SIZEOF_POINTER__ -# define SIZEOF_POINTER __SIZEOF_POINTER__ -#endif -#ifdef __SIZEOF_LONG_LONG__ -# define SIZEOF_LONG_LONG __SIZEOF_LONG_LONG__ -#endif -#ifdef __SIZEOF_LONG__ -# define SIZEOF_LONG __SIZEOF_LONG__ -#endif -#ifdef __SIZEOF_INT__ -# define SIZEOF_INT __SIZEOF_INT__ -#endif - -#ifdef __GNUC__ -# define _TFP_GCC_NO_INLINE_ __attribute__ ((noinline)) -#else -# define _TFP_GCC_NO_INLINE_ -#endif - -/* - * Implementation - */ -struct param { - bool lz; /**< Leading zeros */ - bool alt; /**< alternate form */ - bool uc; /**< Upper case (for base16 only) */ - bool align_left; /**< 0 == align right (default), 1 == align left */ - int width; /**< field width */ - char sign; /**< The sign to display (if any) */ - unsigned int base; /**< number base (e.g.: 8, 10, 16) */ - char *bf; /**< Buffer to output */ - char prec; /**< Floating point precision */ -}; - - -#ifdef PRINTF_LONG_LONG_SUPPORT -static void _TFP_GCC_NO_INLINE_ ulli2a( - unsigned long long int num, struct param *p) -{ - int n = 0; - unsigned long long int d = 1; - char *bf = p->bf; - while (num / d >= p->base) - d *= p->base; - while (d != 0) { - int dgt = num / d; - num %= d; - d /= p->base; - if (n || dgt > 0 || d == 0) { - *bf++ = dgt + (dgt < 10 ? '0' : (p->uc ? 'A' : 'a') - 10); - ++n; - } - } - *bf = 0; -} - -static void lli2a(long long int num, struct param *p) -{ - if (num < 0) { - num = -num; - p->sign = '-'; - } - ulli2a(num, p); -} -#endif - -#ifdef PRINTF_LONG_SUPPORT -static void uli2a(unsigned long int num, struct param *p) -{ - int n = 0; - unsigned long int d = 1; - char *bf = p->bf; - while (num / d >= p->base) - d *= p->base; - while (d != 0) { - int dgt = num / d; - num %= d; - d /= p->base; - if (n || dgt > 0 || d == 0) { - *bf++ = dgt + (dgt < 10 ? '0' : (p->uc ? 'A' : 'a') - 10); - ++n; - } - } - *bf = 0; -} - -static void li2a(long num, struct param *p) -{ - if (num < 0) { - num = -num; - p->sign = '-'; - } - uli2a(num, p); -} -#endif - -static void ui2a(unsigned int num, struct param *p) -{ - int n = 0; - unsigned int d = 1; - char *bf = p->bf; - while (num / d >= p->base) - d *= p->base; - while (d != 0) { - int dgt = num / d; - num %= d; - d /= p->base; - if (n || dgt > 0 || d == 0) { - *bf++ = dgt + (dgt < 10 ? '0' : (p->uc ? 'A' : 'a') - 10); - ++n; - } - } - *bf = 0; -} - -static void i2a(int num, struct param *p) -{ - if (num < 0) { - num = -num; - p->sign = '-'; - } - ui2a(num, p); -} - -static int a2d(char ch) -{ - if (ch >= '0' && ch <= '9') - return ch - '0'; - else if (ch >= 'a' && ch <= 'f') - return ch - 'a' + 10; - else if (ch >= 'A' && ch <= 'F') - return ch - 'A' + 10; - else - return -1; -} - -static char a2u(char ch, const char **src, int base, int *nump) -{ - const char *p = *src; - int num = 0; - int digit; - while ((digit = a2d(ch)) >= 0) { - if (digit > base) - break; - num = num * base + digit; - ch = *p++; - } - *src = p; - *nump = num; - return ch; -} - -static void putchw(void *putp, putcf putf, struct param *p) -{ - char ch; - int n = p->width; - char *bf = p->bf; - - /* Number of filling characters */ - while (*bf++ && n > 0) - n--; - if (p->sign) - n--; - if (p->alt && p->base == 16) - n -= 2; - else if (p->alt && p->base == 8) - n--; - - /* Fill with space to align to the right, before alternate or sign */ - if (!p->lz && !p->align_left) { - while (n-- > 0) - putf(putp, ' '); - } - - /* print sign */ - if (p->sign) - putf(putp, p->sign); - - /* Alternate */ - if (p->alt && p->base == 16) { - putf(putp, '0'); - putf(putp, (p->uc ? 'X' : 'x')); - } else if (p->alt && p->base == 8) { - putf(putp, '0'); - } - - /* Fill with zeros, after alternate or sign */ - if (p->lz) { - while (n-- > 0) - putf(putp, '0'); - } - - /* Put actual buffer */ - bf = p->bf; - while ((ch = *bf++)) - putf(putp, ch); - - /* Fill with space to align to the left, after string */ - if (!p->lz && p->align_left) { - while (n-- > 0) - putf(putp, ' '); - } -} - -void tfp_format(void *putp, putcf putf, const char *fmt, va_list va) -{ - struct param p; - double fval; - int temp_buffer[16]; - int fpart; - int fiter; - int ffactor; - int sign; -#ifdef PRINTF_LONG_SUPPORT - char bf[23]; /* long = 64b on some architectures */ -#else - char bf[12]; /* int = 32b on some architectures */ -#endif - char ch; - p.bf = bf; - - while ((ch = *(fmt++))) { - if (ch != '%') { - putf(putp, ch); - } else { -#ifdef PRINTF_LONG_SUPPORT - char lng = 0; /* 1 for long, 2 for long long */ -#endif - /* Init parameter struct */ - p.lz = 0; - p.alt = 0; - p.width = 0; - p.align_left = 0; - p.sign = 0; - p.prec = TINY_PRINTF_FP_PRECISION; - - /* Flags */ - while ((ch = *(fmt++))) { - switch (ch) { - case '-': - p.align_left = 1; - continue; - case '0': - p.lz = 1; - continue; - case '#': - p.alt = 1; - continue; - case '+': - p.sign = 1; - continue; - default: - break; - } - break; - } - - /* Width */ - if (ch >= '0' && ch <= '9') { - ch = a2u(ch, &fmt, 10, &(p.width)); - } - - /* We accept 'x.y' format but don't support it completely: - * we ignore the 'y' digit => this ignores 0-fill - * size and makes it == width (ie. 'x') */ - if (ch == '.') { - //p.lz = 1; /* zero-padding */ - /* ignore actual 0-fill size: */ - ch = *(fmt++); - if (ch >= '0' && ch <= '9') - p.prec = ch - '0'; - do - { - ch = *(fmt++); - } while (ch >= '0' && ch <= '9'); - - } - -#ifdef PRINTF_SIZE_T_SUPPORT -# ifdef PRINTF_LONG_SUPPORT - if (ch == 'z') { - ch = *(fmt++); - if (sizeof(size_t) == sizeof(unsigned long int)) - lng = 1; -# ifdef PRINTF_LONG_LONG_SUPPORT - else if (sizeof(size_t) == sizeof(unsigned long long int)) - lng = 2; -# endif - } else -# endif -#endif - -#ifdef PRINTF_LONG_SUPPORT - if (ch == 'l') { - ch = *(fmt++); - lng = 1; -#ifdef PRINTF_LONG_LONG_SUPPORT - if (ch == 'l') { - ch = *(fmt++); - lng = 2; - } -#endif - } -#endif - switch (ch) { - case 0: - goto abort; - case 'u': - p.base = 10; -#ifdef PRINTF_LONG_SUPPORT -#ifdef PRINTF_LONG_LONG_SUPPORT - if (2 == lng) - ulli2a(va_arg(va, unsigned long long int), &p); - else -#endif - if (1 == lng) - uli2a(va_arg(va, unsigned long int), &p); - else -#endif - ui2a(va_arg(va, unsigned int), &p); - putchw(putp, putf, &p); - break; - case 'd': - case 'i': - p.base = 10; -#ifdef PRINTF_LONG_SUPPORT -#ifdef PRINTF_LONG_LONG_SUPPORT - if (2 == lng) - lli2a(va_arg(va, long long int), &p); - else -#endif - if (1 == lng) - li2a(va_arg(va, long int), &p); - else -#endif - i2a(va_arg(va, int), &p); - putchw(putp, putf, &p); - break; -#ifdef SIZEOF_POINTER - case 'p': - p.alt = 1; -# if defined(SIZEOF_INT) && SIZEOF_POINTER <= SIZEOF_INT - lng = 0; -# elif defined(SIZEOF_LONG) && SIZEOF_POINTER <= SIZEOF_LONG - lng = 1; -# elif defined(SIZEOF_LONG_LONG) && SIZEOF_POINTER <= SIZEOF_LONG_LONG - lng = 2; -# endif -#endif - case 'x': - case 'X': - p.base = 16; - p.uc = (ch == 'X')?1:0; -#ifdef PRINTF_LONG_SUPPORT -#ifdef PRINTF_LONG_LONG_SUPPORT - if (2 == lng) - ulli2a(va_arg(va, unsigned long long int), &p); - else -#endif - if (1 == lng) - uli2a(va_arg(va, unsigned long int), &p); - else -#endif - ui2a(va_arg(va, unsigned int), &p); - putchw(putp, putf, &p); - break; - case 'o': - p.base = 8; - ui2a(va_arg(va, unsigned int), &p); - putchw(putp, putf, &p); - break; - case 'c': - putf(putp, (char)(va_arg(va, int))); - break; - case 's': - p.bf = va_arg(va, char *); - putchw(putp, putf, &p); - p.bf = bf; - break; - case '%': - putf(putp, ch); - break; - case 'f': - case 'F': - fval = va_arg(va, double); - sign = 0; - if (fval < 0) - { - sign = 1; - p.width--; - fval = - fval; - } - else if (p.sign) { - sign = 2; - p.width--; - } - - fpart = (int)fval; - - fiter = 0; - while (fpart != 0) - { - temp_buffer[fiter++] = fpart % 10; - fpart = fpart / 10; - - } - fiter--; - if (fiter == -1) - p.width--; - /* Leading zeros */ - if (p.lz) { - - if (sign == 1) - putf(putp, '-'); - else if (sign == 2) - putf(putp, '+'); - - while (p.width-- > p.prec + fiter + 2) - { - putf(putp, '0'); - } - } - else - { - - while (p.width-- > p.prec + fiter + 2) - { - putf(putp, ' '); - } - - if (sign == 1) - putf(putp, '-'); - else if (sign == 2) - putf(putp, '+'); - - } - - if (fiter == -1) - putf(putp, '0'); - while (fiter > -1) - { - putf(putp, '0' + (temp_buffer[fiter--])); - } - - putf(putp, '.'); - ffactor = 1; - while (p.prec-- > 0) - { - ffactor *= 10; - fpart = (int)((fval - (int)fval)*ffactor); - if (fpart == 0) - putf(putp, '0'); - } - fiter = 0; - while (fpart != 0) - { - temp_buffer[fiter++] = fpart % 10; - fpart = fpart / 10; - - } - fiter--; - while (fiter > -1) - { - putf(putp, '0' + (temp_buffer[fiter--])); - } - break; - default: - break; - } - } - } - abort:; -} - - -#if TINYPRINTF_DEFINE_TFP_PRINTF -static putcf stdout_putf; -static void *stdout_putp; - -void init_printf(void *putp, putcf putf) -{ - stdout_putf = putf; - stdout_putp = putp; -} - -void tfp_printf(char *fmt, ...) -{ - va_list va; - va_start(va, fmt); - tfp_format(stdout_putp, stdout_putf, fmt, va); - va_end(va); -} -#endif - -#if TINYPRINTF_DEFINE_TFP_SPRINTF -struct _vsnprintf_putcf_data -{ - size_t dest_capacity; - char *dest; - size_t num_chars; -}; - -static void _vsnprintf_putcf(void *p, char c) -{ - struct _vsnprintf_putcf_data *data = (struct _vsnprintf_putcf_data*)p; - if (data->num_chars < data->dest_capacity) - data->dest[data->num_chars] = c; - data->num_chars ++; -} - -int tfp_vsnprintf(char *str, size_t size, const char *format, va_list ap) -{ - struct _vsnprintf_putcf_data data; - - data.dest = str; - data.dest_capacity = size ? size - 1 : 0; - data.num_chars = 0; - tfp_format(&data, _vsnprintf_putcf, format, ap); - - if (data.num_chars < data.dest_capacity) - data.dest[data.num_chars] = '\0'; - else if (size) - data.dest[data.dest_capacity] = '\0'; - - return data.num_chars; -} - -int tfp_snprintf(char *str, size_t size, const char *format, ...) -{ - va_list ap; - int retval; - - va_start(ap, format); - retval = tfp_vsnprintf(str, size, format, ap); - va_end(ap); - return retval; -} - -struct _vsprintf_putcf_data -{ - char *dest; - size_t num_chars; -}; - -static void _vsprintf_putcf(void *p, char c) -{ - struct _vsprintf_putcf_data *data = (struct _vsprintf_putcf_data*)p; - data->dest[data->num_chars++] = c; -} - -int tfp_vsprintf(char *str, const char *format, va_list ap) -{ - struct _vsprintf_putcf_data data; - data.dest = str; - data.num_chars = 0; - tfp_format(&data, _vsprintf_putcf, format, ap); - data.dest[data.num_chars] = '\0'; - return data.num_chars; -} - -int tfp_sprintf(char *str, const char *format, ...) -{ - va_list ap; - int retval; - - va_start(ap, format); - retval = tfp_vsprintf(str, format, ap); - va_end(ap); - return retval; -} - -#endif - -int tfp_vsscanf(const char* str, const char* format, va_list ap) -{ - int value, tmp; - float fvalue; - double Fvalue; - int count = 0; - int pos; - char neg, fmt_code; - - for (count = 0; *format != 0 && *str != 0; format++, str++) - { - while (*format == ' ' && *format != 0) format++; - - if (*format == 0) - break; - - while (*str == ' ' && *str != 0) str++; - - if (*str == 0) - break; - - if (*format == '%') - { - format++; - if (*format == 'n') - { - if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) - { - fmt_code = 'x'; - str += 2; - } - else - if (str[0] == 'b') - { - fmt_code = 'b'; - str++; - } - else - fmt_code = 'd'; - } - else - fmt_code = *format; - - switch (fmt_code) - { - case 'x': - case 'X': - for (value = 0, pos = 0; *str != 0; str++, pos++) - { - if ('0' <= *str && *str <= '9') - tmp = *str - '0'; - else - if ('a' <= *str && *str <= 'f') - tmp = *str - 'a' + 10; - else - if ('A' <= *str && *str <= 'F') - tmp = *str - 'A' + 10; - else - break; - - value *= 16; - value += tmp; - } - if (pos == 0) - return count; - *(va_arg(ap, int*)) = value; - count++; - break; - - case 'b': - for (value = 0, pos = 0; *str != 0; str++, pos++) - { - if (*str != '0' && *str != '1') - break; - - value *= 2; - value += *str - '0'; - } - - if (pos == 0) - return count; - - *(va_arg(ap, int*)) = value; - count++; - break; - - case 'd': - if (*str == '-') - { - neg = 1; - str++; - } - else - neg = 0; - for (value = 0, pos = 0; *str != 0; str++, pos++) - { - if ('0' <= *str && *str <= '9') - value = value*10 + (int)(*str - '0'); - else - break; - } - if (pos == 0) - return count; - *(va_arg(ap, int*)) = neg ? -value : value; - count++; - break; - - case 'f': - if (*str == '-') - { - neg = 1; - str++; - } - else - neg = 0; - - int point_flag = 0; - int exp = 0; - for (fvalue = 0, pos = 0; *str != 0 ; str++, pos++) - { - if (*str == '.') - { - point_flag = 1; - str++; - } - if ('0' <= *str && *str <= '9') - fvalue = fvalue*10 + (int)(*str - '0'); - else - break; - - if (point_flag == 1) - exp++; - - } - - if (pos == 0) - return count; - - for (pos = 0; pos < exp; pos++) - fvalue = fvalue/10.0; - - *(va_arg(ap, float*)) = neg ? -fvalue : fvalue; - count++; - break; - - case 'F': - if (*str == '-') - { - neg = 1; - str++; - } - else - neg = 0; - - int Fpoint_flag = 0; - int Fexp = 0; - for (Fvalue = 0, pos = 0; *str != 0 ; str++, pos++) - { - - if (*str == '.') - { - Fpoint_flag = 1; - str++; - } - if ('0' <= *str && *str <= '9') - Fvalue = Fvalue*10 + (int)(*str - '0'); - else - break; - - if (Fpoint_flag == 1) - Fexp++; - - } - - if (pos == 0) - return count; - for (pos = 0; pos < Fexp; pos++) - Fvalue = Fvalue/10.0; - *(va_arg(ap, double*)) = neg ? -Fvalue : Fvalue; - count++; - break; - - case 'c': - *(va_arg(ap, char*)) = *str; - count++; - break; - - case 's': - pos = 0; - char* tab = va_arg(ap, char*); - while (*str != ' ' && *str != 0) - *(tab++) = *str++; - *tab = 0; - count++; - break; - - default: - return count; - } - } - else - { - if (*format != *str) - break; - } - } - - return count; -} diff --git a/native/src/crt0/tinystdio/tinystdio.h b/native/src/crt0/tinystdio/tinystdio.h deleted file mode 100644 index 89de5dcf0..000000000 --- a/native/src/crt0/tinystdio/tinystdio.h +++ /dev/null @@ -1,204 +0,0 @@ -/* -File: tinyprintf.h - -Copyright (C) 2004 Kustaa Nyholm - -This library is free software; you can redistribute it and/or -modify it under the terms of the GNU Lesser General Public -License as published by the Free Software Foundation; either -version 2.1 of the License, or (at your option) any later version. - -This library is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -Lesser General Public License for more details. - -You should have received a copy of the GNU Lesser General Public -License along with this library; if not, write to the Free Software -Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -This library is really just two files: 'tinyprintf.h' and 'tinyprintf.c'. - -They provide a simple and small (+400 loc) printf functionality to -be used in embedded systems. - -I've found them so useful in debugging that I do not bother with a -debugger at all. - -They are distributed in source form, so to use them, just compile them -into your project. - -Two printf variants are provided: printf and the 'sprintf' family of -functions ('snprintf', 'sprintf', 'vsnprintf', 'vsprintf'). - -The formats supported by this implementation are: -'c' 'd' 'i' 'o' 'p' 'u' 's' 'x' 'X'. - -Zero padding and field width are also supported. - -If the library is compiled with 'PRINTF_SUPPORT_LONG' defined, then -the long specifier is also supported. Note that this will pull in some -long math routines (pun intended!) and thus make your executable -noticeably longer. Likewise with 'PRINTF_LONG_LONG_SUPPORT' for the -long long specifier, and with 'PRINTF_SIZE_T_SUPPORT' for the size_t -specifier. - -The memory footprint of course depends on the target CPU, compiler and -compiler options, but a rough guesstimate (based on a H8S target) is about -1.4 kB for code and some twenty 'int's and 'char's, say 60 bytes of stack space. -Not too bad. Your mileage may vary. By hacking the source code you can -get rid of some hundred bytes, I'm sure, but personally I feel the balance of -functionality and flexibility versus code size is close to optimal for -many embedded systems. - -To use the printf, you need to supply your own character output function, -something like : - -void putc ( void* p, char c) -{ - while (!SERIAL_PORT_EMPTY) ; - SERIAL_PORT_TX_REGISTER = c; -} - -Before you can call printf, you need to initialize it to use your -character output function with something like: - -init_printf(NULL,putc); - -Notice the 'NULL' in 'init_printf' and the parameter 'void* p' in 'putc', -the NULL (or any pointer) you pass into the 'init_printf' will eventually be -passed to your 'putc' routine. This allows you to pass some storage space (or -anything really) to the character output function, if necessary. -This is not often needed but it was implemented like that because it made -implementing the sprintf function so neat (look at the source code). - -The code is re-entrant, except for the 'init_printf' function, so it is safe -to call it from interrupts too, although this may result in mixed output. -If you rely on re-entrancy, take care that your 'putc' function is re-entrant! - -The printf and sprintf functions are actually macros that translate to -'tfp_printf' and 'tfp_sprintf' when 'TINYPRINTF_OVERRIDE_LIBC' is set -(default). Setting it to 0 makes it possible to use them along with -'stdio.h' printf's in a single source file. When -'TINYPRINTF_OVERRIDE_LIBC' is set, please note that printf/sprintf are -not function-like macros, so if you have variables or struct members -with these names, things will explode in your face. Without variadic -macros this is the best we can do to wrap these function. If it is a -problem, just give up the macros and use the functions directly, or -rename them. - -It is also possible to avoid defining tfp_printf and/or tfp_sprintf by -clearing 'TINYPRINTF_DEFINE_TFP_PRINTF' and/or -'TINYPRINTF_DEFINE_TFP_SPRINTF' to 0. This allows for example to -export only tfp_format, which is at the core of all the other -functions. - -For further details see source code. - -regs Kusti, 23.10.2004 - - -31.01.2015 -Update from Cebotari Vladislav - cebotari.vladislav@gmail.com - -- Added floating point support with different precision in x.y format - also with leading zeros possibility (like standard printf function). - Floating point printf is tested on tiva launchpad (tm4c123gh6pm TI mcu) -- Also vsscanf for floats and double %f - float, %F - double - -*/ - -#ifndef __TFP_PRINTF__ -#define __TFP_PRINTF__ - -#include - -/* Global configuration */ - - -/* Set this to 0 if you do not want to provide tfp_printf */ -#ifndef TINYPRINTF_DEFINE_TFP_PRINTF -# define TINYPRINTF_DEFINE_TFP_PRINTF 0 -#endif - -/* Set this to 0 if you do not want to provide - tfp_sprintf/snprintf/vsprintf/vsnprintf */ -#ifndef TINYPRINTF_DEFINE_TFP_SPRINTF -# define TINYPRINTF_DEFINE_TFP_SPRINTF 1 -#endif - -/* Set this to 0 if you do not want tfp_printf and - tfp_{vsn,sn,vs,s}printf to be also available as - printf/{vsn,sn,vs,s}printf */ -#ifndef TINYPRINTF_OVERRIDE_LIBC -# define TINYPRINTF_OVERRIDE_LIBC 1 -#endif - -# define TINY_PRINTF_FP_PRECISION 6 - -/* Optional external types dependencies */ - -#if TINYPRINTF_DEFINE_TFP_SPRINTF -# include /* size_t */ -#endif - -/* Declarations */ - -#ifdef __GNUC__ -# define _TFP_SPECIFY_PRINTF_FMT(fmt_idx,arg1_idx) \ - __attribute__((format (printf, fmt_idx, arg1_idx))) -#else -# define _TFP_SPECIFY_PRINTF_FMT(fmt_idx,arg1_idx) -#endif - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void (*putcf) (void *, char); - -/* - 'tfp_format' really is the central function for all tinyprintf. For - each output character after formatting, the 'putf' callback is - called with 2 args: - - an arbitrary void* 'putp' param defined by the user and - passed unmodified from 'tfp_format', - - the character. - The 'tfp_printf' and 'tfp_sprintf' functions simply define their own - callback and pass to it the right 'putp' it is expecting. -*/ -void tfp_format(void *putp, putcf putf, const char *fmt, va_list va); -# if TINYPRINTF_OVERRIDE_LIBC -# define tfp_vsscanf vsscanf -# endif -int tfp_vsscanf(const char* str, const char* format, va_list va); - -#if TINYPRINTF_DEFINE_TFP_SPRINTF -# if TINYPRINTF_OVERRIDE_LIBC -# define tfp_vsnprintf vsnprintf -# define tfp_snprintf snprintf -# define tfp_vsprintf vsprintf -# define tfp_sprintf sprintf -# endif -int tfp_vsnprintf(char *str, size_t size, const char *fmt, va_list ap); -int tfp_snprintf(char *str, size_t size, const char *fmt, ...) \ - _TFP_SPECIFY_PRINTF_FMT(3, 4); -int tfp_vsprintf(char *str, const char *fmt, va_list ap); -int tfp_sprintf(char *str, const char *fmt, ...) \ - _TFP_SPECIFY_PRINTF_FMT(2, 3); -#endif - -#if TINYPRINTF_DEFINE_TFP_PRINTF -# if TINYPRINTF_OVERRIDE_LIBC -# define tfp_printf printf -# endif -void init_printf(void *putp, putcf putf); -void tfp_printf(char *fmt, ...) _TFP_SPECIFY_PRINTF_FMT(1, 2); -#endif - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/native/src/external/Android.mk b/native/src/external/Android.mk index af88e758e..06f4d7356 100644 --- a/native/src/external/Android.mk +++ b/native/src/external/Android.mk @@ -391,3 +391,7 @@ include $(BUILD_STATIC_LIBRARY) CWD := $(LOCAL_PATH) include $(CWD)/system_properties/Android.mk include $(CWD)/libcxx/Android.mk + +ifdef B_CRT0 +include $(CWD)/crt0/Android.mk +endif diff --git a/native/src/external/crt0 b/native/src/external/crt0 new file mode 160000 index 000000000..ba8166a48 --- /dev/null +++ b/native/src/external/crt0 @@ -0,0 +1 @@ +Subproject commit ba8166a48cf9424d0fb46691c64467c20da7cdfe