From 2ac7b5579704091a2b64265d9f5d1db327722213 Mon Sep 17 00:00:00 2001 From: relapids Date: Mon, 15 Aug 2022 05:26:48 -0700 Subject: [PATCH 01/12] Allow building with clang-cl on Windows. --- qemu/include/qemu/atomic.h | 64 +++++++++++++++++------------------ qemu/include/qemu/int128.h | 2 ++ qemu/target/ppc/excp_helper.c | 4 +++ 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/qemu/include/qemu/atomic.h b/qemu/include/qemu/atomic.h index 3eb72dbe..4e26bedf 100644 --- a/qemu/include/qemu/atomic.h +++ b/qemu/include/qemu/atomic.h @@ -31,38 +31,38 @@ * implicit promotion. int and larger types, as well as pointers, can be * converted to a non-qualified type just by applying a binary operator. */ -#define typeof_strip_qual(expr) \ - typeof( \ - __builtin_choose_expr( \ - __builtin_types_compatible_p(typeof(expr), bool) || \ - __builtin_types_compatible_p(typeof(expr), const bool) || \ - __builtin_types_compatible_p(typeof(expr), volatile bool) || \ - __builtin_types_compatible_p(typeof(expr), const volatile bool), \ - (bool)1, \ - __builtin_choose_expr( \ - __builtin_types_compatible_p(typeof(expr), signed char) || \ - __builtin_types_compatible_p(typeof(expr), const signed char) || \ - __builtin_types_compatible_p(typeof(expr), volatile signed char) || \ - __builtin_types_compatible_p(typeof(expr), const volatile signed char), \ - (signed char)1, \ - __builtin_choose_expr( \ - __builtin_types_compatible_p(typeof(expr), unsigned char) || \ - __builtin_types_compatible_p(typeof(expr), const unsigned char) || \ - __builtin_types_compatible_p(typeof(expr), volatile unsigned char) || \ - __builtin_types_compatible_p(typeof(expr), const volatile unsigned char), \ - (unsigned char)1, \ - __builtin_choose_expr( \ - __builtin_types_compatible_p(typeof(expr), signed short) || \ - __builtin_types_compatible_p(typeof(expr), const signed short) || \ - __builtin_types_compatible_p(typeof(expr), volatile signed short) || \ - __builtin_types_compatible_p(typeof(expr), const volatile signed short), \ - (signed short)1, \ - __builtin_choose_expr( \ - __builtin_types_compatible_p(typeof(expr), unsigned short) || \ - __builtin_types_compatible_p(typeof(expr), const unsigned short) || \ - __builtin_types_compatible_p(typeof(expr), volatile unsigned short) || \ - __builtin_types_compatible_p(typeof(expr), const volatile unsigned short), \ - (unsigned short)1, \ +#define typeof_strip_qual(expr) \ + __typeof__( \ + __builtin_choose_expr( \ + __builtin_types_compatible_p(__typeof__(expr), bool) || \ + __builtin_types_compatible_p(__typeof__(expr), const bool) || \ + __builtin_types_compatible_p(__typeof__(expr), volatile bool) || \ + __builtin_types_compatible_p(__typeof__(expr), const volatile bool), \ + (bool)1, \ + __builtin_choose_expr( \ + __builtin_types_compatible_p(__typeof__(expr), signed char) || \ + __builtin_types_compatible_p(__typeof__(expr), const signed char) || \ + __builtin_types_compatible_p(__typeof__(expr), volatile signed char) || \ + __builtin_types_compatible_p(__typeof__(expr), const volatile signed char), \ + (signed char)1, \ + __builtin_choose_expr( \ + __builtin_types_compatible_p(__typeof__(expr), unsigned char) || \ + __builtin_types_compatible_p(__typeof__(expr), const unsigned char) || \ + __builtin_types_compatible_p(__typeof__(expr), volatile unsigned char) || \ + __builtin_types_compatible_p(__typeof__(expr), const volatile unsigned char), \ + (unsigned char)1, \ + __builtin_choose_expr( \ + __builtin_types_compatible_p(__typeof__(expr), signed short) || \ + __builtin_types_compatible_p(__typeof__(expr), const signed short) || \ + __builtin_types_compatible_p(__typeof__(expr), volatile signed short) || \ + __builtin_types_compatible_p(__typeof__(expr), const volatile signed short), \ + (signed short)1, \ + __builtin_choose_expr( \ + __builtin_types_compatible_p(__typeof__(expr), unsigned short) || \ + __builtin_types_compatible_p(__typeof__(expr), const unsigned short) || \ + __builtin_types_compatible_p(__typeof__(expr), volatile unsigned short) || \ + __builtin_types_compatible_p(__typeof__(expr), const volatile unsigned short), \ + (unsigned short)1, \ (expr)+0)))))) #ifdef __ATOMIC_RELAXED diff --git a/qemu/include/qemu/int128.h b/qemu/include/qemu/int128.h index d32511a6..d1d1ad44 100644 --- a/qemu/include/qemu/int128.h +++ b/qemu/include/qemu/int128.h @@ -146,7 +146,9 @@ static inline Int128 bswap128(Int128 a) #else /* !CONFIG_INT128 */ typedef struct Int128 Int128; +#if !(defined(_MSC_VER) && defined(__clang__)) typedef Int128 __int128_t; +#endif struct Int128 { uint64_t lo; diff --git a/qemu/target/ppc/excp_helper.c b/qemu/target/ppc/excp_helper.c index beaef6cd..f1114b5b 100644 --- a/qemu/target/ppc/excp_helper.c +++ b/qemu/target/ppc/excp_helper.c @@ -1061,7 +1061,11 @@ void helper_store_msr(CPUPPCState *env, target_ulong val) } #if defined(TARGET_PPC64) +#if defined(_MSC_VER) && defined(__clang__) +void helper_pminsn(CPUPPCState *env, uint32_t insn) +#else void helper_pminsn(CPUPPCState *env, powerpc_pm_insn_t insn) +#endif { CPUState *cs; From 5a54b3d7af7fd2477bce9ac61485d0ea260795cb Mon Sep 17 00:00:00 2001 From: relapids Date: Mon, 15 Aug 2022 06:45:25 -0700 Subject: [PATCH 02/12] Fix a segfault inside tb_remove_from_jmp_list by forcing clang-cl to use the same atomic routines as MSVC. --- qemu/include/qemu/atomic.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qemu/include/qemu/atomic.h b/qemu/include/qemu/atomic.h index 4e26bedf..84922100 100644 --- a/qemu/include/qemu/atomic.h +++ b/qemu/include/qemu/atomic.h @@ -65,7 +65,7 @@ (unsigned short)1, \ (expr)+0)))))) -#ifdef __ATOMIC_RELAXED +#if defined(__ATOMIC_RELAXED) && !(defined(_MSC_VER) && defined(__clang__)) /* For C11 atomic ops */ /* Sanity check that the size of an atomic operation isn't "overly large". From fb802575d7a8c37b5a3be00164f9c65c3f9241d1 Mon Sep 17 00:00:00 2001 From: Lily Wang <494550702@qq.com> Date: Fri, 9 Sep 2022 01:22:17 -0700 Subject: [PATCH 03/12] Add vcpkg installation instructions --- docs/COMPILE.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/COMPILE.md b/docs/COMPILE.md index c2e475d5..ad16bcdf 100644 --- a/docs/COMPILE.md +++ b/docs/COMPILE.md @@ -128,3 +128,17 @@ mkdir build; cd build cmake .. -DCMAKE_C_COMPILER=gcc-arm-linux-gnueabihf make ``` + +## Building from vcpkg + +The Unicorn port in vcpkg is kept up to date by Microsoft team members and community contributors. The url of vcpkg is: https://github.com/Microsoft/vcpkg . You can download and install unicorn using the vcpkg dependency manager: + +```bash +git clone https://github.com/Microsoft/vcpkg.git +cd vcpkg +./bootstrap-vcpkg.sh # ./bootstrap-vcpkg.bat for Windows +./vcpkg integrate install +./vcpkg install unicorn +``` + +If the version is out of date, please [create an issue or pull request](https://github.com/Microsoft/vcpkg) on the vcpkg repository. From 325d02e64197374129ac210efa9a0f50eeb588c8 Mon Sep 17 00:00:00 2001 From: mio Date: Sun, 25 Sep 2022 17:17:45 +0200 Subject: [PATCH 04/12] Use ANDROID_NDK --- .github/workflows/build-uc2.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-uc2.yml b/.github/workflows/build-uc2.yml index 5f34b64f..7ee95894 100644 --- a/.github/workflows/build-uc2.yml +++ b/.github/workflows/build-uc2.yml @@ -328,9 +328,9 @@ jobs: brew install p7zip cmake ninja mkdir build mkdir instdir - cmake . -DCMAKE_TOOLCHAIN_FILE="$ANDROID_HOME/ndk/25.0.8775105/build/cmake/android.toolchain.cmake" \ + cmake . -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \ -DANDROID_PLATFORM=android-28 \ - -DANDROID_NDK="$ANDROID_HOME/ndk/25.0.8775105" \ + -DANDROID_NDK="$ANDROID_NDK" \ -DANDROID_ABI=${{ matrix.config.arch }} \ -DOLP_SDK_ENABLE_TESTING=NO \ -DOLP_SDK_BUILD_EXAMPLES=ON \ From 32a3a6865a636d520a90138f48f0023fe51d6d3a Mon Sep 17 00:00:00 2001 From: mio Date: Sun, 25 Sep 2022 17:41:09 +0200 Subject: [PATCH 05/12] Don't resize user alloc-ed memory --- qemu/exec.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/qemu/exec.c b/qemu/exec.c index 719895a3..2a664788 100644 --- a/qemu/exec.c +++ b/qemu/exec.c @@ -1087,8 +1087,12 @@ RAMBlock *qemu_ram_alloc_from_ptr(struct uc_struct *uc, ram_addr_t size, void *h RAMBlock *new_block; ram_addr_t max_size = size; - size = HOST_PAGE_ALIGN(uc, size); - max_size = HOST_PAGE_ALIGN(uc, max_size); + // Don't resize pre-alloced memory as they are given by users. + if (!host) { + size = HOST_PAGE_ALIGN(uc, size); + max_size = HOST_PAGE_ALIGN(uc, max_size); + } + new_block = g_malloc0(sizeof(*new_block)); if (new_block == NULL) return NULL; From e76b2db434382f59661471faae02b022a3ee5a30 Mon Sep 17 00:00:00 2001 From: mio Date: Sun, 25 Sep 2022 18:09:41 +0200 Subject: [PATCH 06/12] Support build in a sdist --- bindings/python/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 51dfb9fe..8920a2eb 100755 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -26,7 +26,7 @@ ROOT_DIR = os.path.dirname(os.path.realpath(__file__)) LIBS_DIR = os.path.join(ROOT_DIR, 'unicorn', 'lib') HEADERS_DIR = os.path.join(ROOT_DIR, 'unicorn', 'include') SRC_DIR = os.path.join(ROOT_DIR, 'src') -UC_DIR = os.path.join(ROOT_DIR, '../..') +UC_DIR = SRC_DIR if os.path.exists(SRC_DIR) else os.path.join(ROOT_DIR, '../..') BUILD_DIR = os.path.join(UC_DIR, 'build_python') VERSION = "2.0.0" From e1e7b252686205006c4d10f66d66857c4f045784 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20R=C3=B6hling?= Date: Sat, 24 Sep 2022 13:58:23 +0200 Subject: [PATCH 07/12] Adjust big memory test for host pagesize On machines with a page size larger than 4K, the requested memory size in `test_map_big_memory` gets rounded up and overflows to zero. This PR adds some code to query the page size and adjust the requested memory size accordingly. --- tests/unit/test_mem.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_mem.c b/tests/unit/test_mem.c index a1db14e1..8910fdcf 100644 --- a/tests/unit/test_mem.c +++ b/tests/unit/test_mem.c @@ -184,8 +184,15 @@ static void test_map_big_memory(void) OK(uc_open(UC_ARCH_X86, UC_MODE_64, &uc)); +#if defined(_WIN32) || defined(__WIN32__) || defined(__WINDOWS__) + uint64_t requested_size = 0xfffffffffffff000; // assume 4K page size +#else + long ps = sysconf(_SC_PAGESIZE); + uint64_t requested_size = (uint64_t)(-ps); +#endif + uc_assert_err(UC_ERR_NOMEM, - uc_mem_map(uc, 0x0, 0xfffffffffffff000, UC_PROT_ALL)); + uc_mem_map(uc, 0x0, requested_size, UC_PROT_ALL)); OK(uc_close(uc)); } From a98cd25747ed933136ecc2f4b48874aa0350a8f3 Mon Sep 17 00:00:00 2001 From: Sh4w Date: Mon, 26 Sep 2022 00:18:31 +0800 Subject: [PATCH 08/12] Fix type hint of reg_write in Python bindings --- bindings/python/unicorn/unicorn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/unicorn/unicorn.py b/bindings/python/unicorn/unicorn.py index 1687ab63..2e6a938f 100644 --- a/bindings/python/unicorn/unicorn.py +++ b/bindings/python/unicorn/unicorn.py @@ -560,7 +560,7 @@ class Uc(object): return reg_read(functools.partial(_uc.uc_reg_read, self._uch), self._arch, reg_id, opt) # write to a register, tuple for arm cp regs. - def reg_write(self, reg_id: Union[int, ARMCPRegValue, ARM64CPRegValue, X86MMRReg, X86FPReg], value: int): + def reg_write(self, reg_id: int, value: Union[int, ARMCPRegValue, ARM64CPRegValue, X86MMRReg, X86FPReg]): return reg_write(functools.partial(_uc.uc_reg_write, self._uch), self._arch, reg_id, value) # read from MSR - X86 only From 97b2e44c777059eff850bf9a9c24994b647f7c04 Mon Sep 17 00:00:00 2001 From: mio Date: Sun, 25 Sep 2022 18:43:09 +0200 Subject: [PATCH 09/12] Also copy cmake files --- bindings/python/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 8920a2eb..20bc2440 100755 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -69,6 +69,7 @@ def copy_sources(): src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.[ch]"))) src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.mk"))) + src.extend(glob.glob(os.path.join(ROOT_DIR, "../../*.cmake"))) src.extend(glob.glob(os.path.join(ROOT_DIR, "../../LICENSE*"))) src.extend(glob.glob(os.path.join(ROOT_DIR, "../../README.md"))) From 7fb6264d02ef410b240d697dc74e03e3f85385f6 Mon Sep 17 00:00:00 2001 From: mio Date: Sun, 25 Sep 2022 18:46:04 +0200 Subject: [PATCH 10/12] Also copy glib_compat and samples --- bindings/python/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bindings/python/setup.py b/bindings/python/setup.py index 20bc2440..f8948d62 100755 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -60,6 +60,9 @@ def copy_sources(): shutil.copytree(os.path.join(ROOT_DIR, '../../include'), os.path.join(SRC_DIR, 'include/')) # make -> configure -> clean -> clean tests fails unless tests is present shutil.copytree(os.path.join(ROOT_DIR, '../../tests'), os.path.join(SRC_DIR, 'tests/')) + shutil.copytree(os.path.join(ROOT_DIR, '../../samples'), os.path.join(SRC_DIR, 'samples/')) + shutil.copytree(os.path.join(ROOT_DIR, '../../glib_compat'), os.path.join(SRC_DIR, 'glib_compat/')) + try: # remove site-specific configuration file # might not exist From 1065c2dff43f026e7f9ea924115ebac8b0cebe45 Mon Sep 17 00:00:00 2001 From: relapids Date: Tue, 16 Aug 2022 00:06:02 -0700 Subject: [PATCH 11/12] Fix test_uc_hook_cached_uaf for MacOS M1 (aarch64). --- tests/unit/test_ctl.c | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_ctl.c b/tests/unit/test_ctl.c index b96d7cf5..5eef28d0 100644 --- a/tests/unit/test_ctl.c +++ b/tests/unit/test_ctl.c @@ -251,15 +251,22 @@ static void test_uc_hook_cached_uaf(void) uc_hook h; uint64_t count = 0; #ifndef _WIN32 - void *callback = mmap(NULL, 4096, PROT_READ | PROT_WRITE | PROT_EXEC, + // Apple Silicon does not allow RWX pages. + void *callback = mmap(NULL, 4096, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + TEST_CHECK(callback != (void*)-1); #else void *callback = VirtualAlloc(NULL, 4096, MEM_RESERVE | MEM_COMMIT, PAGE_EXECUTE_READWRITE); + TEST_CHECK(callback != NULL); #endif memcpy(callback, (void *)test_uc_hook_cached_cb, 4096); +#ifndef _WIN32 + TEST_CHECK(mprotect(callback, 4096, PROT_READ | PROT_EXEC) == 0); +#endif + uc_common_setup(&uc, UC_ARCH_X86, UC_MODE_32, code, sizeof(code) - 1); OK(uc_hook_add(uc, &h, UC_HOOK_CODE, (void *)callback, (void *)&count, 1, @@ -273,8 +280,16 @@ static void test_uc_hook_cached_uaf(void) // This will clear deleted hooks and SHOULD clear cache. OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0)); +#ifndef _WIN32 + TEST_CHECK(mprotect(callback, 4096, PROT_READ | PROT_WRITE) == 0); +#endif + memset(callback, 0, 4096); +#ifndef _WIN32 + TEST_CHECK(mprotect(callback, 4096, PROT_READ | PROT_EXEC) == 0); +#endif + // Now hooks are deleted and thus this will trigger a UAF OK(uc_emu_start(uc, code_start, code_start + sizeof(code) - 1, 0, 0)); From b0b3a57bcdd775ba8629be124deae8f7e6036464 Mon Sep 17 00:00:00 2001 From: mio Date: Tue, 27 Sep 2022 23:13:06 +0200 Subject: [PATCH 12/12] Update FAQ --- docs/FAQ.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index 4bb568ce..20bb58e2 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -30,6 +30,13 @@ On x86, all available instructions are: `in` `out` `syscall` `sysenter` `cpuid`. If you are still using Unicorn1, please upgrade to Unicorn2 for better support. +## Memory hooks get called multiple times for a single instruction + +There are several possibilities, e.g.: + +- The instruction might access memory multiple times like `rep stos` in x86. +- The address to access is bad-aligned and thus the MMU emulation will split the access into several aligned memory access. In worst cases on some arch, it leads to byte by byte access. + ## I can't recover from unmapped read/write even I return `true` in the hook, why? This is a minor change in memory hooks behavior between Unicorn1 and Unicorn2. To gracefully recover from memory read/write error, you have to map the invalid memory before you return true. @@ -38,9 +45,11 @@ It is due to the fact that, if users return `true` without memory mapping set up See the [sample](https://github.com/unicorn-engine/unicorn/blob/c05fbb7e63aed0b60fc2888e08beceb17bce8ac4/samples/sample_x86.c#L1379-L1393) for details. -## My MIPS emulation gets weird read/write error and CPU exceptions. +## My emulation gets weird read/write error and CPU exceptions. -Note you might have an address that falls in MIPS `kseg` segments. In that case, MMU is bypassed and you have to make sure the corresponding physical memory is mapped. See [#217](https://github.com/unicorn-engine/unicorn/issues/217), [#1371](https://github.com/unicorn-engine/unicorn/issues/1371), [#1550](https://github.com/unicorn-engine/unicorn/issues/1371). +For MIPS, you might have an address that falls in MIPS `kseg` segments. In that case, MMU is bypassed and you have to make sure the corresponding physical memory is mapped. See [#217](https://github.com/unicorn-engine/unicorn/issues/217), [#1371](https://github.com/unicorn-engine/unicorn/issues/1371), [#1550](https://github.com/unicorn-engine/unicorn/issues/1371). + +For ARM, you might have an address that falls in some non-executable segments. For example, for m-class ARM cpu, some memory area is not executable according to [the ARM document](https://developer.arm.com/documentation/ddi0403/d/System-Level-Architecture/System-Address-Map/The-system-address-map?lang=en). ## KeyboardInterrupt is not raised during `uc.emu_start`