From dea3c376d0aa15b22b68da0b982080f9cd23e458 Mon Sep 17 00:00:00 2001 From: "@Antelox" Date: Thu, 13 Feb 2025 11:24:04 +0100 Subject: [PATCH] CI(full),CI(release): Python bindings: (#2100) - Switched to the ubuntu-24.04-arm runner - Bumped Windows runner to windows-2022 and Visual Studio 17 2022 GENERATORS - Minor changes about checks in workflow jobs - Fixed the pagefile job (even though should not be really needed) - Refreshed the TO BE CHECKED regress tests to either update or remove the skip conditions - Added a test to check if the created sdist archive is ok --- .github/workflows/build-wheels-publish.yml | 67 ++++++++++++---------- tests/regress/arm_movr12_hang.py | 4 +- tests/regress/core_ctl.py | 3 +- tests/regress/init.py | 3 +- tests/regress/potential_memory_leak.py | 1 - tests/regress/translator_buffer.py | 5 +- 6 files changed, 43 insertions(+), 40 deletions(-) diff --git a/.github/workflows/build-wheels-publish.yml b/.github/workflows/build-wheels-publish.yml index 7bcba62c..bec9795e 100644 --- a/.github/workflows/build-wheels-publish.yml +++ b/.github/workflows/build-wheels-publish.yml @@ -45,7 +45,7 @@ jobs: fail-fast: false matrix: include: - # NOTE: aarch64 builds are super slow due to QEMU emulation. Making this to parallelize and speed up workflow + # NOTE: Doing this to parallelize and speed up workflow # i686 - manylinux - { os: ubuntu-latest, arch: i686, cibw_build: 'cp38-manylinux*', cibw_skip: '' } # i686 - musllinux @@ -55,13 +55,13 @@ jobs: # x86_64 - musllinux - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp38-musllinux*', cibw_skip: '' } # aarch64 - manylinux - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp38-manylinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp38-manylinux*', cibw_skip: '' } # aarch64 - musllinux - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp38-musllinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp38-musllinux*', cibw_skip: '' } - { os: macos-13, arch: x86_64, cibw_build: 'cp38*', cibw_skip: '' } - { os: macos-latest, arch: arm64, cibw_build: 'cp38*', cibw_skip: '' } - - { os: windows-2019, arch: AMD64, cibw_build: 'cp38*', cibw_skip: '' } - - { os: windows-2019, arch: x86, cibw_build: 'cp38*', cibw_skip: '' } + - { os: windows-2022, arch: AMD64, cibw_build: 'cp38*', cibw_skip: '' } + - { os: windows-2022, arch: x86, cibw_build: 'cp38*', cibw_skip: '' } steps: - uses: actions/checkout@v4 @@ -79,15 +79,16 @@ jobs: - name: "Increase pagefile to avoid OOM" if: runner.os == 'Windows' - uses: al-cheb/configure-pagefile-action@v1.2 + uses: al-cheb/configure-pagefile-action@v1.4 with: minimum-size: 8GB + disk-root: "C:" - name: '🛠️ Add msbuild to PATH' if: runner.os == 'Windows' uses: microsoft/setup-msbuild@v2 with: - vs-version: '16.5' + vs-version: '17' # https://cibuildwheel.pypa.io/en/stable/faq/#macos-building-cpython-38-wheels-on-arm64 - uses: actions/setup-python@v5 @@ -108,7 +109,7 @@ jobs: arch: x64 - name: '🛠️ Set up QEMU' - if: runner.os == 'Linux' && matrix.arch != 'x86_64' + if: runner.os == 'Linux' && matrix.arch == 'i686' uses: docker/setup-qemu-action@v3 - name: '🚧 cibuildwheel run' @@ -118,7 +119,7 @@ jobs: CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: ${{ matrix.cibw_skip }} CIBW_ARCHS: ${{ matrix.arch }} - CIBW_ENVIRONMENT: DEBUG=${{ env.UNICORN_DEBUG }} + CIBW_ENVIRONMENT: DEBUG=${{ env.UNICORN_DEBUG }} GENERATORS='Visual Studio 17 2022' CIBW_ENVIRONMENT_PASS_LINUX: DEBUG CIBW_TEST_EXTRAS: test CIBW_TEST_COMMAND: 'python -m unittest discover -v {project}/tests/regress "*.py"' @@ -138,13 +139,13 @@ jobs: python3 -m wheel tags --python-tag='py2' --abi-tag=none wheelhouse/*cp38*.whl - uses: LizardByte/setup-python-action@v2024.919.163656 - if: (matrix.os == 'ubuntu-latest' && matrix.arch == 'x86_64' && matrix.cibw_build == 'cp38-manylinux*') || matrix.os == 'macos-latest' || (matrix.os == 'windows-2019' && matrix.arch == 'AMD64') + if: (runner.os == 'Linux' && (matrix.arch == 'x86_64' || matrix.arch == 'aarch64') && matrix.cibw_build == 'cp38-manylinux*') || (runner.os == 'macOS' && matrix.arch == 'arm64') || (runner.os == 'Windows' && matrix.arch == 'AMD64') with: python-version: 2.7 # we install and test python2.7 wheels only on native arch - name: 'Python 2.7 tests - Windows' - if: matrix.os == 'windows-2019' && matrix.arch == 'AMD64' + if: runner.os == 'Windows' && matrix.arch == 'AMD64' shell: bash run: | C:/Python27/python.exe -m pip install capstone==4.0.2 wheelhouse/*py2*.whl @@ -152,7 +153,7 @@ jobs: # NOTE: no python2.7 support for macos-13: https://github.com/LizardByte/setup-python-action/issues/2 - name: 'Python 2.7 tests - Non-Windows' - if: (matrix.os == 'ubuntu-latest' && matrix.arch == 'x86_64' && matrix.cibw_build == 'cp38-manylinux*') || matrix.os == 'macos-latest' + if: (runner.os == 'Linux' && (matrix.arch == 'x86_64' || matrix.arch == 'aarch64') && matrix.cibw_build == 'cp38-manylinux*') || (runner.os == 'macOS' && matrix.arch == 'arm64') run: | python2 -m pip install capstone==4.0.2 wheelhouse/*py2*.whl python2 -m unittest discover tests/regress "*.py" @@ -170,7 +171,7 @@ jobs: fail-fast: false matrix: include: - # NOTE: aarch64 builds are super slow due to QEMU emulation. Making this to parallelize and speed up workflow + # NOTE: Doing this to parallelize and speed up workflow # i686 - manylinux - { os: ubuntu-latest, arch: i686, cibw_build: 'cp37-manylinux*', cibw_skip: '' } - { os: ubuntu-latest, arch: i686, cibw_build: 'cp39-manylinux*', cibw_skip: '' } @@ -200,23 +201,23 @@ jobs: - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp312-musllinux*', cibw_skip: '' } - { os: ubuntu-latest, arch: x86_64, cibw_build: 'cp313-musllinux*', cibw_skip: '' } # aarch64 - manylinux - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp37-manylinux*', cibw_skip: '' } - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp39-manylinux*', cibw_skip: '' } - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp310-manylinux*', cibw_skip: '' } - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp311-manylinux*', cibw_skip: '' } - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp312-manylinux*', cibw_skip: '' } - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp313-manylinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp37-manylinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp39-manylinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp310-manylinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp311-manylinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp312-manylinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp313-manylinux*', cibw_skip: '' } # aarch64 - musllinux - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp37-musllinux*', cibw_skip: '' } - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp39-musllinux*', cibw_skip: '' } - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp310-musllinux*', cibw_skip: '' } - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp311-musllinux*', cibw_skip: '' } - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp312-musllinux*', cibw_skip: '' } - - { os: ubuntu-22.04, arch: aarch64, cibw_build: 'cp313-musllinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp37-musllinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp39-musllinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp310-musllinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp311-musllinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp312-musllinux*', cibw_skip: '' } + - { os: ubuntu-24.04-arm, arch: aarch64, cibw_build: 'cp313-musllinux*', cibw_skip: '' } - { os: macos-13, arch: x86_64, cibw_build: 'cp*', cibw_skip: '*36* *38*' } - { os: macos-latest, arch: arm64, cibw_build: 'cp*', cibw_skip: '*36* *37* *38*' } - - { os: windows-2019, arch: AMD64, cibw_build: 'cp*', cibw_skip: '*36* *38*' } - - { os: windows-2019, arch: x86, cibw_build: 'cp*', cibw_skip: '*36* *38*' } + - { os: windows-2022, arch: AMD64, cibw_build: 'cp*', cibw_skip: '*36* *38*' } + - { os: windows-2022, arch: x86, cibw_build: 'cp*', cibw_skip: '*36* *38*' } if: ${{ inputs.fullMode == 1 || startsWith(github.ref, 'refs/tags') || contains(github.event.head_commit.message, 'CI(full)') }} steps: - uses: actions/checkout@v4 @@ -234,15 +235,16 @@ jobs: - name: "Increase pagefile to avoid OOM" if: runner.os == 'Windows' - uses: al-cheb/configure-pagefile-action@v1.2 + uses: al-cheb/configure-pagefile-action@v1.4 with: minimum-size: 8GB + disk-root: "C:" - name: '🛠️ Add msbuild to PATH' if: runner.os == 'Windows' uses: microsoft/setup-msbuild@v2 with: - vs-version: '16.5' + vs-version: '17' - name: '🛠️ Win MSVC 32 dev cmd setup' if: runner.os == 'Windows' && matrix.arch == 'x86' @@ -257,7 +259,7 @@ jobs: arch: x64 - name: '🛠️ Set up QEMU' - if: runner.os == 'Linux' && matrix.arch != 'x86_64' + if: runner.os == 'Linux' && matrix.arch == 'i686' uses: docker/setup-qemu-action@v3 - name: '🚧 cibuildwheel run' @@ -267,7 +269,7 @@ jobs: CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: ${{ matrix.cibw_skip }} CIBW_ARCHS: ${{ matrix.arch }} - CIBW_ENVIRONMENT: DEBUG=${{ env.UNICORN_DEBUG }} + CIBW_ENVIRONMENT: DEBUG=${{ env.UNICORN_DEBUG }} GENERATORS='Visual Studio 17 2022' CIBW_ENVIRONMENT_PASS_LINUX: DEBUG CIBW_TEST_EXTRAS: test CIBW_TEST_COMMAND: 'python -m unittest discover -v {project}/tests/regress "*.py"' @@ -293,6 +295,9 @@ jobs: cd bindings/python python3 -m pip install -U pip build python3 -m build --sdist + python3 -m pip install dist/*.tar.gz + cd ~ + python3 -c 'import unicorn; print(f"Unicorn version installed from sdist: {unicorn.__version__}")' - uses: actions/upload-artifact@v4 with: diff --git a/tests/regress/arm_movr12_hang.py b/tests/regress/arm_movr12_hang.py index 9e4eeba0..528e31d9 100755 --- a/tests/regress/arm_movr12_hang.py +++ b/tests/regress/arm_movr12_hang.py @@ -1,13 +1,11 @@ -import platform import regress -import unittest from unicorn import * from unicorn.arm_const import * class MovHang(regress.RegressTest): - @unittest.skipIf(platform.machine().lower() == 'aarch64', reason='TO BE CHECKED!') + # NOTE: This test was failing when workflow was using ubuntu-latest + qemu. Fixed once switched to native arm runner def runTest(self): uc = Uc(UC_ARCH_ARM, UC_MODE_ARM) uc.mem_map(0x1000, 0x1000) diff --git a/tests/regress/core_ctl.py b/tests/regress/core_ctl.py index 3a2ffc5c..0c6e7067 100644 --- a/tests/regress/core_ctl.py +++ b/tests/regress/core_ctl.py @@ -44,7 +44,7 @@ class TestCtl(regress.RegressTest): self.assertEqual(UC_CPU_ARM_CORTEX_M0, uc.ctl_get_cpu_model()) @unittest.skipIf(sys.version_info < (3, 7), reason="requires python3.7 or higher") - @unittest.skipIf(sys.platform == 'win32' or platform.machine().lower() not in ('x86_64', 'arm64'), 'TO BE CHECKED!') + @unittest.skip('TO BE CHECKED!') def test_page_size(self): SIZE_4KB = 4 * 1024 ** 1 SIZE_2MB = 2 * 1024 ** 2 @@ -78,7 +78,6 @@ class TestCtl(regress.RegressTest): # are we still with the valid value? self.assertEqual(SIZE_2MB, uc.ctl_get_page_size()) - @unittest.skipIf(platform.machine().lower() == 'aarch64', reason='TO BE CHECKED!') def test_timeout(self): MILLIS_1S = 1000 diff --git a/tests/regress/init.py b/tests/regress/init.py index fa89fb54..c59b4260 100755 --- a/tests/regress/init.py +++ b/tests/regress/init.py @@ -14,6 +14,7 @@ class Init(regress.RegressTest): def init_unicorn(self, ip, sp, counter): regress.logger.debug("[+] Emulating IP: %x SP: %x - Counter: %x" % (ip, sp, counter)) self.emulator = Uc(UC_ARCH_X86, UC_MODE_64) + self.emulator.ctl_set_tlb_mode(UC_TLB_VIRTUAL) self.emulator.mem_map(0x1000000, 2 * 1024 * 1024) self.emulator.mem_write(0x1000000, b"\x90") self.emulator.mem_map(0x8000000, 8 * 1024 * 1024) @@ -55,7 +56,7 @@ class Init(regress.RegressTest): return True @unittest.skipIf(sys.version_info < (3, 7), reason="requires python3.7 or higher") - @unittest.skipIf(sys.platform == 'win32' or platform.machine().lower() not in ('x86_64', 'arm64'), 'TO BE CHECKED!') + @unittest.skipIf(sys.platform == 'win32', 'TO BE CHECKED!') def runTest(self): ips = range(0x1000000, 0x1001000) sps = range(0x8000000, 0x8001000) diff --git a/tests/regress/potential_memory_leak.py b/tests/regress/potential_memory_leak.py index a8705eae..5e5041ce 100755 --- a/tests/regress/potential_memory_leak.py +++ b/tests/regress/potential_memory_leak.py @@ -16,7 +16,6 @@ ITERATIONS = 10000 class MemoryLeak(regress.RegressTest): @unittest.skipIf(sys.platform == 'win32', reason='Test for Unix only') - @unittest.skipIf(platform.machine().lower() == 'aarch64', reason='TO BE CHECKED!') def test(self): if platform.system() == "Darwin": rusage_multiplier = 1 diff --git a/tests/regress/translator_buffer.py b/tests/regress/translator_buffer.py index a3e440a4..abeff0ee 100755 --- a/tests/regress/translator_buffer.py +++ b/tests/regress/translator_buffer.py @@ -1,8 +1,8 @@ # By Mariano Graziano -import platform import regress import struct +import sys import unittest from unicorn import * from unicorn.x86_const import * @@ -17,6 +17,7 @@ class Emulator: self.unicorn_stack = stack self.mu = Uc(UC_ARCH_X86, UC_MODE_64) + self.mu.ctl_set_tlb_mode(UC_TLB_VIRTUAL) regress.logger.debug("mapping code : %#x", __page_aligned(code)) regress.logger.debug("mapping stack : %#x", __page_aligned(stack)) @@ -75,7 +76,7 @@ class TranslatorBuffer(regress.RegressTest): emu.emu(1) - @unittest.skipIf(platform.machine().lower() == 'aarch64', reason='TO BE CHECKED!') + @unittest.skipIf(sys.version_info < (3, 7), reason="requires python3.7 or higher") def runTest(self): ip_base = 0x000fffff816a0000 # was: 0xffffffff816a0000 sp_base = 0x000f88001b800000 # was: 0xffff88001b800000