From 0c715b1e75c7c934354998cbc097d6469134be17 Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:37:17 -0700 Subject: [PATCH 01/19] Normalize line endings with .gitattributes Files were a mix of LF and CRLF, so edits from Windows showed as whole-file diffs. Store everything as LF; checkouts get the platform's line endings. No content changes. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 2 + CompileTimePattern.hpp | 274 ++++++------- Pattern.hpp | 888 ++++++++++++++++++++--------------------- README.md | 86 ++-- RuntimePattern.hpp | 152 +++---- XORPattern.hpp | 232 +++++------ 6 files changed, 818 insertions(+), 816 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1aeec22 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Stored with LF; checked out with the platform's line endings +* text=auto diff --git a/CompileTimePattern.hpp b/CompileTimePattern.hpp index 14af549..711d454 100644 --- a/CompileTimePattern.hpp +++ b/CompileTimePattern.hpp @@ -1,137 +1,137 @@ -#pragma once -#include "Pattern.hpp" -#include - -namespace patterns { - - namespace detail { - constexpr size_t pattern_length(const char* s, size_t nstr, bool pad = true) { - size_t res = 0; -#ifdef __arm64__ - bool align = false; -#endif - for (auto i = 0; i < nstr - 1; i += 2) { - auto c = s[i]; - if (c == 'X' || c == 'x') { - while (s[i + 1] != ' ') - ++i; - continue; - } - else if (c == '/') { -#ifdef __arm64__ - while (i < nstr - 1 && (c = s[++i])) { - if (c == 'a') { - align = true; - break; - } - } -#endif - break; - } - else if (c != ' ') - ++res; - else --i; - } -#ifdef __arm64__ - while (pad && align && res % sizeof(uint32_t)) - ++res; -#else - while (pad && res % sizeof(void*)) - ++res; -#endif - return res; - } - - template - constexpr size_t pattern_length(const char(&s)[nstr], bool pad = true) { - return pattern_length(s, nstr, pad); - } - -#if __cplusplus > 201703L - template - struct const_string { - char data[nstr]{}; - static constexpr size_t length = nstr - 1; - static constexpr size_t size = nstr; - - constexpr const_string(const char(&s)[nstr]) { - std::copy_n(s, length, data); - } - }; -#endif - } - - template - class CompileTimePattern : public Pattern { - std::array m_pattern; - std::array m_mask; - public: - constexpr CompileTimePattern(const char* p) : - m_pattern{}, m_mask{} - { - length_ = narr; - auto n = 0; - for (auto i = 0; i < nstr; i += 2) { - auto ptr = &p[i]; - if (p[i] == '?') { - m_pattern[n] = 0; - m_mask[n] = 0; - ++n; - } - // Capture where we have our offset marker 'X' at - else if (*ptr == 'X' || *ptr == 'x') { - offset_ = n; - if (p[i + 1] != ' ') { -#ifndef __arm64__ - insn_len_ = get_inst_len_opt(&p[++i]); -#endif - while (p[i + 1] != ' ') - ++i; - } - } - // Break from parsing the pattern, since / at the end starts the flags - else if (*ptr == '/') { - ++ptr; - handle_options(ptr); - break; - } - else if (*ptr != ' ') { - m_pattern[n] = value(ptr); - m_mask[n] = 0xFF; - ++n; - } - else --i; - } -#ifdef __arm64__ - while (align_ && n % align_size_) { - m_pattern[n] = 0; - m_mask[n] = 0; - ++n; - } -#else - while (n % sizeof(void*)) { - m_pattern[n] = 0; - m_mask[n] = 0; - ++n; - } -#endif - } - virtual const uint8_t* pattern() const override { - return m_pattern.data(); - } - virtual const uint8_t* mask() const override { - return m_mask.data(); - } - }; -} - -#if __cplusplus > 201703L -template -constexpr auto operator"" _ctpattern() { - return patterns::CompileTimePattern(str.data); -} -#else -#ifndef COMPILETIME_PATTERN -#define COMPILETIME_PATTERN(x) patterns::CompileTimePattern(x) -#endif -#endif +#pragma once +#include "Pattern.hpp" +#include + +namespace patterns { + + namespace detail { + constexpr size_t pattern_length(const char* s, size_t nstr, bool pad = true) { + size_t res = 0; +#ifdef __arm64__ + bool align = false; +#endif + for (auto i = 0; i < nstr - 1; i += 2) { + auto c = s[i]; + if (c == 'X' || c == 'x') { + while (s[i + 1] != ' ') + ++i; + continue; + } + else if (c == '/') { +#ifdef __arm64__ + while (i < nstr - 1 && (c = s[++i])) { + if (c == 'a') { + align = true; + break; + } + } +#endif + break; + } + else if (c != ' ') + ++res; + else --i; + } +#ifdef __arm64__ + while (pad && align && res % sizeof(uint32_t)) + ++res; +#else + while (pad && res % sizeof(void*)) + ++res; +#endif + return res; + } + + template + constexpr size_t pattern_length(const char(&s)[nstr], bool pad = true) { + return pattern_length(s, nstr, pad); + } + +#if __cplusplus > 201703L + template + struct const_string { + char data[nstr]{}; + static constexpr size_t length = nstr - 1; + static constexpr size_t size = nstr; + + constexpr const_string(const char(&s)[nstr]) { + std::copy_n(s, length, data); + } + }; +#endif + } + + template + class CompileTimePattern : public Pattern { + std::array m_pattern; + std::array m_mask; + public: + constexpr CompileTimePattern(const char* p) : + m_pattern{}, m_mask{} + { + length_ = narr; + auto n = 0; + for (auto i = 0; i < nstr; i += 2) { + auto ptr = &p[i]; + if (p[i] == '?') { + m_pattern[n] = 0; + m_mask[n] = 0; + ++n; + } + // Capture where we have our offset marker 'X' at + else if (*ptr == 'X' || *ptr == 'x') { + offset_ = n; + if (p[i + 1] != ' ') { +#ifndef __arm64__ + insn_len_ = get_inst_len_opt(&p[++i]); +#endif + while (p[i + 1] != ' ') + ++i; + } + } + // Break from parsing the pattern, since / at the end starts the flags + else if (*ptr == '/') { + ++ptr; + handle_options(ptr); + break; + } + else if (*ptr != ' ') { + m_pattern[n] = value(ptr); + m_mask[n] = 0xFF; + ++n; + } + else --i; + } +#ifdef __arm64__ + while (align_ && n % align_size_) { + m_pattern[n] = 0; + m_mask[n] = 0; + ++n; + } +#else + while (n % sizeof(void*)) { + m_pattern[n] = 0; + m_mask[n] = 0; + ++n; + } +#endif + } + virtual const uint8_t* pattern() const override { + return m_pattern.data(); + } + virtual const uint8_t* mask() const override { + return m_mask.data(); + } + }; +} + +#if __cplusplus > 201703L +template +constexpr auto operator"" _ctpattern() { + return patterns::CompileTimePattern(str.data); +} +#else +#ifndef COMPILETIME_PATTERN +#define COMPILETIME_PATTERN(x) patterns::CompileTimePattern(x) +#endif +#endif diff --git a/Pattern.hpp b/Pattern.hpp index 7be9ad2..8af24e1 100644 --- a/Pattern.hpp +++ b/Pattern.hpp @@ -1,444 +1,444 @@ -#pragma once -#include -#include - -#ifndef _MSVC_VER -#define __forceinline inline __attribute__((always_inline)) -#endif - -namespace patterns { - namespace detail { - constexpr bool __forceinline is_digit(char c) { - return c <= '9' && c >= '0'; - } - constexpr bool __forceinline is_hex_digit(char c) { - return is_digit(c) || ((c | 0x20) >= 'a' && (c | 0x20) <= 'f'); - } - constexpr __forceinline char get_bits(char c) { - return is_digit(c) ? (c - '0') : ((c & (~0x20)) - 'A' + 0xA); - } - template - constexpr T stoi_impl(const char* str, T value = 0, bool negative = false, bool hex = false) { - if (*str == '\0') return negative ? -value : value; - if (hex) { - if (is_hex_digit(*str)) - return stoi_impl(str + 1, get_bits(*str) + value * 16, negative, hex); - } else { - if (is_digit(*str)) - return stoi_impl(str + 1, (*str - '0') + value * 10, negative, hex); - } - return negative ? -value : value; - } - template - constexpr T stoi(const char* str) { - if (*str == '-') { - ++str; - if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) - return stoi_impl(str + 2, 0, true, true); - return stoi_impl(str, 0, true); - } - if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) - return stoi_impl(str + 2, 0, false, true); - return stoi_impl(str); - } -#ifdef __arm64__ - template - T extract_bitfield(uint32_t insn, unsigned width, unsigned offset) { - constexpr size_t int_width = sizeof(int32_t) * 8; - return (static_cast(insn) << (int_width - (offset + width))) >> (int_width - width); - } - - bool __forceinline decode_masked_match(uint32_t insn, uint32_t mask, uint32_t pattern) { - return (insn & mask) == pattern; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/NOP--No-Operation- - bool a64_decode_nop(uint32_t insn) { - if (insn == 0b1101'0101'0000'0011'0010'0000'0001'1111) - return true; - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/B--Branch- - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/BL--Branch-with-Link- - bool a64_decode_b(uint32_t insn, bool* is_bl, int64_t* offset) { - // x001 01?? ???? ???? ???? ???? ???? ???? - // 1 - BL - // 0 - B - if (decode_masked_match(insn, 0b0111'1100'0000'0000'0000'0000'0000'0000, 0b0001'0100'0000'0000'0000'0000'0000'0000)) { - if (is_bl) *is_bl = (insn >> 31) & 0x1; - *offset = extract_bitfield(insn, 26, 0) << 2; - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADR--Form-PC-relative-address- - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADRP--Form-PC-relative-address-to-4KB-page- - bool a64_decode_adr(uint32_t insn, bool* is_adrp, unsigned* rd, int64_t* offset) { - // rd - destination register - // x??1 0000 ???? ???? ???? ???? ???? ???? - // 1 - ADRP - // 0 - ADR - // bits 30 & 29 - immlo - // bits 23 -> 5 - immhi - // bits 4 -> 0 - rd - if (decode_masked_match(insn, 0b0001'1111'0000'0000'0000'0000'0000'0000, 0b0001'0000'0000'0000'0000'0000'0000'0000)) { - if (rd) *rd = insn & 0x1f; - uint32_t immlo = (insn >> 29) & 0x3; - int32_t immhi = extract_bitfield(insn, 19, 5) << 2; - auto adrp = (insn >> 31) & 0x1; - if (is_adrp) *is_adrp = adrp; - *offset = (adrp ? (immhi | immlo) * 4096 : (immhi | immlo)); - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate-- - bool a64_decode_ldrh(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { - // rn - name of general-purpose base register or stack pointer - // rt - name of general-purpose base register to be transfered - // Post-index & Pre-index masks - // 0111 1000 010? ???? ???? x1?? ???? ???? - // 1 - pre-index - // 0 - post-index - if (decode_masked_match(insn, 0b1111'1111'1110'0000'0000'0100'0000'0000, 0b0111'1000'0100'0000'0000'0100'0000'0000)) { - *offset = extract_bitfield(insn, 9, 12); - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - // Unsigned offset mask - // 0111 1001 01?? ???? ???? ???? ???? ???? - if (decode_masked_match(insn, 0b1111'1111'1100'0000'0000'0000'0000'0000, 0b0111'1001'0100'0000'0000'0000'0000'0000)) { - *offset = extract_bitfield(insn, 12, 10) << 1; - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/LDR--immediate---Load-Register--immediate-- - bool a64_decode_ldr(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { - // rn - name of general-purpose base register or stack pointer - // rt - name of general-purpose register to be transferred - // Post-index & Pre-index masks - // 1x11 1000 010? ???? ???? x1?? ???? ???? - // 1 - 64 bit 1 - pre-index - // 0 - 32 bit 0 - post-index - if (decode_masked_match(insn, 0b1011'1111'1110'0000'0000'0100'0000'0000, 0b1011'1000'0100'0000'0000'0100'0000'0000)) { - // Have not tested this yet - *offset = extract_bitfield(insn, 9, 12); - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - // Unsigned offset mask - // 1x11 1001 01?? ???? ???? ???? ???? ???? - // 1 - 64 bit - // 0 - 32 bit - if (decode_masked_match(insn, 0b1011'1011'1100'0000'0000'0000'0000'0000, 0b1011'1001'0100'0000'0000'0000'0000'0000)) { - *offset = extract_bitfield(insn, 12, 10) << (insn >> 30); - // bit 26 annotated as "VR" - Variant Register (Not sure it's use yet) - //auto vr = (insn >> 26) & 1; - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/STR--immediate---Store-Register--immediate-- - bool a64_decode_str(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { - // rn - name of general-purpose base register or stack pointer - // rt - name of general-purpose register to be transferred - // Post-index & Pre-index masks - // 1x11 1000 000? ???? ???? x1?? ???? ???? - // 1 - 64 bit 1 - pre-index - // 0 - 32 bit 0 - post-index - if (decode_masked_match(insn, 0b1011'1011'1110'0000'0000'0100'0000'0000, 0b1011'1000'0000'0000'0000'0100'0000'0000)) { - *offset = extract_bitfield(insn, 9, 12); - // bit 26 annotated as "VR" - Variant Register (Not sure it's use yet) - //auto vr = (insn >> 26) & 1; - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - // Unsigned offset mask - // 1x11 1000 000? ???? ???? ???? ???? ???? - // 1 - 64 bit - // 0 - 32 bit - if (decode_masked_match(insn, 0b1011'1011'1110'0000'0000'0000'0000'0000, 0b1011'1001'0000'0000'0000'0000'0000'0000)) { - *offset = extract_bitfield(insn, 12, 10) << (insn >> 30); - // bit 26 annotated as "VR" - Variant Register (Not sure it's use yet) - //auto vr = (insn >> 26) & 1; - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/MOVZ--Move-wide-with-zero- - bool a64_decode_movz(uint32_t insn, unsigned* sf, unsigned* rd, int64_t* offset) { - // sf - 1 for 64 bit, 0 for 32 bit - // rd - destination register - // x101 0010 1xx? ???? ???? ???? ???? ???? - // 1 = 64 bit 0x - shift ammount - // 0 = 32 bit - if (decode_masked_match(insn, 0b0111'1111'1000'0000'0000'0000'0000'0000, 0b0101'0010'1000'0000'0000'0000'0000'0000)) { - auto hw = (insn >> 21) & 3; - *offset = hw ? extract_bitfield(insn, 16, 5) << hw : extract_bitfield(insn, 16, 5); - if (sf) *sf = (insn >> 31) & 0x1; - if (rd) *rd = insn & 0x1f; - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADD--immediate---Add--immediate-- - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/SUB--immediate---Subtract--immediate-- - bool a64_decode_arithmetic(uint32_t insn, bool* is_sub, unsigned* sf, unsigned* rd, unsigned* rn, int64_t* offset) { - // rd - destination register - // rn - source register - // sf - 1 for 64 bit, 0 for 32 bit - // ?x01 0001 0??? ???? ???? ???? ???? ???? - // 1 = subtraction - // 0 = addition - if (decode_masked_match(insn, 0b0011'1111'1000'0000'0000'0000'0000'0000, 0b0001'0001'0000'0000'0000'0000'0000'0000)) { - if (sf) *sf = (insn >> 31) & 0x1; - if (rd) *rd = insn & 0x1f; - if (rn) *rn = (insn >> 5) & 0x1f; - if (is_sub) *is_sub = (insn >> 30) & 0x1; - uint32_t imm12 = extract_bitfield(insn, 12, 10); - // sh = 1 is LSL#12 - auto sh = (insn >> 22) & 0x1; - *offset = (sh ? (imm12 << 12) : imm12); - return true; - } - return false; - } -#endif - } - - class Pattern { - protected: - uint32_t length_ = 0; - uint32_t offset_ = 0; - bool deref_ = false; -#ifndef __arm64__ - // Remaining bytes from the offset when used in dereferencing - uint32_t insn_len_ = 0; - // Since all arm64 instructions are 32 bit and encoded, just doing deref instead of both deref and relative - // Defaulting 4 byte relative address reading - uint32_t size_ = 4; - bool rel_ = false; -#endif - bool align_ = false; -#ifdef __arm64__ - // arm64 instructions are all 32 bits, so the align scanning will be at the instruction level - static constexpr size_t align_size_ = 4; -#else - static constexpr size_t align_size_ = sizeof(void*); -#endif - public: - Pattern() = default; - ~Pattern() = default; - const uint32_t __forceinline length() const { - return length_; - } - const uint32_t __forceinline offset() const { - return offset_; - } - const bool __forceinline deref() const { - return deref_; - } -#ifndef __arm64__ - const bool __forceinline relative() const { - return rel_; - } -#endif - const bool __forceinline aligned() const { - return align_; - } - virtual const uint8_t* mask() const = 0; - virtual const uint8_t* pattern() const = 0; - template - T find(const uint8_t* bytes, size_t size) const { - return reinterpret_cast(find(bytes, size)); - } - virtual void* find(const uint8_t* bytes, size_t size) const { - void* result = nullptr; - const auto pattern = const_cast(this->pattern()); - const auto mask = const_cast(this->mask()); - const auto end = bytes + size - length_; - for (auto i = const_cast(bytes); i < end; align_ ? i += align_size_ : ++i) { - bool found = true; -#ifdef __arm64__ - // Doing byte by byte match due to arm being encoded instructions, unless specified to align scan - if (align_) { - for (auto j = 0U; j < length_; j += align_size_) { - const auto data = *reinterpret_cast(pattern + j); - const auto msk = *reinterpret_cast(mask + j); - const auto mem = *reinterpret_cast(i + j); - if ((data ^ mem) & msk) { - found = false; - break; - } - } - } else { - for (auto j = 0U; j < length_; ++j) { - if (mask[j] == 0xFF && pattern[j] != i[j]) { - found = false; - break; - } - } - } -#else - for (auto j = 0U; j < length_; j += align_size_) { - const auto data = *reinterpret_cast(pattern + j); - const auto msk = *reinterpret_cast(mask + j); - const auto mem = *reinterpret_cast(i + j); - if ((data ^ mem) & msk) { - found = false; - break; - } - } -#endif - if (found) - return get_result(i, end); - } - return result; - } - virtual uint8_t operator[](size_t idx) const { - return pattern()[idx]; - } - protected: - // Credits to EJT for the helper functions here! - constexpr __forceinline uint8_t value(const char* c) const { - return (detail::get_bits(c[0]) << 4 | detail::get_bits(c[1])); - } - constexpr void handle_options(const char* ptr) { - while (*ptr) { -#ifdef __arm64__ - if (*ptr == 'd') - deref_ = true; -#else - if (*ptr == 'd') { - if (rel_) throw std::logic_error("Cannot use relative and deref together!"); - deref_ = true; - } - else if (*ptr == 'r') { - if (deref_) throw std::logic_error("Cannot use relative and deref together!"); - rel_ = true; - } -#endif - else if (*ptr == 'a') - align_ = true; -#ifndef __arm64__ - // Check the next character to see what size we're reading at this relative address - else if (*ptr > '0' && (sizeof(void*) == 0x8 ? *ptr < '9' : *ptr < '5')) { - size_ = *ptr - '0'; - if ((size_ & (size_ - 1)) != 0) throw std::logic_error("Size is not a valid data type size!"); - } -#endif - ++ptr; - } - } - void* get_result(uint8_t* address, const uint8_t* end) const { -#ifdef __arm64__ - if (deref_) { - auto insn = reinterpret_cast(address + offset_); - int64_t offset = 0; - bool is_sub = false, is_adrp = false; - unsigned rd = 0, sf = 0, rn = 0; - if (detail::a64_decode_b(*insn, nullptr, &offset)) { - return reinterpret_cast(reinterpret_cast(insn) + offset); - } - else if (detail::a64_decode_adr(*insn, &is_adrp, &rd, &offset)) { - auto saved_offset = offset; - auto saved_rd = rd; - auto curr_insn = insn + 1; - if (is_adrp) { - while (*curr_insn) { - unsigned rt = 0; - if (detail::a64_decode_arithmetic(*curr_insn, &is_sub, &sf, &rd, &rn, &offset)) { - if (saved_rd == rd && rn == rd) { - uint64_t val = (sf ? offset : static_cast(offset)); - if (is_sub) saved_offset -= val; - else saved_offset += val; - ++curr_insn; - continue; - } - // Placing this here for sneaky programs that want to put nops in between ADRL - } else if (detail::a64_decode_nop(*curr_insn)) { - ++curr_insn; - continue; - } else if (detail::a64_decode_ldr(*curr_insn, &rn, &rt, &offset) - || detail::a64_decode_ldrh(*curr_insn, &rn, &rt, &offset) - || detail::a64_decode_str(*curr_insn, &rn, &rt, &offset)) { - if (saved_rd == rn) { - saved_offset += (offset < 0 ? -offset : offset); - } - } - break; - } - } - const auto from_addr = is_adrp ? reinterpret_cast(insn) & ~0xfff : reinterpret_cast(insn); - return reinterpret_cast(from_addr + saved_offset); - } - else if (detail::a64_decode_ldr(*insn, nullptr, nullptr, &offset) - || detail::a64_decode_ldrh(*insn, nullptr, nullptr, &offset) - || detail::a64_decode_str(*insn, nullptr, nullptr, &offset)) { - return reinterpret_cast(offset); - } - else if (detail::a64_decode_movz(*insn, &sf, nullptr, &offset) - || detail::a64_decode_arithmetic(*insn, nullptr, &sf, nullptr, nullptr, &offset)) { - return reinterpret_cast((sf ? offset : static_cast(offset))); - } - else - throw std::logic_error("Failed to decode instruction with defined functions"); -#else - if (deref_ || rel_) { - const auto relative_address = relative_value(address + offset_); - if (deref_) { - return reinterpret_cast(reinterpret_cast(address) - + offset_ + insn_len_ + relative_address); - } - else { - return reinterpret_cast(relative_address); - } -#endif - } - else { - return reinterpret_cast(reinterpret_cast(address) + offset_); - } - return nullptr; - } -#ifndef __arm64__ - const intptr_t relative_value(uint8_t* ptr) const { - switch (size_) { - case 1: - return static_cast(*reinterpret_cast(ptr)); - case 2: - return static_cast(*reinterpret_cast(ptr)); - case 4: - return static_cast(*reinterpret_cast(ptr)); -#if ((ULLONG_MAX) != (UINT_MAX)) - case 8: - return static_cast(*reinterpret_cast(ptr)); -#endif - } - // Should never hit here - return NULL; - } -#endif - constexpr int32_t get_inst_len_opt(const char* ptr) const { - if (*ptr > '9' || *ptr < '0') - throw std::logic_error("Invalid data for calculating remaining instruction size!"); - return detail::stoi(ptr); - } - }; -} +#pragma once +#include +#include + +#ifndef _MSVC_VER +#define __forceinline inline __attribute__((always_inline)) +#endif + +namespace patterns { + namespace detail { + constexpr bool __forceinline is_digit(char c) { + return c <= '9' && c >= '0'; + } + constexpr bool __forceinline is_hex_digit(char c) { + return is_digit(c) || ((c | 0x20) >= 'a' && (c | 0x20) <= 'f'); + } + constexpr __forceinline char get_bits(char c) { + return is_digit(c) ? (c - '0') : ((c & (~0x20)) - 'A' + 0xA); + } + template + constexpr T stoi_impl(const char* str, T value = 0, bool negative = false, bool hex = false) { + if (*str == '\0') return negative ? -value : value; + if (hex) { + if (is_hex_digit(*str)) + return stoi_impl(str + 1, get_bits(*str) + value * 16, negative, hex); + } else { + if (is_digit(*str)) + return stoi_impl(str + 1, (*str - '0') + value * 10, negative, hex); + } + return negative ? -value : value; + } + template + constexpr T stoi(const char* str) { + if (*str == '-') { + ++str; + if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) + return stoi_impl(str + 2, 0, true, true); + return stoi_impl(str, 0, true); + } + if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) + return stoi_impl(str + 2, 0, false, true); + return stoi_impl(str); + } +#ifdef __arm64__ + template + T extract_bitfield(uint32_t insn, unsigned width, unsigned offset) { + constexpr size_t int_width = sizeof(int32_t) * 8; + return (static_cast(insn) << (int_width - (offset + width))) >> (int_width - width); + } + + bool __forceinline decode_masked_match(uint32_t insn, uint32_t mask, uint32_t pattern) { + return (insn & mask) == pattern; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/NOP--No-Operation- + bool a64_decode_nop(uint32_t insn) { + if (insn == 0b1101'0101'0000'0011'0010'0000'0001'1111) + return true; + return false; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/B--Branch- + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/BL--Branch-with-Link- + bool a64_decode_b(uint32_t insn, bool* is_bl, int64_t* offset) { + // x001 01?? ???? ???? ???? ???? ???? ???? + // 1 - BL + // 0 - B + if (decode_masked_match(insn, 0b0111'1100'0000'0000'0000'0000'0000'0000, 0b0001'0100'0000'0000'0000'0000'0000'0000)) { + if (is_bl) *is_bl = (insn >> 31) & 0x1; + *offset = extract_bitfield(insn, 26, 0) << 2; + return true; + } + return false; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADR--Form-PC-relative-address- + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADRP--Form-PC-relative-address-to-4KB-page- + bool a64_decode_adr(uint32_t insn, bool* is_adrp, unsigned* rd, int64_t* offset) { + // rd - destination register + // x??1 0000 ???? ???? ???? ???? ???? ???? + // 1 - ADRP + // 0 - ADR + // bits 30 & 29 - immlo + // bits 23 -> 5 - immhi + // bits 4 -> 0 - rd + if (decode_masked_match(insn, 0b0001'1111'0000'0000'0000'0000'0000'0000, 0b0001'0000'0000'0000'0000'0000'0000'0000)) { + if (rd) *rd = insn & 0x1f; + uint32_t immlo = (insn >> 29) & 0x3; + int32_t immhi = extract_bitfield(insn, 19, 5) << 2; + auto adrp = (insn >> 31) & 0x1; + if (is_adrp) *is_adrp = adrp; + *offset = (adrp ? (immhi | immlo) * 4096 : (immhi | immlo)); + return true; + } + return false; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate-- + bool a64_decode_ldrh(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { + // rn - name of general-purpose base register or stack pointer + // rt - name of general-purpose base register to be transfered + // Post-index & Pre-index masks + // 0111 1000 010? ???? ???? x1?? ???? ???? + // 1 - pre-index + // 0 - post-index + if (decode_masked_match(insn, 0b1111'1111'1110'0000'0000'0100'0000'0000, 0b0111'1000'0100'0000'0000'0100'0000'0000)) { + *offset = extract_bitfield(insn, 9, 12); + if (rn) *rn = (insn >> 5) & 0x1f; + if (rt) *rt = insn & 0x1f; + return true; + } + // Unsigned offset mask + // 0111 1001 01?? ???? ???? ???? ???? ???? + if (decode_masked_match(insn, 0b1111'1111'1100'0000'0000'0000'0000'0000, 0b0111'1001'0100'0000'0000'0000'0000'0000)) { + *offset = extract_bitfield(insn, 12, 10) << 1; + if (rn) *rn = (insn >> 5) & 0x1f; + if (rt) *rt = insn & 0x1f; + return true; + } + return false; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/LDR--immediate---Load-Register--immediate-- + bool a64_decode_ldr(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { + // rn - name of general-purpose base register or stack pointer + // rt - name of general-purpose register to be transferred + // Post-index & Pre-index masks + // 1x11 1000 010? ???? ???? x1?? ???? ???? + // 1 - 64 bit 1 - pre-index + // 0 - 32 bit 0 - post-index + if (decode_masked_match(insn, 0b1011'1111'1110'0000'0000'0100'0000'0000, 0b1011'1000'0100'0000'0000'0100'0000'0000)) { + // Have not tested this yet + *offset = extract_bitfield(insn, 9, 12); + if (rn) *rn = (insn >> 5) & 0x1f; + if (rt) *rt = insn & 0x1f; + return true; + } + // Unsigned offset mask + // 1x11 1001 01?? ???? ???? ???? ???? ???? + // 1 - 64 bit + // 0 - 32 bit + if (decode_masked_match(insn, 0b1011'1011'1100'0000'0000'0000'0000'0000, 0b1011'1001'0100'0000'0000'0000'0000'0000)) { + *offset = extract_bitfield(insn, 12, 10) << (insn >> 30); + // bit 26 annotated as "VR" - Variant Register (Not sure it's use yet) + //auto vr = (insn >> 26) & 1; + if (rn) *rn = (insn >> 5) & 0x1f; + if (rt) *rt = insn & 0x1f; + return true; + } + return false; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/STR--immediate---Store-Register--immediate-- + bool a64_decode_str(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { + // rn - name of general-purpose base register or stack pointer + // rt - name of general-purpose register to be transferred + // Post-index & Pre-index masks + // 1x11 1000 000? ???? ???? x1?? ???? ???? + // 1 - 64 bit 1 - pre-index + // 0 - 32 bit 0 - post-index + if (decode_masked_match(insn, 0b1011'1011'1110'0000'0000'0100'0000'0000, 0b1011'1000'0000'0000'0000'0100'0000'0000)) { + *offset = extract_bitfield(insn, 9, 12); + // bit 26 annotated as "VR" - Variant Register (Not sure it's use yet) + //auto vr = (insn >> 26) & 1; + if (rn) *rn = (insn >> 5) & 0x1f; + if (rt) *rt = insn & 0x1f; + return true; + } + // Unsigned offset mask + // 1x11 1000 000? ???? ???? ???? ???? ???? + // 1 - 64 bit + // 0 - 32 bit + if (decode_masked_match(insn, 0b1011'1011'1110'0000'0000'0000'0000'0000, 0b1011'1001'0000'0000'0000'0000'0000'0000)) { + *offset = extract_bitfield(insn, 12, 10) << (insn >> 30); + // bit 26 annotated as "VR" - Variant Register (Not sure it's use yet) + //auto vr = (insn >> 26) & 1; + if (rn) *rn = (insn >> 5) & 0x1f; + if (rt) *rt = insn & 0x1f; + return true; + } + return false; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/MOVZ--Move-wide-with-zero- + bool a64_decode_movz(uint32_t insn, unsigned* sf, unsigned* rd, int64_t* offset) { + // sf - 1 for 64 bit, 0 for 32 bit + // rd - destination register + // x101 0010 1xx? ???? ???? ???? ???? ???? + // 1 = 64 bit 0x - shift ammount + // 0 = 32 bit + if (decode_masked_match(insn, 0b0111'1111'1000'0000'0000'0000'0000'0000, 0b0101'0010'1000'0000'0000'0000'0000'0000)) { + auto hw = (insn >> 21) & 3; + *offset = hw ? extract_bitfield(insn, 16, 5) << hw : extract_bitfield(insn, 16, 5); + if (sf) *sf = (insn >> 31) & 0x1; + if (rd) *rd = insn & 0x1f; + return true; + } + return false; + } + + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADD--immediate---Add--immediate-- + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/SUB--immediate---Subtract--immediate-- + bool a64_decode_arithmetic(uint32_t insn, bool* is_sub, unsigned* sf, unsigned* rd, unsigned* rn, int64_t* offset) { + // rd - destination register + // rn - source register + // sf - 1 for 64 bit, 0 for 32 bit + // ?x01 0001 0??? ???? ???? ???? ???? ???? + // 1 = subtraction + // 0 = addition + if (decode_masked_match(insn, 0b0011'1111'1000'0000'0000'0000'0000'0000, 0b0001'0001'0000'0000'0000'0000'0000'0000)) { + if (sf) *sf = (insn >> 31) & 0x1; + if (rd) *rd = insn & 0x1f; + if (rn) *rn = (insn >> 5) & 0x1f; + if (is_sub) *is_sub = (insn >> 30) & 0x1; + uint32_t imm12 = extract_bitfield(insn, 12, 10); + // sh = 1 is LSL#12 + auto sh = (insn >> 22) & 0x1; + *offset = (sh ? (imm12 << 12) : imm12); + return true; + } + return false; + } +#endif + } + + class Pattern { + protected: + uint32_t length_ = 0; + uint32_t offset_ = 0; + bool deref_ = false; +#ifndef __arm64__ + // Remaining bytes from the offset when used in dereferencing + uint32_t insn_len_ = 0; + // Since all arm64 instructions are 32 bit and encoded, just doing deref instead of both deref and relative + // Defaulting 4 byte relative address reading + uint32_t size_ = 4; + bool rel_ = false; +#endif + bool align_ = false; +#ifdef __arm64__ + // arm64 instructions are all 32 bits, so the align scanning will be at the instruction level + static constexpr size_t align_size_ = 4; +#else + static constexpr size_t align_size_ = sizeof(void*); +#endif + public: + Pattern() = default; + ~Pattern() = default; + const uint32_t __forceinline length() const { + return length_; + } + const uint32_t __forceinline offset() const { + return offset_; + } + const bool __forceinline deref() const { + return deref_; + } +#ifndef __arm64__ + const bool __forceinline relative() const { + return rel_; + } +#endif + const bool __forceinline aligned() const { + return align_; + } + virtual const uint8_t* mask() const = 0; + virtual const uint8_t* pattern() const = 0; + template + T find(const uint8_t* bytes, size_t size) const { + return reinterpret_cast(find(bytes, size)); + } + virtual void* find(const uint8_t* bytes, size_t size) const { + void* result = nullptr; + const auto pattern = const_cast(this->pattern()); + const auto mask = const_cast(this->mask()); + const auto end = bytes + size - length_; + for (auto i = const_cast(bytes); i < end; align_ ? i += align_size_ : ++i) { + bool found = true; +#ifdef __arm64__ + // Doing byte by byte match due to arm being encoded instructions, unless specified to align scan + if (align_) { + for (auto j = 0U; j < length_; j += align_size_) { + const auto data = *reinterpret_cast(pattern + j); + const auto msk = *reinterpret_cast(mask + j); + const auto mem = *reinterpret_cast(i + j); + if ((data ^ mem) & msk) { + found = false; + break; + } + } + } else { + for (auto j = 0U; j < length_; ++j) { + if (mask[j] == 0xFF && pattern[j] != i[j]) { + found = false; + break; + } + } + } +#else + for (auto j = 0U; j < length_; j += align_size_) { + const auto data = *reinterpret_cast(pattern + j); + const auto msk = *reinterpret_cast(mask + j); + const auto mem = *reinterpret_cast(i + j); + if ((data ^ mem) & msk) { + found = false; + break; + } + } +#endif + if (found) + return get_result(i, end); + } + return result; + } + virtual uint8_t operator[](size_t idx) const { + return pattern()[idx]; + } + protected: + // Credits to EJT for the helper functions here! + constexpr __forceinline uint8_t value(const char* c) const { + return (detail::get_bits(c[0]) << 4 | detail::get_bits(c[1])); + } + constexpr void handle_options(const char* ptr) { + while (*ptr) { +#ifdef __arm64__ + if (*ptr == 'd') + deref_ = true; +#else + if (*ptr == 'd') { + if (rel_) throw std::logic_error("Cannot use relative and deref together!"); + deref_ = true; + } + else if (*ptr == 'r') { + if (deref_) throw std::logic_error("Cannot use relative and deref together!"); + rel_ = true; + } +#endif + else if (*ptr == 'a') + align_ = true; +#ifndef __arm64__ + // Check the next character to see what size we're reading at this relative address + else if (*ptr > '0' && (sizeof(void*) == 0x8 ? *ptr < '9' : *ptr < '5')) { + size_ = *ptr - '0'; + if ((size_ & (size_ - 1)) != 0) throw std::logic_error("Size is not a valid data type size!"); + } +#endif + ++ptr; + } + } + void* get_result(uint8_t* address, const uint8_t* end) const { +#ifdef __arm64__ + if (deref_) { + auto insn = reinterpret_cast(address + offset_); + int64_t offset = 0; + bool is_sub = false, is_adrp = false; + unsigned rd = 0, sf = 0, rn = 0; + if (detail::a64_decode_b(*insn, nullptr, &offset)) { + return reinterpret_cast(reinterpret_cast(insn) + offset); + } + else if (detail::a64_decode_adr(*insn, &is_adrp, &rd, &offset)) { + auto saved_offset = offset; + auto saved_rd = rd; + auto curr_insn = insn + 1; + if (is_adrp) { + while (*curr_insn) { + unsigned rt = 0; + if (detail::a64_decode_arithmetic(*curr_insn, &is_sub, &sf, &rd, &rn, &offset)) { + if (saved_rd == rd && rn == rd) { + uint64_t val = (sf ? offset : static_cast(offset)); + if (is_sub) saved_offset -= val; + else saved_offset += val; + ++curr_insn; + continue; + } + // Placing this here for sneaky programs that want to put nops in between ADRL + } else if (detail::a64_decode_nop(*curr_insn)) { + ++curr_insn; + continue; + } else if (detail::a64_decode_ldr(*curr_insn, &rn, &rt, &offset) + || detail::a64_decode_ldrh(*curr_insn, &rn, &rt, &offset) + || detail::a64_decode_str(*curr_insn, &rn, &rt, &offset)) { + if (saved_rd == rn) { + saved_offset += (offset < 0 ? -offset : offset); + } + } + break; + } + } + const auto from_addr = is_adrp ? reinterpret_cast(insn) & ~0xfff : reinterpret_cast(insn); + return reinterpret_cast(from_addr + saved_offset); + } + else if (detail::a64_decode_ldr(*insn, nullptr, nullptr, &offset) + || detail::a64_decode_ldrh(*insn, nullptr, nullptr, &offset) + || detail::a64_decode_str(*insn, nullptr, nullptr, &offset)) { + return reinterpret_cast(offset); + } + else if (detail::a64_decode_movz(*insn, &sf, nullptr, &offset) + || detail::a64_decode_arithmetic(*insn, nullptr, &sf, nullptr, nullptr, &offset)) { + return reinterpret_cast((sf ? offset : static_cast(offset))); + } + else + throw std::logic_error("Failed to decode instruction with defined functions"); +#else + if (deref_ || rel_) { + const auto relative_address = relative_value(address + offset_); + if (deref_) { + return reinterpret_cast(reinterpret_cast(address) + + offset_ + insn_len_ + relative_address); + } + else { + return reinterpret_cast(relative_address); + } +#endif + } + else { + return reinterpret_cast(reinterpret_cast(address) + offset_); + } + return nullptr; + } +#ifndef __arm64__ + const intptr_t relative_value(uint8_t* ptr) const { + switch (size_) { + case 1: + return static_cast(*reinterpret_cast(ptr)); + case 2: + return static_cast(*reinterpret_cast(ptr)); + case 4: + return static_cast(*reinterpret_cast(ptr)); +#if ((ULLONG_MAX) != (UINT_MAX)) + case 8: + return static_cast(*reinterpret_cast(ptr)); +#endif + } + // Should never hit here + return NULL; + } +#endif + constexpr int32_t get_inst_len_opt(const char* ptr) const { + if (*ptr > '9' || *ptr < '0') + throw std::logic_error("Invalid data for calculating remaining instruction size!"); + return detail::stoi(ptr); + } + }; +} diff --git a/README.md b/README.md index 3d48b58..9789c82 100644 --- a/README.md +++ b/README.md @@ -1,44 +1,44 @@ -# Pattern Scanning - -#### Classes -- Pattern (base class) -- RuntimePattern -- CompileTimePattern -- XORPattern -- ArmPattern - -RuntimePattern will allocate the pattern & mask with std::vectors default allocator along with leaving the pattern string in the binary. - -CompileTimePattern creates the pattern & mask during compile time and stored in a std::array. This means the data is in the binaries module, and no pattern string left. - -XORPattern creates the pattern & mask along with a keys array during compile time. Pattern stored is XOR'd bytes, otherwise it acts just the same as the CompileTimePattern. (the pattern() method however will return the XOR'd pattern, access the original bytes with the [] operator) - -ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Instructions not implemented would result in a wildcard portion for those 4 bytes. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). - -If using C++20 there is a user defined literal for the compile time pattern. - -#### Pattern Flags: -- d: Dereference from the found address, or found address + offset ( addr + size + relative address ) -- r: Read the relative value at the address, or at the address + offset (not used on arm) -- a: Perform a byte aligned scan for a faster scan. 4 byte alignment x32, 8 byte alignment x64 (rather than byte by byte scan) -- x: This one is used inside of the pattern itself, to show where the offset is at (start address of what we want to capture). This can be followed by a 1/2/4 or 8 for the size of the operand when doing /r or /d - - -#### Examples: -```c++ -// Scan will read the relative value from the X offset as 4 bytes -constexpr auto compiletime_pattern = "AB CC 11 22 33 44 AB 6D X4 EF BE AD DE /r"_ctpattern; -// Scan will read the address from where the marked X is pointed to (defaults as a relative address), and perform the scan byte aligned (4 - 32bit, 8 - 64bit) -// This also uses the instruction len (notice 4 after X). This tells it during the dereference to start the RIP after the E8 instruction -constexpr auto xor_pattern = "FE ED FA CE E8 X4 ? ? ? ? EF BE AD DE /da"_xorpattern; -// Scan will read the address from where the marked X is pointed to (as a single byte; i.e. short jump) -auto runtime_pattern = "BA BE CA FE 72 X1 ? 11 22 /d"_rtpattern -// Scan will find the value in ADRP, and add the next ADD instruction to it (like ADRL pseudocode). Without the dereference option, this would return the address of that ADRP instruction -constexpr auto arm_pattern = ARM_PATTERN( - "LDR W?, [X?,#0x18]\n" - "X ADRP X?, #?\n" - "ADD X?, X?, #?\n" - "STR W?, [SP,#?]\n" - "/d" -) +# Pattern Scanning + +#### Classes +- Pattern (base class) +- RuntimePattern +- CompileTimePattern +- XORPattern +- ArmPattern + +RuntimePattern will allocate the pattern & mask with std::vectors default allocator along with leaving the pattern string in the binary. + +CompileTimePattern creates the pattern & mask during compile time and stored in a std::array. This means the data is in the binaries module, and no pattern string left. + +XORPattern creates the pattern & mask along with a keys array during compile time. Pattern stored is XOR'd bytes, otherwise it acts just the same as the CompileTimePattern. (the pattern() method however will return the XOR'd pattern, access the original bytes with the [] operator) + +ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Instructions not implemented would result in a wildcard portion for those 4 bytes. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). + +If using C++20 there is a user defined literal for the compile time pattern. + +#### Pattern Flags: +- d: Dereference from the found address, or found address + offset ( addr + size + relative address ) +- r: Read the relative value at the address, or at the address + offset (not used on arm) +- a: Perform a byte aligned scan for a faster scan. 4 byte alignment x32, 8 byte alignment x64 (rather than byte by byte scan) +- x: This one is used inside of the pattern itself, to show where the offset is at (start address of what we want to capture). This can be followed by a 1/2/4 or 8 for the size of the operand when doing /r or /d + + +#### Examples: +```c++ +// Scan will read the relative value from the X offset as 4 bytes +constexpr auto compiletime_pattern = "AB CC 11 22 33 44 AB 6D X4 EF BE AD DE /r"_ctpattern; +// Scan will read the address from where the marked X is pointed to (defaults as a relative address), and perform the scan byte aligned (4 - 32bit, 8 - 64bit) +// This also uses the instruction len (notice 4 after X). This tells it during the dereference to start the RIP after the E8 instruction +constexpr auto xor_pattern = "FE ED FA CE E8 X4 ? ? ? ? EF BE AD DE /da"_xorpattern; +// Scan will read the address from where the marked X is pointed to (as a single byte; i.e. short jump) +auto runtime_pattern = "BA BE CA FE 72 X1 ? 11 22 /d"_rtpattern +// Scan will find the value in ADRP, and add the next ADD instruction to it (like ADRL pseudocode). Without the dereference option, this would return the address of that ADRP instruction +constexpr auto arm_pattern = ARM_PATTERN( + "LDR W?, [X?,#0x18]\n" + "X ADRP X?, #?\n" + "ADD X?, X?, #?\n" + "STR W?, [SP,#?]\n" + "/d" +) ``` \ No newline at end of file diff --git a/RuntimePattern.hpp b/RuntimePattern.hpp index 38decf3..f99f621 100644 --- a/RuntimePattern.hpp +++ b/RuntimePattern.hpp @@ -1,77 +1,77 @@ -#pragma once -#include "Pattern.hpp" -#include -#include -#include - -namespace patterns { - - class RuntimePattern : public Pattern { - std::vector m_pattern; - std::vector m_mask; - public: - RuntimePattern(const char* p, size_t len) { - int n = 0; - for (int i = 0; i < len; i += 2) { - auto ptr = &p[i]; - if (*ptr == '?') { - m_pattern.push_back(0); - m_mask.push_back(0); - ++n; - } - // Capture where we have our offset marker 'X' at - else if (*ptr == 'X' || *ptr == 'x') { - offset_ = n; - if (p[i + 1] != ' ') { -#ifndef __arm64__ - insn_len_ = get_inst_len_opt(&p[++i]); -#endif - while (p[i + 1] != ' ') - ++i; - } - } - // Break from parsing the pattern, since / at the end starts the flags - else if (*ptr == '/') { - ++ptr; - handle_options(ptr); - break; - } - else if (*ptr != ' ') { - m_pattern.push_back(value(ptr)); - m_mask.push_back(0xFF); - ++n; - } - else i--; - } -#ifdef __arm64__ - while (align_ && n % align_size_) { - m_pattern.push_back(0); - m_mask.push_back(0); - ++n; - } -#else - while (n % sizeof(void*)) { - m_pattern.push_back(0); - m_mask.push_back(0); - ++n; - } -#endif - length_ = n; - } - RuntimePattern(const char* p) : - RuntimePattern(p, strlen(p)) - { - } - ~RuntimePattern() = default; - virtual const uint8_t* pattern() const override { - return m_pattern.data(); - } - virtual const uint8_t* mask() const override { - return m_mask.data(); - } - }; -} - -auto operator"" _rtpattern(const char* str, size_t size) { - return patterns::RuntimePattern(str, size); +#pragma once +#include "Pattern.hpp" +#include +#include +#include + +namespace patterns { + + class RuntimePattern : public Pattern { + std::vector m_pattern; + std::vector m_mask; + public: + RuntimePattern(const char* p, size_t len) { + int n = 0; + for (int i = 0; i < len; i += 2) { + auto ptr = &p[i]; + if (*ptr == '?') { + m_pattern.push_back(0); + m_mask.push_back(0); + ++n; + } + // Capture where we have our offset marker 'X' at + else if (*ptr == 'X' || *ptr == 'x') { + offset_ = n; + if (p[i + 1] != ' ') { +#ifndef __arm64__ + insn_len_ = get_inst_len_opt(&p[++i]); +#endif + while (p[i + 1] != ' ') + ++i; + } + } + // Break from parsing the pattern, since / at the end starts the flags + else if (*ptr == '/') { + ++ptr; + handle_options(ptr); + break; + } + else if (*ptr != ' ') { + m_pattern.push_back(value(ptr)); + m_mask.push_back(0xFF); + ++n; + } + else i--; + } +#ifdef __arm64__ + while (align_ && n % align_size_) { + m_pattern.push_back(0); + m_mask.push_back(0); + ++n; + } +#else + while (n % sizeof(void*)) { + m_pattern.push_back(0); + m_mask.push_back(0); + ++n; + } +#endif + length_ = n; + } + RuntimePattern(const char* p) : + RuntimePattern(p, strlen(p)) + { + } + ~RuntimePattern() = default; + virtual const uint8_t* pattern() const override { + return m_pattern.data(); + } + virtual const uint8_t* mask() const override { + return m_mask.data(); + } + }; +} + +auto operator"" _rtpattern(const char* str, size_t size) { + return patterns::RuntimePattern(str, size); } \ No newline at end of file diff --git a/XORPattern.hpp b/XORPattern.hpp index 41ddc16..e308125 100644 --- a/XORPattern.hpp +++ b/XORPattern.hpp @@ -1,117 +1,117 @@ -#pragma once -#include "CompileTimePattern.hpp" -#include - -namespace patterns { - namespace detail { - // Used to generate the pattern string hash - constexpr uint32_t fnv1a(const char* s, size_t count) { - return ((count ? fnv1a(s, count - 1) : 2166136261u) ^ s[count]) * 16777619u; - } - - template - constexpr __forceinline uint32_t fnv1a(const char(&s)[count]) { - return fnv1a(s, count - 1); - } - - template - constexpr __forceinline uint8_t generate_key() { - uint32_t value = 2166136261u + seed; - for (char c : __TIME__) - value = static_cast((value ^ c) * 16777619ull); - return value; - } - - template - constexpr auto generate_keys(std::index_sequence) { - return std::array{ detail::generate_key()...}; - } - } - - template - class XORPattern : public Pattern { - std::array pattern_; - std::array mask_; - std::array keys_; - public: - constexpr XORPattern(const char* p) : - pattern_{}, mask_{}, keys_{ detail::generate_keys(std::make_index_sequence()) } - { - length_ = narr; - auto n = 0; - for (auto i = 0; i < nstr; i += 2) { - auto ptr = &p[i]; - if (*ptr == '?') { - pattern_[n] = 0 ^ keys_[n]; - mask_[n] = 0; - ++n; - } - // Capture where we have our offset marker 'X' at - else if (*ptr == 'X' || *ptr == 'x') { - offset_ = n; - if (p[i + 1] != ' ') { -#ifndef __arm64__ - insn_len_ = get_inst_len_opt(&p[++i]); -#endif - while (p[i + 1] != ' ') - ++i; - } - } - // Break from parsing the pattern, since / at the end starts the flags - else if (*ptr == '/') { - ++ptr; - handle_options(ptr); - break; - } - else if (*ptr != ' ') { - pattern_[n] = value(ptr) ^ keys_[n]; - mask_[n] = 0xFF; - ++n; - } - else --i; - } - } - virtual const uint8_t* pattern() const override { - return pattern_.data(); - } - virtual const uint8_t* mask() const override { - return mask_.data(); - } - // Override on [] operator to perform XOR for original byte - virtual uint8_t operator[](size_t idx) const override { - return pattern_[idx] ^ keys_[idx]; - } - virtual void* find(const uint8_t* bytes, size_t size) const override { - void* result = nullptr; - const auto end = bytes + size - length_; - for (auto i = const_cast(bytes); i < end; align_ ? i += align_size_ : ++i) { - bool found = true; - for (auto j = 0U; j < length_; ++j) { - if (mask_[j] == 0xFF && (*this)[j] != i[j]) { - found = false; - break; - } - } - if (found) - return get_result(i, end); - } - return result; - } - template - T find(const uint8_t* bytes, size_t size) const { - return reinterpret_cast(find(bytes, size)); - } - }; - -} - -#if __cplusplus > 201703L -template -constexpr auto operator"" _xorpattern() { - return patterns::XORPattern(str.data); -} -#else -#ifndef XOR_PATTERN -#define XOR_PATTERN(x) patterns::XORPattern(x) -#endif +#pragma once +#include "CompileTimePattern.hpp" +#include + +namespace patterns { + namespace detail { + // Used to generate the pattern string hash + constexpr uint32_t fnv1a(const char* s, size_t count) { + return ((count ? fnv1a(s, count - 1) : 2166136261u) ^ s[count]) * 16777619u; + } + + template + constexpr __forceinline uint32_t fnv1a(const char(&s)[count]) { + return fnv1a(s, count - 1); + } + + template + constexpr __forceinline uint8_t generate_key() { + uint32_t value = 2166136261u + seed; + for (char c : __TIME__) + value = static_cast((value ^ c) * 16777619ull); + return value; + } + + template + constexpr auto generate_keys(std::index_sequence) { + return std::array{ detail::generate_key()...}; + } + } + + template + class XORPattern : public Pattern { + std::array pattern_; + std::array mask_; + std::array keys_; + public: + constexpr XORPattern(const char* p) : + pattern_{}, mask_{}, keys_{ detail::generate_keys(std::make_index_sequence()) } + { + length_ = narr; + auto n = 0; + for (auto i = 0; i < nstr; i += 2) { + auto ptr = &p[i]; + if (*ptr == '?') { + pattern_[n] = 0 ^ keys_[n]; + mask_[n] = 0; + ++n; + } + // Capture where we have our offset marker 'X' at + else if (*ptr == 'X' || *ptr == 'x') { + offset_ = n; + if (p[i + 1] != ' ') { +#ifndef __arm64__ + insn_len_ = get_inst_len_opt(&p[++i]); +#endif + while (p[i + 1] != ' ') + ++i; + } + } + // Break from parsing the pattern, since / at the end starts the flags + else if (*ptr == '/') { + ++ptr; + handle_options(ptr); + break; + } + else if (*ptr != ' ') { + pattern_[n] = value(ptr) ^ keys_[n]; + mask_[n] = 0xFF; + ++n; + } + else --i; + } + } + virtual const uint8_t* pattern() const override { + return pattern_.data(); + } + virtual const uint8_t* mask() const override { + return mask_.data(); + } + // Override on [] operator to perform XOR for original byte + virtual uint8_t operator[](size_t idx) const override { + return pattern_[idx] ^ keys_[idx]; + } + virtual void* find(const uint8_t* bytes, size_t size) const override { + void* result = nullptr; + const auto end = bytes + size - length_; + for (auto i = const_cast(bytes); i < end; align_ ? i += align_size_ : ++i) { + bool found = true; + for (auto j = 0U; j < length_; ++j) { + if (mask_[j] == 0xFF && (*this)[j] != i[j]) { + found = false; + break; + } + } + if (found) + return get_result(i, end); + } + return result; + } + template + T find(const uint8_t* bytes, size_t size) const { + return reinterpret_cast(find(bytes, size)); + } + }; + +} + +#if __cplusplus > 201703L +template +constexpr auto operator"" _xorpattern() { + return patterns::XORPattern(str.data); +} +#else +#ifndef XOR_PATTERN +#define XOR_PATTERN(x) patterns::XORPattern(x) +#endif #endif \ No newline at end of file From 02dce9011dced65e6f2d29d67fa668b9cb93572e Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:37:48 -0700 Subject: [PATCH 02/19] Add SIMD scanning, compile-time-only patterns and a test suite - Pattern: SSE2/NEON scan filtering on two anchor bytes, with a portable 64-bit fallback (PATTERNS_NO_SIMD); aligned scans align to absolute addresses - Compile-time patterns are consteval (PATTERNS_CONSTEVAL) so the pattern text never reaches the binary; string literals for every pattern type (_rtpattern, _ctpattern, _xorpattern, _armpattern) - XORPattern: keys per definition site (source location / __COUNTER__) and per build; decoded only on the stack during find() - Patterns are no longer padded; bounds checks when parsing X offsets - tests/: CMake project with correctness tests, fuzzing against a naive scanner, benchmarks and a multi-pattern pointer table example Co-Authored-By: Claude Opus 5.5 --- .gitignore | 3 +- ArmPattern.hpp | 11 +- CompileTimePattern.hpp | 73 ++-------- Pattern.hpp | 236 ++++++++++++++++++++++++-------- README.md | 31 ++++- RuntimePattern.hpp | 21 +-- XORPattern.hpp | 67 ++++----- tests/CMakeLists.txt | 34 +++++ tests/bench_multi.cpp | 67 +++++++++ tests/bench_scan.cpp | 66 +++++++++ tests/example_pointer_table.cpp | 81 +++++++++++ tests/multi_scan.hpp | 97 +++++++++++++ tests/test_cpp17.cpp | 20 +++ tests/test_patterns.cpp | 215 +++++++++++++++++++++++++++++ 14 files changed, 835 insertions(+), 187 deletions(-) create mode 100644 tests/CMakeLists.txt create mode 100644 tests/bench_multi.cpp create mode 100644 tests/bench_scan.cpp create mode 100644 tests/example_pointer_table.cpp create mode 100644 tests/multi_scan.hpp create mode 100644 tests/test_cpp17.cpp create mode 100644 tests/test_patterns.cpp diff --git a/.gitignore b/.gitignore index dbe9c82..9baa48d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.vscode/ \ No newline at end of file +.vscode/ +build*/ diff --git a/ArmPattern.hpp b/ArmPattern.hpp index 5f10d53..4ef7a3f 100644 --- a/ArmPattern.hpp +++ b/ArmPattern.hpp @@ -622,11 +622,11 @@ namespace patterns { } template - class ArmPattern : public Pattern { + class ArmPattern final : public Pattern { std::array m_instructions{}; std::array m_masks{}; public: - constexpr ArmPattern(const char* p) : + PATTERNS_CONSTEVAL ArmPattern(const char* p) : m_instructions{}, m_masks{} { length_ = narr * 4; @@ -655,6 +655,11 @@ namespace patterns { }; } +template +PATTERNS_CONSTEVAL auto operator""_armpattern() { + return patterns::ArmPattern(str.data); +} + #ifndef ARM_PATTERN -#define ARM_PATTERN(x) patterns::ArmPattern(x) +#define ARM_PATTERN(x) ([] { constexpr auto p = patterns::ArmPattern(x); return p; }()) #endif \ No newline at end of file diff --git a/CompileTimePattern.hpp b/CompileTimePattern.hpp index 711d454..f3164ac 100644 --- a/CompileTimePattern.hpp +++ b/CompileTimePattern.hpp @@ -5,68 +5,36 @@ namespace patterns { namespace detail { - constexpr size_t pattern_length(const char* s, size_t nstr, bool pad = true) { + constexpr size_t pattern_length(const char* s, size_t nstr) { size_t res = 0; -#ifdef __arm64__ - bool align = false; -#endif for (auto i = 0; i < nstr - 1; i += 2) { auto c = s[i]; if (c == 'X' || c == 'x') { - while (s[i + 1] != ' ') + while (i + 1 < nstr && s[i + 1] != ' ') ++i; continue; } - else if (c == '/') { -#ifdef __arm64__ - while (i < nstr - 1 && (c = s[++i])) { - if (c == 'a') { - align = true; - break; - } - } -#endif + else if (c == '/') break; - } else if (c != ' ') ++res; else --i; } -#ifdef __arm64__ - while (pad && align && res % sizeof(uint32_t)) - ++res; -#else - while (pad && res % sizeof(void*)) - ++res; -#endif return res; } template - constexpr size_t pattern_length(const char(&s)[nstr], bool pad = true) { - return pattern_length(s, nstr, pad); + constexpr size_t pattern_length(const char(&s)[nstr]) { + return pattern_length(s, nstr); } - -#if __cplusplus > 201703L - template - struct const_string { - char data[nstr]{}; - static constexpr size_t length = nstr - 1; - static constexpr size_t size = nstr; - - constexpr const_string(const char(&s)[nstr]) { - std::copy_n(s, length, data); - } - }; -#endif } template - class CompileTimePattern : public Pattern { + class CompileTimePattern final : public Pattern { std::array m_pattern; std::array m_mask; public: - constexpr CompileTimePattern(const char* p) : + PATTERNS_CONSTEVAL CompileTimePattern(const char* p) : m_pattern{}, m_mask{} { length_ = narr; @@ -81,11 +49,11 @@ namespace patterns { // Capture where we have our offset marker 'X' at else if (*ptr == 'X' || *ptr == 'x') { offset_ = n; - if (p[i + 1] != ' ') { + if (i + 1 < nstr && p[i + 1] != ' ') { #ifndef __arm64__ insn_len_ = get_inst_len_opt(&p[++i]); #endif - while (p[i + 1] != ' ') + while (i + 1 < nstr && p[i + 1] != ' ') ++i; } } @@ -102,19 +70,6 @@ namespace patterns { } else --i; } -#ifdef __arm64__ - while (align_ && n % align_size_) { - m_pattern[n] = 0; - m_mask[n] = 0; - ++n; - } -#else - while (n % sizeof(void*)) { - m_pattern[n] = 0; - m_mask[n] = 0; - ++n; - } -#endif } virtual const uint8_t* pattern() const override { return m_pattern.data(); @@ -125,13 +80,13 @@ namespace patterns { }; } -#if __cplusplus > 201703L +#ifdef PATTERNS_LITERALS template -constexpr auto operator"" _ctpattern() { +PATTERNS_CONSTEVAL auto operator""_ctpattern() { return patterns::CompileTimePattern(str.data); } -#else -#ifndef COMPILETIME_PATTERN -#define COMPILETIME_PATTERN(x) patterns::CompileTimePattern(x) #endif + +#ifndef COMPILETIME_PATTERN +#define COMPILETIME_PATTERN(x) ([] { constexpr auto p = patterns::CompileTimePattern(x); return p; }()) #endif diff --git a/Pattern.hpp b/Pattern.hpp index 8af24e1..5dbdc3a 100644 --- a/Pattern.hpp +++ b/Pattern.hpp @@ -1,13 +1,75 @@ #pragma once +#include #include +#include #include -#ifndef _MSVC_VER +// Define PATTERNS_NO_SIMD where vector registers are off-limits (e.g. kernel code); find() then uses 64-bit integer (SWAR) scanning +#if defined(PATTERNS_NO_SIMD) +#elif (defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2)) && !defined(_M_ARM64EC) +#include +#define PATTERNS_SSE2 +#elif defined(__ARM_NEON) || defined(_M_ARM64) +#include +#define PATTERNS_NEON +#endif + +#ifndef _MSC_VER #define __forceinline inline __attribute__((always_inline)) #endif +// Compile-time patterns must never be built at runtime: that would put the pattern text in the binary. +// consteval enforces it (C++20); before that the macros force it with a constexpr local. +#ifdef __cpp_consteval +#define PATTERNS_CONSTEVAL consteval +#else +#define PATTERNS_CONSTEVAL constexpr +#endif + +// String literal operators (_ctpattern, _xorpattern, _armpattern) need C++20. MSVC reports __cplusplus as +// 199711L unless /Zc:__cplusplus is passed, so check _MSVC_LANG as well +#if __cplusplus > 201703L || (defined(_MSVC_LANG) && _MSVC_LANG > 201703L) +#define PATTERNS_LITERALS +#if __has_include() +#include +#endif +#endif + namespace patterns { namespace detail { + constexpr uint32_t hash(const char* s, uint32_t h = 2166136261u) { + while (*s) + h = (h ^ static_cast(*s++)) * 16777619u; + return h; + } + + // Unique per definition site (file + line + column/counter), so identical patterns get different XOR keys + constexpr uint32_t site_seed(const char* file, uint32_t line, uint32_t column) { + return hash(file) ^ line * 0x9E3779B1u ^ column * 0x85EBCA77u; + } + +#ifdef PATTERNS_LITERALS + // Literal operator template argument. The defaulted source_location is evaluated where the literal is + // written, which is how "..."_xorpattern gets per-definition keys without a macro + template + struct const_string { + char data[nstr]{}; + uint32_t seed = 0; + static constexpr size_t length = nstr - 1; + static constexpr size_t size = nstr; + +#ifdef __cpp_lib_source_location + PATTERNS_CONSTEVAL const_string(const char(&s)[nstr], std::source_location loc = std::source_location::current()) + : seed(site_seed(loc.file_name(), loc.line(), loc.column())) { +#else + // No source_location (older standard library): keys still vary per pattern and per build, not per site + PATTERNS_CONSTEVAL const_string(const char(&s)[nstr]) : seed(hash(s)) { +#endif + for (size_t i = 0; i < nstr; ++i) + data[i] = s[i]; + } + }; +#endif constexpr bool __forceinline is_digit(char c) { return c <= '9' && c >= '0'; } @@ -41,6 +103,19 @@ namespace patterns { return stoi_impl(str + 2, 0, false, true); return stoi_impl(str); } + + // Masked compare, 8 bytes at a time (memcpy keeps unaligned loads legal; compilers emit a single mov) + inline bool masked_equal(const uint8_t* at, const uint8_t* pat, const uint8_t* msk, size_t len) { + size_t j = 0; + for (; j + 8 <= len; j += 8) { + uint64_t d, p, m; + memcpy(&d, at + j, 8); memcpy(&p, pat + j, 8); memcpy(&m, msk + j, 8); + if ((d ^ p) & m) return false; + } + for (; j < len; ++j) + if ((at[j] ^ pat[j]) & msk[j]) return false; + return true; + } #ifdef __arm64__ template T extract_bitfield(uint32_t insn, unsigned width, unsigned offset) { @@ -53,7 +128,7 @@ namespace patterns { } // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/NOP--No-Operation- - bool a64_decode_nop(uint32_t insn) { + inline bool a64_decode_nop(uint32_t insn) { if (insn == 0b1101'0101'0000'0011'0010'0000'0001'1111) return true; return false; @@ -61,7 +136,7 @@ namespace patterns { // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/B--Branch- // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/BL--Branch-with-Link- - bool a64_decode_b(uint32_t insn, bool* is_bl, int64_t* offset) { + inline bool a64_decode_b(uint32_t insn, bool* is_bl, int64_t* offset) { // x001 01?? ???? ???? ???? ???? ???? ???? // 1 - BL // 0 - B @@ -75,7 +150,7 @@ namespace patterns { // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADR--Form-PC-relative-address- // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADRP--Form-PC-relative-address-to-4KB-page- - bool a64_decode_adr(uint32_t insn, bool* is_adrp, unsigned* rd, int64_t* offset) { + inline bool a64_decode_adr(uint32_t insn, bool* is_adrp, unsigned* rd, int64_t* offset) { // rd - destination register // x??1 0000 ???? ???? ???? ???? ???? ???? // 1 - ADRP @@ -96,7 +171,7 @@ namespace patterns { } // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate-- - bool a64_decode_ldrh(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { + inline bool a64_decode_ldrh(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { // rn - name of general-purpose base register or stack pointer // rt - name of general-purpose base register to be transfered // Post-index & Pre-index masks @@ -121,7 +196,7 @@ namespace patterns { } // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/LDR--immediate---Load-Register--immediate-- - bool a64_decode_ldr(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { + inline bool a64_decode_ldr(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { // rn - name of general-purpose base register or stack pointer // rt - name of general-purpose register to be transferred // Post-index & Pre-index masks @@ -151,7 +226,7 @@ namespace patterns { } // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/STR--immediate---Store-Register--immediate-- - bool a64_decode_str(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { + inline bool a64_decode_str(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { // rn - name of general-purpose base register or stack pointer // rt - name of general-purpose register to be transferred // Post-index & Pre-index masks @@ -182,7 +257,7 @@ namespace patterns { } // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/MOVZ--Move-wide-with-zero- - bool a64_decode_movz(uint32_t insn, unsigned* sf, unsigned* rd, int64_t* offset) { + inline bool a64_decode_movz(uint32_t insn, unsigned* sf, unsigned* rd, int64_t* offset) { // sf - 1 for 64 bit, 0 for 32 bit // rd - destination register // x101 0010 1xx? ???? ???? ???? ???? ???? @@ -200,7 +275,7 @@ namespace patterns { // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/ADD--immediate---Add--immediate-- // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/SUB--immediate---Subtract--immediate-- - bool a64_decode_arithmetic(uint32_t insn, bool* is_sub, unsigned* sf, unsigned* rd, unsigned* rn, int64_t* offset) { + inline bool a64_decode_arithmetic(uint32_t insn, bool* is_sub, unsigned* sf, unsigned* rd, unsigned* rn, int64_t* offset) { // rd - destination register // rn - source register // sf - 1 for 64 bit, 0 for 32 bit @@ -246,21 +321,21 @@ namespace patterns { public: Pattern() = default; ~Pattern() = default; - const uint32_t __forceinline length() const { + constexpr uint32_t length() const { return length_; } - const uint32_t __forceinline offset() const { + constexpr uint32_t offset() const { return offset_; } - const bool __forceinline deref() const { + constexpr bool deref() const { return deref_; } #ifndef __arm64__ - const bool __forceinline relative() const { + constexpr bool relative() const { return rel_; } #endif - const bool __forceinline aligned() const { + constexpr bool aligned() const { return align_; } virtual const uint8_t* mask() const = 0; @@ -270,54 +345,103 @@ namespace patterns { return reinterpret_cast(find(bytes, size)); } virtual void* find(const uint8_t* bytes, size_t size) const { - void* result = nullptr; - const auto pattern = const_cast(this->pattern()); - const auto mask = const_cast(this->mask()); - const auto end = bytes + size - length_; - for (auto i = const_cast(bytes); i < end; align_ ? i += align_size_ : ++i) { - bool found = true; -#ifdef __arm64__ - // Doing byte by byte match due to arm being encoded instructions, unless specified to align scan - if (align_) { - for (auto j = 0U; j < length_; j += align_size_) { - const auto data = *reinterpret_cast(pattern + j); - const auto msk = *reinterpret_cast(mask + j); - const auto mem = *reinterpret_cast(i + j); - if ((data ^ mem) & msk) { - found = false; - break; - } - } - } else { - for (auto j = 0U; j < length_; ++j) { - if (mask[j] == 0xFF && pattern[j] != i[j]) { - found = false; - break; - } - } - } -#else - for (auto j = 0U; j < length_; j += align_size_) { - const auto data = *reinterpret_cast(pattern + j); - const auto msk = *reinterpret_cast(mask + j); - const auto mem = *reinterpret_cast(i + j); - if ((data ^ mem) & msk) { - found = false; - break; - } - } -#endif - if (found) - return get_result(i, end); - } - return result; + const auto hit = scan(bytes, size, pattern(), mask()); + return hit ? get_result(const_cast(hit), bytes + size) : nullptr; } virtual uint8_t operator[](size_t idx) const { return pattern()[idx]; } protected: + // First match of pat/msk in [bytes, bytes + size). Two anchor bytes are checked first + // (16 positions at a time with SIMD), and only positions passing both get a full compare. + const uint8_t* scan(const uint8_t* bytes, size_t size, const uint8_t* pat, const uint8_t* msk) const { + if (size < length_) return nullptr; + if (!length_) return bytes; + const size_t last = size - length_; + const size_t step = align_ ? align_size_ : 1; + // Aligned scans align to the absolute address, not to wherever the caller's buffer starts + size_t i = (step - reinterpret_cast(bytes) % step) % step; + // Anchors: first and last of the most specific bytes (most mask bits set). Far apart, so random + // data rarely satisfies both at once + size_t a1 = 0, a2 = 0; + int best = 0; + for (size_t j = 0; j < length_; ++j) { + int bits = 0; + for (unsigned m = msk[j]; m; m &= m - 1) ++bits; + if (bits > best) { best = bits; a1 = a2 = j; } + else if (bits && bits == best) a2 = j; + } + // All wildcards leaves m1 = m2 = 0, so every position passes the filter and gets compared + const uint8_t m1 = msk[a1], p1 = pat[a1] & m1, m2 = msk[a2], p2 = pat[a2] & m2; +#if defined(PATTERNS_SSE2) + const auto vm1 = _mm_set1_epi8(static_cast(m1)), vp1 = _mm_set1_epi8(static_cast(p1)); + const auto vm2 = _mm_set1_epi8(static_cast(m2)), vp2 = _mm_set1_epi8(static_cast(p2)); + auto block = [&](size_t at) { + const auto v1 = _mm_loadu_si128(reinterpret_cast(bytes + at + a1)); + const auto v2 = _mm_loadu_si128(reinterpret_cast(bytes + at + a2)); + return static_cast(static_cast(_mm_movemask_epi8(_mm_and_si128( + _mm_cmpeq_epi8(_mm_and_si128(v1, vm1), vp1), _mm_cmpeq_epi8(_mm_and_si128(v2, vm2), vp2))))); + }; + // 64 positions per branch (measured ~1.4x over 16 on real x64 code), then 16 at a time for the remainder + for (; i + 63 <= last; i += 64) { + const uint64_t hits = block(i) | block(i + 16) << 16 | block(i + 32) << 32 | block(i + 48) << 48; + if (hits) + for (size_t k = 0; k < 64; k += step) + if ((hits >> k & 1) && detail::masked_equal(bytes + i + k, pat, msk, length_)) + return bytes + i + k; + } + for (; i + 15 <= last; i += 16) { + const auto hits = block(i); + if (hits) + for (size_t k = 0; k < 16; k += step) + if ((hits >> k & 1) && detail::masked_equal(bytes + i + k, pat, msk, length_)) + return bytes + i + k; + } +#elif defined(PATTERNS_NEON) + const auto vm1 = vdupq_n_u8(m1), vp1 = vdupq_n_u8(p1), vm2 = vdupq_n_u8(m2), vp2 = vdupq_n_u8(p2); + for (; i + 15 <= last; i += 16) { + const auto eq = vandq_u8(vceqq_u8(vandq_u8(vld1q_u8(bytes + i + a1), vm1), vp1), + vceqq_u8(vandq_u8(vld1q_u8(bytes + i + a2), vm2), vp2)); + // NEON has no movemask; a narrowing shift packs lane k into bits 4k..4k+3 + const uint64_t hits = vget_lane_u64(vreinterpret_u64_u8(vshrn_n_u16(vreinterpretq_u16_u8(eq), 4)), 0); + if (hits) + for (size_t k = 0; k < 16; k += step) + if ((hits >> (k * 4) & 1) && detail::masked_equal(bytes + i + k, pat, msk, length_)) + return bytes + i + k; + } +#else + // No vector registers: same filter on 8 positions at once in a plain 64-bit integer (SWAR) + constexpr uint64_t ones = 0x0101010101010101ULL; + for (; i + 7 <= last; i += 8) { + uint64_t v1, v2; + memcpy(&v1, bytes + i + a1, 8); + memcpy(&v2, bytes + i + a2, 8); + // Byte k of x is zero when position i + k passes both anchors + const uint64_t x = ((v1 & m1 * ones) ^ p1 * ones) | ((v2 & m2 * ones) ^ p2 * ones); + // High bit set for zero bytes (may also flag a byte above a zero; masked_equal weeds those out) + const uint64_t hits = (x - ones) & ~x & 0x8080808080808080ULL; + if (hits) + for (size_t k = 0; k < 8; k += step) { +#if defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + const auto lane = 7 - k; +#else + const auto lane = k; +#endif + if ((hits >> (lane * 8 + 7) & 1) && detail::masked_equal(bytes + i + k, pat, msk, length_)) + return bytes + i + k; + } + } +#endif + // Remainder that doesn't fill a full block + for (; i <= last; i += step) + if ((bytes[i + a1] & m1) == p1 && (bytes[i + a2] & m2) == p2 && detail::masked_equal(bytes + i, pat, msk, length_)) + return bytes + i; + return nullptr; + } // Credits to EJT for the helper functions here! constexpr __forceinline uint8_t value(const char* c) const { + if (!detail::is_hex_digit(c[0]) || !detail::is_hex_digit(c[1])) + throw std::logic_error("Invalid hex byte in pattern!"); return (detail::get_bits(c[0]) << 4 | detail::get_bits(c[1])); } constexpr void handle_options(const char* ptr) { @@ -426,7 +550,7 @@ namespace patterns { return static_cast(*reinterpret_cast(ptr)); case 4: return static_cast(*reinterpret_cast(ptr)); -#if ((ULLONG_MAX) != (UINT_MAX)) +#if UINTPTR_MAX > 0xFFFFFFFF case 8: return static_cast(*reinterpret_cast(ptr)); #endif diff --git a/README.md b/README.md index 9789c82..bba1d44 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,16 @@ XORPattern creates the pattern & mask along with a keys array during compile tim ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Instructions not implemented would result in a wildcard portion for those 4 bytes. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). -If using C++20 there is a user defined literal for the compile time pattern. +Every pattern type has a string literal: `_rtpattern` (any standard), and with C++20 `_ctpattern`, `_xorpattern`, `_armpattern`. Each `_xorpattern` gets its own random keys from where it's written (file, line, column) and the build time, so the same pattern written twice is stored differently. Before C++20, use the `COMPILETIME_PATTERN`, `XOR_PATTERN` and `ARM_PATTERN` macros (`XOR_PATTERN` gets per-definition keys from `__FILE__`/`__LINE__`/`__COUNTER__`). + +Compile-time patterns (`_ctpattern`, `_xorpattern`, `_armpattern` and their macros) are always built at compile time, so the pattern text never ends up in the binary. The class names still do, through RTTI, because `Pattern` is polymorphic. Build with `/GR-` (MSVC) or `-fno-rtti` (clang/gcc) to strip them. + +Scanning uses SSE2 (x86/x64) or NEON (arm64) when available. Define `PATTERNS_NO_SIMD` before including to use a portable 64-bit integer scan instead (e.g. kernel code). #### Pattern Flags: - d: Dereference from the found address, or found address + offset ( addr + size + relative address ) - r: Read the relative value at the address, or at the address + offset (not used on arm) -- a: Perform a byte aligned scan for a faster scan. 4 byte alignment x32, 8 byte alignment x64 (rather than byte by byte scan) +- a: Only match at aligned addresses. 4 byte alignment x32 & arm64, 8 byte alignment x64 - x: This one is used inside of the pattern itself, to show where the offset is at (start address of what we want to capture). This can be followed by a 1/2/4 or 8 for the size of the operand when doing /r or /d @@ -32,13 +36,26 @@ constexpr auto compiletime_pattern = "AB CC 11 22 33 44 AB 6D X4 EF BE AD DE /r" // This also uses the instruction len (notice 4 after X). This tells it during the dereference to start the RIP after the E8 instruction constexpr auto xor_pattern = "FE ED FA CE E8 X4 ? ? ? ? EF BE AD DE /da"_xorpattern; // Scan will read the address from where the marked X is pointed to (as a single byte; i.e. short jump) -auto runtime_pattern = "BA BE CA FE 72 X1 ? 11 22 /d"_rtpattern +auto runtime_pattern = "BA BE CA FE 72 X1 ? 11 22 /d"_rtpattern; // Scan will find the value in ADRP, and add the next ADD instruction to it (like ADRL pseudocode). Without the dereference option, this would return the address of that ADRP instruction -constexpr auto arm_pattern = ARM_PATTERN( +constexpr auto arm_pattern = "LDR W?, [X?,#0x18]\n" "X ADRP X?, #?\n" "ADD X?, X?, #?\n" "STR W?, [SP,#?]\n" - "/d" -) -``` \ No newline at end of file + "/d"_armpattern; +``` + +#### Tests & benchmarks +`tests/` is a separate CMake project (the library itself stays header-only): +```sh +cmake -S tests -B build -DCMAKE_BUILD_TYPE=Release +cmake --build build --config Release +ctest --test-dir build -C Release --output-on-failure +./build/bench_scan [module file] # single-pattern speed; with a file, 40 real signatures sampled from it +./build/bench_multi [module file] # many patterns: sequential vs parallel vs cache-chunked +``` +- `test_patterns`: edge cases, fuzzing against a naive scanner, per-definition XOR keys, and a check that its own binary contains no pattern plaintext. Also built with `PATTERNS_NO_SIMD`. +- `test_cpp17`: the macro path. +- `example_pointer_table`: globals resolved from one pattern list (X-macro) with a single chunked, multi-threaded scan. See the notes at the top of the file. +- `multi_scan.hpp`: the multi-pattern strategies used by the above. diff --git a/RuntimePattern.hpp b/RuntimePattern.hpp index f99f621..e54327e 100644 --- a/RuntimePattern.hpp +++ b/RuntimePattern.hpp @@ -6,7 +6,7 @@ namespace patterns { - class RuntimePattern : public Pattern { + class RuntimePattern final : public Pattern { std::vector m_pattern; std::vector m_mask; public: @@ -22,11 +22,11 @@ namespace patterns { // Capture where we have our offset marker 'X' at else if (*ptr == 'X' || *ptr == 'x') { offset_ = n; - if (p[i + 1] != ' ') { + if (i + 1 < len && p[i + 1] != ' ') { #ifndef __arm64__ insn_len_ = get_inst_len_opt(&p[++i]); #endif - while (p[i + 1] != ' ') + while (i + 1 < len && p[i + 1] != ' ') ++i; } } @@ -43,19 +43,6 @@ namespace patterns { } else i--; } -#ifdef __arm64__ - while (align_ && n % align_size_) { - m_pattern.push_back(0); - m_mask.push_back(0); - ++n; - } -#else - while (n % sizeof(void*)) { - m_pattern.push_back(0); - m_mask.push_back(0); - ++n; - } -#endif length_ = n; } RuntimePattern(const char* p) : @@ -72,6 +59,6 @@ namespace patterns { }; } -auto operator"" _rtpattern(const char* str, size_t size) { +inline auto operator""_rtpattern(const char* str, size_t size) { return patterns::RuntimePattern(str, size); } \ No newline at end of file diff --git a/XORPattern.hpp b/XORPattern.hpp index e308125..d14021f 100644 --- a/XORPattern.hpp +++ b/XORPattern.hpp @@ -4,39 +4,26 @@ namespace patterns { namespace detail { - // Used to generate the pattern string hash - constexpr uint32_t fnv1a(const char* s, size_t count) { - return ((count ? fnv1a(s, count - 1) : 2166136261u) ^ s[count]) * 16777619u; - } - - template - constexpr __forceinline uint32_t fnv1a(const char(&s)[count]) { - return fnv1a(s, count - 1); - } - - template - constexpr __forceinline uint8_t generate_key() { + // Key byte from the definition site's seed, mixed with the build time so keys also change every build + constexpr uint8_t generate_key(uint32_t seed) { uint32_t value = 2166136261u + seed; for (char c : __TIME__) value = static_cast((value ^ c) * 16777619ull); - return value; - } - - template - constexpr auto generate_keys(std::index_sequence) { - return std::array{ detail::generate_key()...}; + return static_cast(value); } } - template - class XORPattern : public Pattern { + template + class XORPattern final : public Pattern { std::array pattern_; std::array mask_; std::array keys_; public: - constexpr XORPattern(const char* p) : - pattern_{}, mask_{}, keys_{ detail::generate_keys(std::make_index_sequence()) } + PATTERNS_CONSTEVAL XORPattern(const char* p, uint32_t seed) : + pattern_{}, mask_{}, keys_{} { + for (size_t j = 0; j < narr; ++j) + keys_[j] = detail::generate_key(seed + static_cast(j)); length_ = narr; auto n = 0; for (auto i = 0; i < nstr; i += 2) { @@ -49,11 +36,11 @@ namespace patterns { // Capture where we have our offset marker 'X' at else if (*ptr == 'X' || *ptr == 'x') { offset_ = n; - if (p[i + 1] != ' ') { + if (i + 1 < nstr && p[i + 1] != ' ') { #ifndef __arm64__ insn_len_ = get_inst_len_opt(&p[++i]); #endif - while (p[i + 1] != ' ') + while (i + 1 < nstr && p[i + 1] != ' ') ++i; } } @@ -82,20 +69,12 @@ namespace patterns { return pattern_[idx] ^ keys_[idx]; } virtual void* find(const uint8_t* bytes, size_t size) const override { - void* result = nullptr; - const auto end = bytes + size - length_; - for (auto i = const_cast(bytes); i < end; align_ ? i += align_size_ : ++i) { - bool found = true; - for (auto j = 0U; j < length_; ++j) { - if (mask_[j] == 0xFF && (*this)[j] != i[j]) { - found = false; - break; - } - } - if (found) - return get_result(i, end); - } - return result; + // Decoded only on the stack for the duration of the scan; the binary still holds just the XOR'd bytes + std::array plain{}; + for (size_t j = 0; j < narr; ++j) + plain[j] = pattern_[j] ^ keys_[j]; + const auto hit = scan(bytes, size, plain.data(), mask_.data()); + return hit ? get_result(const_cast(hit), bytes + size) : nullptr; } template T find(const uint8_t* bytes, size_t size) const { @@ -105,13 +84,13 @@ namespace patterns { } -#if __cplusplus > 201703L +#ifdef PATTERNS_LITERALS template -constexpr auto operator"" _xorpattern() { - return patterns::XORPattern(str.data); +PATTERNS_CONSTEVAL auto operator""_xorpattern() { + return patterns::XORPattern(str.data, str.seed); } -#else -#ifndef XOR_PATTERN -#define XOR_PATTERN(x) patterns::XORPattern(x) #endif + +#ifndef XOR_PATTERN +#define XOR_PATTERN(x) ([] { constexpr auto p = patterns::XORPattern(x, patterns::detail::site_seed(__FILE__, __LINE__, __COUNTER__)); return p; }()) #endif \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..99b618a --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,34 @@ +# Tests and benchmarks. Not part of the library; the headers stay include-only. +# cmake -S tests -B build -DCMAKE_BUILD_TYPE=Release +# cmake --build build --config Release +# ctest --test-dir build -C Release --output-on-failure +cmake_minimum_required(VERSION 3.16) +project(PatternScannerTests CXX) + +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + set(CMAKE_BUILD_TYPE Release) +endif() +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +find_package(Threads REQUIRED) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..) +add_compile_definitions(_CRT_SECURE_NO_WARNINGS) # fopen in the test code +enable_testing() + +add_executable(test_patterns test_patterns.cpp) +add_executable(test_patterns_nosimd test_patterns.cpp) +target_compile_definitions(test_patterns_nosimd PRIVATE PATTERNS_NO_SIMD) +add_executable(test_cpp17 test_cpp17.cpp) +set_target_properties(test_cpp17 PROPERTIES CXX_STANDARD 17) +add_executable(example_pointer_table example_pointer_table.cpp) +add_executable(bench_scan bench_scan.cpp) +add_executable(bench_multi bench_multi.cpp) + +foreach(t test_patterns test_patterns_nosimd example_pointer_table bench_multi) + target_link_libraries(${t} PRIVATE Threads::Threads) +endforeach() + +add_test(NAME test_patterns COMMAND test_patterns) +add_test(NAME test_patterns_nosimd COMMAND test_patterns_nosimd) +add_test(NAME test_cpp17 COMMAND test_cpp17) +add_test(NAME example_pointer_table COMMAND example_pointer_table) diff --git a/tests/bench_multi.cpp b/tests/bench_multi.cpp new file mode 100644 index 0000000..cce26f1 --- /dev/null +++ b/tests/bench_multi.cpp @@ -0,0 +1,67 @@ +// Resolving many patterns over one module: sequential vs parallel per pattern vs cache-chunked. +// bench_multi synthetic 96 MB module with 60 planted signatures +// bench_multi 60 unique signatures sampled from that file +#include "RuntimePattern.hpp" +#include "multi_scan.hpp" +#include +#include +#include +#include +#include + +template static double ms(F f) { + const auto t = std::chrono::steady_clock::now(); + f(); + return std::chrono::duration(std::chrono::steady_clock::now() - t).count(); +} + +static std::vector load(const char* path) { + std::vector v; + if (FILE* f = fopen(path, "rb")) { + uint8_t chunk[65536]; + for (size_t n; (n = fread(chunk, 1, sizeof chunk, f)) > 0;) v.insert(v.end(), chunk, chunk + n); + fclose(f); + } + return v; +} + +int main(int argc, char** argv) { + std::mt19937 rng(5); + std::vector mod; + if (argc > 1) { + mod = load(argv[1]); + if (mod.size() < 1 << 16) { printf("can't read %s\n", argv[1]); return 1; } + } else { + mod.resize(96 << 20); + for (auto& b : mod) b = static_cast(rng()); + } + + // Signatures from the back 3/4 of the module, kept only if unique, so every scan is a long one + std::vector owned; + while (owned.size() < 60) { + const size_t off = mod.size() / 4 + rng() % (mod.size() * 3 / 4 - 64); + std::string s; + char hex[4]; + for (int k = 0; k < 16; ++k) { + if (k >= 3 && k <= 6) s += "? "; + else { snprintf(hex, sizeof hex, "%02X ", mod[off + k]); s += hex; } + } + patterns::RuntimePattern p(s.c_str()); + if (p.find(mod.data(), mod.size()) == mod.data() + off) owned.push_back(std::move(p)); + } + std::vector pats; + for (auto& p : owned) pats.push_back(&p); + + const unsigned threads = std::thread::hardware_concurrency(); + printf("%zu signatures, %.0f MB module, %u hardware threads\n", pats.size(), mod.size() / 1048576.0, threads); + std::vector want, got; + const auto run = [&](const char* name, auto&& f) { + double best = 1e9; + for (int r = 0; r < 3; ++r) best = std::min(best, ms([&] { f(); })); + printf(" %-34s %8.1f ms %s\n", name, best, got == want ? "" : "RESULTS DIFFER"); + }; + run("sequential (find() per pattern)", [&] { multi::sequential(mod.data(), mod.size(), pats, want); got = want; }); + run("chunked, 1 thread", [&] { multi::chunked(mod.data(), mod.size(), pats, got, 1); }); + run("parallel per pattern", [&] { multi::per_pattern(mod.data(), mod.size(), pats, got, threads); }); + run("chunked, all threads", [&] { multi::chunked(mod.data(), mod.size(), pats, got, threads); }); +} diff --git a/tests/bench_scan.cpp b/tests/bench_scan.cpp new file mode 100644 index 0000000..8665824 --- /dev/null +++ b/tests/bench_scan.cpp @@ -0,0 +1,66 @@ +// Single-pattern scan speed. +// bench_scan synthetic 64 MB of random bytes +// bench_scan also 40 real signatures sampled from that file (e.g. a game exe or dylib) +#include "RuntimePattern.hpp" +#include +#include +#include +#include +#include + +template static double ms(F f) { + const auto t = std::chrono::steady_clock::now(); + f(); + return std::chrono::duration(std::chrono::steady_clock::now() - t).count(); +} + +static std::vector load(const char* path) { + std::vector v; + if (FILE* f = fopen(path, "rb")) { + uint8_t chunk[65536]; + for (size_t n; (n = fread(chunk, 1, sizeof chunk, f)) > 0;) v.insert(v.end(), chunk, chunk + n); + fclose(f); + } + return v; +} + +// 16 bytes from the module with a rel32 wildcarded, like "48 8B 05 ? ? ? ? 48 85 C0 ..." +static std::string signature_at(const std::vector& d, size_t off) { + std::string s; + char hex[4]; + for (int k = 0; k < 16; ++k) { + if (k >= 3 && k <= 6) s += "? "; + else { snprintf(hex, sizeof hex, "%02X ", d[off + k]); s += hex; } + } + return s; +} + +int main(int argc, char** argv) { + { + std::vector buf(64 << 20); + std::mt19937 rng(1); + for (auto& b : buf) b = static_cast(rng()); + const uint8_t sig[] = { 0x48, 0x8B, 0x05, 0xAA, 0xBB, 0xCC, 0xDD, 0x48, 0x85, 0xC0, 0x74, 0x10 }; + memcpy(&buf[buf.size() - 4096], sig, sizeof sig); + auto p = "48 8B 05 ? ? ? ? 48 85 C0 74 10"_rtpattern; + double best = 1e9; + for (int r = 0; r < 5; ++r) best = std::min(best, ms([&] { p.find(buf.data(), buf.size()); })); + printf("synthetic: 64 MB random, match at end %7.2f ms (%.1f GB/s)\n", best, 64 / 1024.0 / (best / 1000)); + } + if (argc > 1) { + const auto d = load(argv[1]); + if (d.size() < 1024) { printf("can't read %s\n", argv[1]); return 1; } + std::mt19937 rng(42); + double total = 0; + size_t scanned = 0; + for (int s = 0; s < 40; ++s) { + const size_t off = rng() % (d.size() - 64); + patterns::RuntimePattern p(signature_at(d, off).c_str()); + const uint8_t* hit = nullptr; + total += ms([&] { hit = p.find(d.data(), d.size()); }); + scanned += hit - d.data() + 16; // signatures can match earlier than where they were sampled + } + printf("real: 40 signatures from %s (%.1f MB) %7.2f ms (%.1f GB/s)\n", argv[1], d.size() / 1048576.0, total, + scanned / 1073741824.0 / (total / 1000)); + } +} diff --git a/tests/example_pointer_table.cpp b/tests/example_pointer_table.cpp new file mode 100644 index 0000000..2778003 --- /dev/null +++ b/tests/example_pointer_table.cpp @@ -0,0 +1,81 @@ +// Pattern-resolved globals declared in one list, resolved later with one call. +// +// - The list lives in a header next to the globals it fills. +// - Everything generated from it is built at compile time (no static-initializer code), so the table is usable +// from a TLS callback, before the CRT has run any constructors. +// - The module range is only needed at resolve time, so it can run after decryption. +// - Patterns live in read-only data instead of being re-materialized on a function's stack. +// +// Threads: don't start threads and wait on them under the loader lock (DllMain, TLS callbacks, static +// initializers of a DLL). New threads can't start until the lock is released, so waiting on them deadlocks. +// There, resolve with threads = 1 (still ~2x faster than sequential find() calls via chunking), or resolve +// from a thread you start and don't wait on. +#include "CompileTimePattern.hpp" +#include "multi_scan.hpp" +#include +#include + +// ---- pointers.hpp --------------------------------------------------------------------------------------------- +#define GAME_POINTERS(X) \ + X(void*, luaO_nilobject_, "48 83 EC ? E8 ? ? ? ? 48 8D 0D X4 ? ? ? ? 48 3B C1 75 /d") \ + X(void*, dummynode_, "44 8B C3 48 89 47 20 48 8B D7 89 47 40 48 8B CD 48 8D 05 X4 ? /d") \ + X(int*, s_errorHandlerRef, "48 8B CB E8 ? ? ? ? 44 8B 05 X4 ? ? ? ? BA F0 D8 FF FF 48 8B CF E8 /d") + +#define DECLARE_POINTER(type, name, sig) \ + inline type name = nullptr; \ + inline constexpr auto name##_pattern = sig##_ctpattern; +GAME_POINTERS(DECLARE_POINTER) + +struct PointerEntry { + const patterns::Pattern* pattern; + void (*set)(void*); + const char* name; // drop in release builds if the names shouldn't be in the binary +}; +#define POINTER_ENTRY(type, name, sig) { &name##_pattern, [](void* p) { name = static_cast(p); }, #name }, +inline constexpr PointerEntry g_pointers[] = { GAME_POINTERS(POINTER_ENTRY) }; + +// Returns how many resolved +inline size_t resolve_pointers(const uint8_t* start, size_t size, unsigned threads) { + std::vector pats; + for (auto& e : g_pointers) pats.push_back(e.pattern); + std::vector out; + multi::chunked(start, size, pats, out, threads); + size_t found = 0; + for (size_t i = 0; i < out.size(); ++i) { + g_pointers[i].set(out[i]); + found += out[i] != nullptr; + } + return found; +} +// --------------------------------------------------------------------------------------------------------------- + +// Proves the table needs no runtime initialization +constinit const PointerEntry* table_is_constant = g_pointers; + +int main() { +#ifdef __arm64__ + puts("x86 signatures with x86 /d semantics; skipped on arm64"); + return 0; +#endif + // Synthetic 32 MB "module" with the three code sequences planted at known places + std::vector mod(32 << 20); + std::mt19937 rng(9); + for (auto& b : mod) b = static_cast(rng()); + auto plant = [&](size_t at, std::initializer_list bytes) { std::copy(bytes.begin(), bytes.end(), mod.begin() + at); }; + plant(0x0123450, { 0x48, 0x83, 0xEC, 0x28, 0xE8, 1, 2, 3, 4, 0x48, 0x8D, 0x0D, 0x00, 0x10, 0x00, 0x00, 0x48, 0x3B, 0xC1, 0x75 }); + plant(0x0ABCDE0, { 0x44, 0x8B, 0xC3, 0x48, 0x89, 0x47, 0x20, 0x48, 0x8B, 0xD7, 0x89, 0x47, 0x40, 0x48, 0x8B, 0xCD, 0x48, 0x8D, 0x05, + 0x00, 0x20, 0x00, 0x00 }); + plant(0x1F00010, { 0x48, 0x8B, 0xCB, 0xE8, 1, 2, 3, 4, 0x44, 0x8B, 0x05, 0xF0, 0xFF, 0xFF, 0xFF, 0xBA, 0xF0, 0xD8, 0xFF, 0xFF, 0x48, 0x8B, 0xCF, 0xE8 }); + const uint8_t* base = mod.data(); + + const size_t found = resolve_pointers(base, mod.size(), std::thread::hardware_concurrency()); + // /d result = match + X offset + 4 (rel32 size) + rel32 + const bool ok = found == 3 + && luaO_nilobject_ == base + 0x0123450 + 12 + 4 + 0x1000 + && dummynode_ == base + 0x0ABCDE0 + 19 + 4 + 0x2000 + && reinterpret_cast(s_errorHandlerRef) == base + 0x1F00010 + 11 + 4 - 0x10; + printf("%-20s %p\n%-20s %p\n%-20s %p\n", "luaO_nilobject_", luaO_nilobject_, "dummynode_", dummynode_, + "s_errorHandlerRef", static_cast(s_errorHandlerRef)); + puts(ok ? "pointer table ok" : "FAIL pointer table"); + return !ok; +} diff --git a/tests/multi_scan.hpp b/tests/multi_scan.hpp new file mode 100644 index 0000000..35b3278 --- /dev/null +++ b/tests/multi_scan.hpp @@ -0,0 +1,97 @@ +#pragma once +// Strategies for resolving many patterns over one module. Test/benchmark code, not part of the library. +// Built only on the public Pattern API, so any pattern type works (XOR included). +#include "Pattern.hpp" +#include +#include +#include +#include + +namespace multi { + // Plain copy of a pattern that exposes the raw match address (before /d /r handling), + // needed to merge results from different chunks by lowest address + struct Raw : patterns::Pattern { + std::vector p, m; + explicit Raw(const patterns::Pattern& src) { + length_ = src.length(); + align_ = src.aligned(); + for (uint32_t i = 0; i < length_; ++i) { + p.push_back(src[i]); // operator[] decodes XOR patterns + m.push_back(src.mask()[i]); + } + } + const uint8_t* pattern() const override { return p.data(); } + const uint8_t* mask() const override { return m.data(); } + const uint8_t* match(const uint8_t* bytes, size_t size) const { return scan(bytes, size, p.data(), m.data()); } + }; + + // Applies the pattern's own result handling (/d, /r, X offset) to a known match: a scan of exactly + // length() bytes can only match at that address + inline void* finish(const patterns::Pattern& p, const uint8_t* match) { + return match ? p.find(match, p.length()) : nullptr; + } + + // One full scan per pattern, one after another (what a list of find() calls does) + inline void sequential(const uint8_t* start, size_t size, const std::vector& pats, std::vector& out) { + out.assign(pats.size(), nullptr); + for (size_t i = 0; i < pats.size(); ++i) + out[i] = pats[i]->find(start, size); + } + + // One full scan per pattern, patterns spread over threads + inline void per_pattern(const uint8_t* start, size_t size, const std::vector& pats, std::vector& out, + unsigned threads = std::thread::hardware_concurrency()) { + out.assign(pats.size(), nullptr); + std::atomic next{0}; + std::vector pool; + for (unsigned t = 0; t < std::max(1u, threads); ++t) + pool.emplace_back([&] { + for (size_t i; (i = next++) < pats.size();) + out[i] = pats[i]->find(start, size); + }); + for (auto& t : pool) t.join(); + } + + // Module split into cache-sized chunks; every pattern runs over a chunk while it's hot in cache, so the module + // is read from RAM once instead of once per pattern. Chunks are spread over threads; threads = 1 runs inline. + inline void chunked(const uint8_t* start, size_t size, const std::vector& pats, std::vector& out, + unsigned threads = std::thread::hardware_concurrency(), size_t chunk = 1 << 20) { + std::vector raw; + size_t overlap = 0; + for (auto p : pats) { + raw.emplace_back(*p); + overlap = std::max(overlap, p->length()); + } + // Lowest match per pattern; chunks finish out of order, so keep the minimum rather than the first found + std::vector> best(pats.size()); + for (auto& b : best) b = UINTPTR_MAX; + const size_t chunks = (size + chunk - 1) / chunk; + std::atomic next{0}; + auto work = [&] { + for (size_t c; (c = next++) < chunks;) { + const uint8_t* s = start + c * chunk; + // Overlap into the next chunk so matches straddling the boundary are found + const size_t n = std::min(chunk + overlap - 1, size - c * chunk); + for (size_t i = 0; i < raw.size(); ++i) { + if (best[i].load(std::memory_order_relaxed) < reinterpret_cast(s)) + continue; // an earlier chunk already matched + if (auto hit = raw[i].match(s, n)) { + uintptr_t v = reinterpret_cast(hit), cur = best[i]; + while (v < cur && !best[i].compare_exchange_weak(cur, v)) {} + } + } + } + }; + if (threads <= 1) { + work(); + } else { + std::vector pool; + for (unsigned t = 0; t < threads; ++t) pool.emplace_back(work); + for (auto& t : pool) t.join(); + } + out.assign(pats.size(), nullptr); + for (size_t i = 0; i < pats.size(); ++i) + if (best[i] != UINTPTR_MAX) + out[i] = finish(*pats[i], reinterpret_cast(best[i].load())); + } +} diff --git a/tests/test_cpp17.cpp b/tests/test_cpp17.cpp new file mode 100644 index 0000000..2e99388 --- /dev/null +++ b/tests/test_cpp17.cpp @@ -0,0 +1,20 @@ +// The pre-C++20 path: macros instead of literals. Built as C++17 by tests/CMakeLists.txt. +#include "RuntimePattern.hpp" +#include "CompileTimePattern.hpp" +#include "XORPattern.hpp" +#include + +int main() { + static const uint8_t buf[] = { 1, 2, 0x48, 0x8B, 5, 6, 7, 8 }; + auto ct = COMPILETIME_PATTERN("48 8B ? ? 07"); + auto x1 = XOR_PATTERN("48 8B ? ? 07"); + auto x2 = XOR_PATTERN("48 8B ? ? 07"); + patterns::RuntimePattern rp("48 8B ? ? 07"); + const bool ok = ct.find(buf, 8) == buf + 2 + && x1.find(buf, 8) == buf + 2 + && x2.find(buf, 8) == buf + 2 + && rp.find(buf, 8) == buf + 2 + && memcmp(x1.pattern(), x2.pattern(), 5) != 0; // per-definition keys + puts(ok ? "c++17 macros ok" : "FAIL c++17 macros"); + return !ok; +} diff --git a/tests/test_patterns.cpp b/tests/test_patterns.cpp new file mode 100644 index 0000000..680bdb2 --- /dev/null +++ b/tests/test_patterns.cpp @@ -0,0 +1,215 @@ +// Correctness tests. Exit code 0 = pass. Build with tests/CMakeLists.txt. +#include "RuntimePattern.hpp" +#include "CompileTimePattern.hpp" +#include "XORPattern.hpp" +#ifdef __arm64__ +#include "ArmPattern.hpp" +#endif +#include "multi_scan.hpp" +#include +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); exit(1); } } while (0) + +// Reference implementation: every position, every byte +static const uint8_t* naive(const uint8_t* b, size_t n, const uint8_t* p, const uint8_t* m, size_t len, size_t step) { + for (size_t i = 0; i + len <= n; ++i) { + if (reinterpret_cast(b + i) % step) continue; + size_t j = 0; + while (j < len && !((b[i + j] ^ p[j]) & m[j])) ++j; + if (j == len) return b + i; + } + return nullptr; +} + +// Arbitrary bit masks (like ArmPattern produces), usable on any platform +struct MaskPattern : patterns::Pattern { + std::vector p, m; + MaskPattern(std::vector p_, std::vector m_, bool align) : p(std::move(p_)), m(std::move(m_)) { + length_ = static_cast(p.size()); + align_ = align; + } + const uint8_t* pattern() const override { return p.data(); } + const uint8_t* mask() const override { return m.data(); } +}; + +static void edge_cases() { + uint8_t t[16] = {}; + memcpy(t + 8, "\xAA\xBB\xCC\xDD\xEE\xFF\x11\x22", 8); + CHECK("AA BB CC DD EE FF 11 22"_rtpattern.find(t, 16) == t + 8); // match in the last position + CHECK("EE FF 11 22"_rtpattern.find(t, 16) == t + 12); // short pattern at the very end + CHECK("AA BB CC DD EE FF 11 22"_rtpattern.find(t, 4) == nullptr); // buffer smaller than pattern + CHECK("? ? ? ?"_rtpattern.find(t, 16) == t); // all wildcards + CHECK("EE FF 11 22"_rtpattern.length() == 4); // no padding + bool threw = false; + try { "48 8G"_rtpattern; } catch (const std::logic_error&) { threw = true; } + CHECK(threw); // invalid hex rejected + { const char s[] = "48 8B X4"; patterns::RuntimePattern rp(s, 8); CHECK(rp.length() == 2); } // X at the end, no overrun + + // X offset, /r (relative value) and /d (rip-relative deref) on a call: E8 + alignas(8) uint8_t code[16] = { 0x90, 0xE8, 0x10, 0x00, 0x00, 0x00, 0xC3 }; + CHECK("E8 X ? ? ? ? C3"_rtpattern.find(code, 16) == code + 2); +#ifndef __arm64__ // arm64 has no /r, and its /d decodes arm instructions + CHECK("E8 X4 ? ? ? ? C3 /r"_rtpattern.find(code, 16) == 0x10); + CHECK("E8 X4 ? ? ? ? C3 /d"_rtpattern.find(code, 16) == code + 6 + 0x10); +#endif + puts(" edge cases ok"); +} + +static void literals_and_macros() { + static const uint8_t buf[] = { 0, 1, 0xDE, 0xAD, 0xBE, 0xEF, 0x13, 0x37, 0xC0, 0xDE, 9 }; +#ifdef PATTERNS_LITERALS + // Same text, different definition sites -> different stored bytes, same results + constexpr auto a = "DE AD BE EF 13 37 C0 DE"_xorpattern; + constexpr auto b = "DE AD BE EF 13 37 C0 DE"_xorpattern; + constexpr auto c = "DE AD BE EF 13 37 C0 DE"_xorpattern; constexpr auto d = "DE AD BE EF 13 37 C0 DE"_xorpattern; + CHECK(memcmp(a.pattern(), b.pattern(), 8) != 0); + CHECK(memcmp(c.pattern(), d.pattern(), 8) != 0); + CHECK(a.find(buf, sizeof buf) == buf + 2); + CHECK(d.find(buf, sizeof buf) == buf + 2); + constexpr auto ct = "DE AD ? EF X 13"_ctpattern; + static_assert(ct.length() == 5 && ct.offset() == 4); + CHECK(ct.find(buf, sizeof buf) == buf + 6); +#endif + auto m1 = XOR_PATTERN("DE AD BE EF 13 37 C0 DE"); // deliberately not constexpr: still built at compile time + auto m2 = XOR_PATTERN("DE AD BE EF 13 37 C0 DE"); + CHECK(memcmp(m1.pattern(), m2.pattern(), 8) != 0); + CHECK(m2.find(buf, sizeof buf) == buf + 2); + CHECK(COMPILETIME_PATTERN("DE AD ? EF").find(buf, sizeof buf) == buf + 2); + puts(" literals and macros ok"); +} + +// The XOR pattern's plaintext and source text must not be in our own executable. The needle is stored +// scrambled and rebuilt at runtime, otherwise the test itself would put it in the binary. +static void no_plaintext_in_binary(const char* exe) { + static const uint8_t scrambled[] = { 0x62, 0xBB, 0xFF, 0x56, 0x38, 0x17, 0x43, 0xEA }; + volatile uint8_t key = 0xA5; // volatile, or the optimizer folds the descramble back into the plaintext + uint8_t plain[8]; + for (int i = 0; i < 8; ++i) plain[i] = scrambled[i] ^ key; // C7 1E 5A F3 9D B2 E6 4F + auto xp = XOR_PATTERN("C7 1E 5A F3 9D B2 E6 4F"); + CHECK(xp.find(plain, 8) == plain); + std::string text; + char hex[4]; + for (auto c : plain) { snprintf(hex, sizeof hex, "%02X ", c); text += hex; } + text.pop_back(); + + FILE* f = fopen(exe, "rb"); + if (!f) { printf(" binary check skipped (can't open %s)\n", exe); return; } + std::vector bin; + uint8_t chunk[65536]; + for (size_t n; (n = fread(chunk, 1, sizeof chunk, f)) > 0;) bin.insert(bin.end(), chunk, chunk + n); + fclose(f); + auto contains = [&](const void* needle, size_t len) { + return std::search(bin.begin(), bin.end(), (const uint8_t*)needle, (const uint8_t*)needle + len) != bin.end(); + }; + CHECK(!contains(plain, 8)); + CHECK(!contains(text.data(), text.size())); + puts(" no plaintext in binary ok"); +} + +// Random patterns with random bit masks against the naive reference: both scan modes, +// sizes around the SIMD block boundaries, odd base addresses +static void fuzz(unsigned rounds) { + std::mt19937 rng(7); + std::vector buf(300); + for (unsigned r = 0; r < rounds; ++r) { + for (auto& x : buf) x = rng() % 4 ? rng() % 3 : rng(); // small alphabet so partial matches are common + const size_t len = 1 + rng() % 24, n = rng() % 200, base = rng() % 9; + std::vector p(len), m(len); + for (size_t j = 0; j < len; ++j) { + const int kind = rng() % 4; + m[j] = kind == 0 ? 0 : kind == 1 ? static_cast(rng()) : 0xFF; + p[j] = (rng() % 2 ? rng() % 3 : rng()) & m[j]; + } + if (rng() % 3 == 0 && n >= len) { // plant a match + const size_t at = rng() % (n - len + 1); + for (size_t j = 0; j < len; ++j) + buf[base + at + j] = (buf[base + at + j] & ~m[j]) | p[j]; + } + const bool align = rng() % 4 == 0; + MaskPattern mp(p, m, align); + const auto got = mp.find(buf.data() + base, n); + const auto want = naive(buf.data() + base, n, p.data(), m.data(), len, align ? sizeof(void*) : 1); + if (got != want) { + printf("FAIL fuzz round %u: len=%zu n=%zu base=%zu align=%d\n", r, len, n, base, align); + exit(1); + } + } + printf(" fuzz ok (%u rounds)\n", rounds); +} + +// Every multi-pattern strategy must agree with plain find() calls, including /d results and matches +// that straddle chunk boundaries (small chunks force many boundaries) +static void multi_strategies() { + std::mt19937 rng(3); + std::vector mod(1 << 20); + for (auto& x : mod) x = static_cast(rng()); + std::vector owned; + std::vector pats; + for (int k = 0; k < 40; ++k) { + const size_t off = rng() % (mod.size() - 32); + std::string s; + char hex[4]; + for (int j = 0; j < 12; ++j) { + if (j == 3) s += "X "; + if (j >= 4 && j < 8) s += "? "; + else { snprintf(hex, sizeof hex, "%02X ", mod[off + j]); s += hex; } + } +#ifndef __arm64__ // x86-style /d on random bytes; arm64 /d would try to decode them as instructions + s += k % 2 ? "/d" : ""; +#endif + owned.emplace_back(s.c_str()); + } + owned.emplace_back("01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F"); // absent + for (auto& p : owned) pats.push_back(&p); + std::vector want, got; + multi::sequential(mod.data(), mod.size(), pats, want); + multi::per_pattern(mod.data(), mod.size(), pats, got); + CHECK(got == want); + for (size_t chunk : { 7, 64, 4096, 1 << 20 }) { + multi::chunked(mod.data(), mod.size(), pats, got, 1, chunk); + CHECK(got == want); + multi::chunked(mod.data(), mod.size(), pats, got, 4, chunk); + CHECK(got == want); + } + puts(" multi-pattern strategies agree ok"); +} + +#ifdef __arm64__ +static void arm_patterns() { + constexpr auto adrl = "LDR W?, [X?,#0x18]\n" + "X ADRP X?, #?\n" + "ADD X?, X?, #?\n" + "STR W?, [SP,#?]\n" + "/d"_armpattern; + static_assert(adrl.length() == 16 && adrl.offset() == 4 && adrl.deref() && adrl.aligned()); + alignas(4) static const uint32_t code[] = { 0xD503201F, 0xD503201F, 0xD65F03C0, 0 }; // NOP NOP RET + constexpr auto ret = "NOP\nRET"_armpattern; + CHECK(ret.find(reinterpret_cast(code), sizeof code) == code + 1); + puts(" arm64 patterns ok"); +} +#endif + +int main(int, char** argv) { +#ifdef PATTERNS_NO_SIMD + puts("scan path: scalar (PATTERNS_NO_SIMD)"); +#elif defined(PATTERNS_SSE2) + puts("scan path: SSE2"); +#elif defined(PATTERNS_NEON) + puts("scan path: NEON"); +#else + puts("scan path: scalar"); +#endif + edge_cases(); + literals_and_macros(); + no_plaintext_in_binary(argv[0]); + fuzz(300000); + multi_strategies(); +#ifdef __arm64__ + arm_patterns(); +#endif + puts("all tests passed"); +} From 35aeff5d5feb207048b3606cc0be1f9849ca01ce Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:37:52 -0700 Subject: [PATCH 03/19] Fix the fuzz test's alignment step on arm64 Aligned scans on arm64 step by the 4-byte instruction size, but the reference scanner used sizeof(void*), so the fuzz test failed there. Co-Authored-By: Claude Opus 5.5 --- tests/test_patterns.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_patterns.cpp b/tests/test_patterns.cpp index 680bdb2..5cf5d47 100644 --- a/tests/test_patterns.cpp +++ b/tests/test_patterns.cpp @@ -132,7 +132,12 @@ static void fuzz(unsigned rounds) { const bool align = rng() % 4 == 0; MaskPattern mp(p, m, align); const auto got = mp.find(buf.data() + base, n); - const auto want = naive(buf.data() + base, n, p.data(), m.data(), len, align ? sizeof(void*) : 1); +#ifdef __arm64__ + constexpr size_t align_step = 4; // instruction size, not pointer size (see Pattern::align_size_) +#else + constexpr size_t align_step = sizeof(void*); +#endif + const auto want = naive(buf.data() + base, n, p.data(), m.data(), len, align ? align_step : 1); if (got != want) { printf("FAIL fuzz round %u: len=%zu n=%zu base=%zu align=%d\n", r, len, n, base, align); exit(1); From 916257db66da60c2e2408427ba61554932c2b21f Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:39:05 -0700 Subject: [PATCH 04/19] Fix ArmPattern parsing and arm64 /d resolution, add test_arm Found by checking patterns against clang-assembled instructions: - Each line is now parsed on its own (trimmed, uppercased). Previously a mnemonic only ended at a space, so NOP/RET before another line, the line before the options, tab-indented lines and CRLF all became wildcards - B/BL #imm and B/BL #? encoded offset 0 under a full mask - ORR Xd, Xn, #imm with an immediate that fits in 32 bits had the wrong N bit - ADD/SUB #imm, LSL #12 (with a space) left the shift unmasked - BR/BLR ignored their register; RET Xn is supported - /d: negative ADR/ADRP offsets were decoded as unsigned, ADRP + ADD into a different register returned just the page, and the ADRP follow-up could read past the scanned buffer test_arm checks 70 concrete encodings, 65 wildcard match/no-match pairs, line formats and /d. On Apple arm64 it also re-assembles its tables and resolves /d on linker-relocated code. Co-Authored-By: Claude Opus 5.5 --- ArmPattern.hpp | 86 +++++++++---- Pattern.hpp | 11 +- README.md | 3 +- tests/CMakeLists.txt | 2 + tests/test_arm.cpp | 299 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 373 insertions(+), 28 deletions(-) create mode 100644 tests/test_arm.cpp diff --git a/ArmPattern.hpp b/ArmPattern.hpp index 4ef7a3f..695f3eb 100644 --- a/ArmPattern.hpp +++ b/ArmPattern.hpp @@ -127,7 +127,7 @@ namespace patterns { constexpr size_t skip_mnemonic(const char* instr) { size_t i = 0; - while (instr[i] && instr[i] != ' ') ++i; + while (instr[i] && !isspace(instr[i])) ++i; return i; } @@ -152,9 +152,11 @@ namespace patterns { return n & (n + 1); } - template - constexpr std::tuple encode_bitmask_immediate(T value) { - uint64_t val = (value > 0xFFFFFFFFULL) ? value : (static_cast(static_cast(value)) << 32) | static_cast(value); + // 32-bit (W) immediates are encoded as the value repeated in both halves; 64-bit (X) ones as they are + constexpr std::tuple encode_bitmask_immediate(uint64_t value, bool is64) { + if (!is64 && value > 0xFFFFFFFFULL) + return { false, 0, 0, 0 }; + uint64_t val = is64 ? value : (value << 32) | value; if (val == 0 || ~val == 0) return { false, 0, 0, 0 }; @@ -196,7 +198,7 @@ namespace patterns { } constexpr ArmOpcode parse_branch_imm26(const char* instr, ArmOpcode base) { - size_t i = skip_whitespace(instr, skip_mnemonic(instr)); + size_t i = skip_char(instr, skip_whitespace(instr, skip_mnemonic(instr)), '#'); if (instr[i] == '?') { base.pattern &= 0xFC000000; @@ -208,6 +210,14 @@ namespace patterns { return base; } + // BR/BLR/RET Xn (a bare RET returns through X30) + constexpr ArmOpcode parse_branch_reg(const char* instr, ArmOpcode base) { + size_t i = skip_whitespace(instr, skip_mnemonic(instr)); + if (instr[i]) parse_register(instr, i, 0x1F, 5, base); + else base.set_bits(30, 0x1F, 5); + return base; + } + constexpr ArmOpcode parse_adr(const char* instr, ArmOpcode base) { size_t i = skip_whitespace(instr, skip_mnemonic(instr)); // Rd @@ -341,8 +351,11 @@ namespace patterns { i = parse_imm_token(instr, i, imm); if (imm[0] != '?') { uint64_t immVal = stoi(imm); - auto [ ok, N, imms, immr ] = encode_bitmask_immediate(immVal); - if (ok) base.set_bits((N << 12 | immr << 6 | imms), (1 << 12 | 0x3F << 6 | 0x3F), 10); + // Unknown width (Rd is "?"): a value that fits in 32 bits can be either, and the two only differ in N + const bool is64 = rd[0] == 'X' || (rd[0] != 'W' && immVal > 0xFFFFFFFFULL); + const uint32_t n_mask = (rd[0] == 'X' || rd[0] == 'W' || is64) ? 1 : 0; + auto [ ok, N, imms, immr ] = encode_bitmask_immediate(immVal, is64); + if (ok) base.set_bits((N << 12 | immr << 6 | imms), (n_mask << 12 | 0x3F << 6 | 0x3F), 10); } return base; } @@ -387,8 +400,10 @@ namespace patterns { if (instr[i] == ',') { i = skip_whitespace(instr, i+1); - if (starts_with(&instr[i], "LSL#0")) base.set_mask(1, 22); - else if (starts_with(&instr[i], "LSL#12")) base.set_bits(1, 1, 22); + if (starts_with(&instr[i], "LSL")) { + i = skip_char(instr, skip_whitespace(instr, i + 3), '#'); + if (instr[i] != '?') base.set_bits(stoi(&instr[i]) == 12 ? 1 : 0, 1, 22); + } } else { base.set_mask(1, 22); } @@ -549,6 +564,9 @@ namespace patterns { { "STR", parse_str }, { "CBZ", parse_cbz }, { "CBNZ", parse_cbz }, + { "BR", parse_branch_reg }, + { "BLR", parse_branch_reg }, + { "RET", parse_branch_reg }, }; constexpr InstructionDescriptor instructions[] = { @@ -573,7 +591,7 @@ namespace patterns { { "CBZ", { 0x34000000, 0x7F000000 } }, { "CBNZ", { 0x35000000, 0x7F000000 } }, { "NOP", { 0xD503201F, 0xFFFFFFFF } }, - { "RET", { 0xD65F03C0, 0xFFFFFFFF } }, + { "RET", { 0xD65F0000, 0xFFFFFC1F } }, }; constexpr ArmOpcode find_opcode(const char* mnemonic) { @@ -588,7 +606,7 @@ namespace patterns { constexpr ArmOpcode a64_encode_instruction(const char* instr) { char mnemonic[8] = {}; size_t i = 0; - while (instr[i] && instr[i] != ' ' && i < 7) { + while (instr[i] && !isspace(instr[i]) && i < 7) { mnemonic[i] = instr[i]; ++i; } @@ -605,14 +623,33 @@ namespace patterns { return base; } + // Copies the line at str[i] into out, trimmed and uppercased (lldb/objdump print lowercase), so each + // instruction is parsed on its own and can't run into the next line. Returns where the next line starts + template + constexpr size_t read_line(const char* str, size_t i, char (&out)[N]) { + while (str[i] == ' ' || str[i] == '\t') ++i; + size_t j = 0; + for (; str[i] && str[i] != '\n'; ++i) { + if (j + 1 >= N) throw std::logic_error("Instruction line is too long!"); + out[j++] = (str[i] >= 'a' && str[i] <= 'z') ? static_cast(str[i] - 'a' + 'A') : str[i]; + } + while (j && isspace(out[j - 1])) --j; + out[j] = '\0'; + return str[i] ? i + 1 : i; + } + + constexpr size_t max_line = 128; + + // Non-empty lines up to the options line ("/d" etc.); must agree with the ArmPattern constructor constexpr size_t total_instructions(const char* str, size_t nstr) { - size_t line = 0, count = 0; - for (size_t i = 0; i < nstr - 1; ++i) { - if (str[i] == '\n') ++line, count = 0; - else if (count == 0 && str[i] == '/') break; - else ++count; + size_t count = 0; + for (size_t i = 0; i < nstr - 1 && str[i];) { + char line[max_line] = {}; + i = read_line(str, i, line); + if (line[0] == '/') break; + if (line[0]) ++count; } - return line + (count > 0 ? 1 : 0); + return count; } template @@ -635,15 +672,16 @@ namespace patterns { size_t options = detail::skip_past(p, 0, '/'); if (p[options - 1] == '/') handle_options(&p[options]); align_ = true; - for (size_t i = detail::skip_whitespace(p, 0); i < nstr && idx < narr;) { - size_t start = detail::skip_char(p, i, ' '); - if (p[start] == 'X') { offset_ = idx * 4; start = detail::skip_whitespace(p, start + 1); } - size_t end = detail::skip_past(p, start, '\n'); - if (idx + 1 == narr && end > options) end = options; - auto encoded = detail::a64_encode_instruction(&p[start]); + for (size_t i = 0; i < nstr && p[i] && idx < narr;) { + char line[detail::max_line] = {}; + i = detail::read_line(p, i, line); + if (line[0] == '/') break; + if (!line[0]) continue; + size_t start = 0; + if (line[0] == 'X') { offset_ = idx * 4; start = detail::skip_whitespace(line, 1); } + auto encoded = detail::a64_encode_instruction(&line[start]); m_instructions[idx] = encoded.pattern; m_masks[idx++] = encoded.mask; - i = end; } } virtual const uint8_t* pattern() const override { diff --git a/Pattern.hpp b/Pattern.hpp index 5dbdc3a..5291120 100644 --- a/Pattern.hpp +++ b/Pattern.hpp @@ -164,7 +164,9 @@ namespace patterns { int32_t immhi = extract_bitfield(insn, 19, 5) << 2; auto adrp = (insn >> 31) & 0x1; if (is_adrp) *is_adrp = adrp; - *offset = (adrp ? (immhi | immlo) * 4096 : (immhi | immlo)); + // Back to signed (immlo is unsigned) before widening, and a 64-bit multiply: ADRP reaches +-4GB + const int64_t imm = static_cast(immhi | immlo); + *offset = (adrp ? imm * 4096 : imm); return true; } return false; @@ -486,13 +488,16 @@ namespace patterns { auto saved_rd = rd; auto curr_insn = insn + 1; if (is_adrp) { - while (*curr_insn) { + // Follow the instructions after ADRP, never reading past the end of the scanned buffer + while (reinterpret_cast(curr_insn + 1) <= end && *curr_insn) { unsigned rt = 0; if (detail::a64_decode_arithmetic(*curr_insn, &is_sub, &sf, &rd, &rn, &offset)) { - if (saved_rd == rd && rn == rd) { + // ADD may write a different register (ADRP X8 / ADD X0, X8, #lo); keep following that one + if (rn == saved_rd) { uint64_t val = (sf ? offset : static_cast(offset)); if (is_sub) saved_offset -= val; else saved_offset += val; + saved_rd = rd; ++curr_insn; continue; } diff --git a/README.md b/README.md index bba1d44..a28e080 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ CompileTimePattern creates the pattern & mask during compile time and stored in XORPattern creates the pattern & mask along with a keys array during compile time. Pattern stored is XOR'd bytes, otherwise it acts just the same as the CompileTimePattern. (the pattern() method however will return the XOR'd pattern, access the original bytes with the [] operator) -ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Instructions not implemented would result in a wildcard portion for those 4 bytes. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). +ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Instructions not implemented would result in a wildcard portion for those 4 bytes. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). One instruction per line; case, indentation and spacing don't matter, so lines pasted from IDA (`LDR W8, [X8,#0x18]`) or lldb (`ldr w8, [x8, #0x18]`) both work. lldb prints ADRP operands as page counts and branch targets as absolute addresses, so wildcard those (`#?`). Aliases such as `MOV` aren't implemented (they become wildcards); write the underlying `MOVZ`/`ORR` instead. Every pattern type has a string literal: `_rtpattern` (any standard), and with C++20 `_ctpattern`, `_xorpattern`, `_armpattern`. Each `_xorpattern` gets its own random keys from where it's written (file, line, column) and the build time, so the same pattern written twice is stored differently. Before C++20, use the `COMPILETIME_PATTERN`, `XOR_PATTERN` and `ARM_PATTERN` macros (`XOR_PATTERN` gets per-definition keys from `__FILE__`/`__LINE__`/`__COUNTER__`). @@ -57,5 +57,6 @@ ctest --test-dir build -C Release --output-on-failure ``` - `test_patterns`: edge cases, fuzzing against a naive scanner, per-definition XOR keys, and a check that its own binary contains no pattern plaintext. Also built with `PATTERNS_NO_SIMD`. - `test_cpp17`: the macro path. +- `test_arm`: ArmPattern encodings checked against clang's assembler (concrete instructions must encode exactly, wildcard patterns must match / not match), line formats, and `/d` resolution. On Apple arm64 it also re-assembles its tables and resolves `/d` on linker-relocated ADRP/ADD/LDR/BL code. - `example_pointer_table`: globals resolved from one pattern list (X-macro) with a single chunked, multi-threaded scan. See the notes at the top of the file. - `multi_scan.hpp`: the multi-pattern strategies used by the above. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 99b618a..4d8aea7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -19,6 +19,7 @@ add_executable(test_patterns test_patterns.cpp) add_executable(test_patterns_nosimd test_patterns.cpp) target_compile_definitions(test_patterns_nosimd PRIVATE PATTERNS_NO_SIMD) add_executable(test_cpp17 test_cpp17.cpp) +add_executable(test_arm test_arm.cpp) set_target_properties(test_cpp17 PROPERTIES CXX_STANDARD 17) add_executable(example_pointer_table example_pointer_table.cpp) add_executable(bench_scan bench_scan.cpp) @@ -31,4 +32,5 @@ endforeach() add_test(NAME test_patterns COMMAND test_patterns) add_test(NAME test_patterns_nosimd COMMAND test_patterns_nosimd) add_test(NAME test_cpp17 COMMAND test_cpp17) +add_test(NAME test_arm COMMAND test_arm) add_test(NAME example_pointer_table COMMAND example_pointer_table) diff --git a/tests/test_arm.cpp b/tests/test_arm.cpp new file mode 100644 index 0000000..097ecd9 --- /dev/null +++ b/tests/test_arm.cpp @@ -0,0 +1,299 @@ +// ArmPattern tests. Exit code 0 = pass. Build with tests/CMakeLists.txt. +// The encoder is checked on every platform against encodings produced by clang's assembler. On Apple arm64 the +// test also re-assembles every table line to confirm the tables, and resolves /d on linker-relocated code. +#include "ArmPattern.hpp" +#include +#include + +#define CHECK(x) do { if (!(x)) { printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); exit(1); } } while (0) + +// Fully specified instruction -> its encoding. The pattern must be exactly the encoding, every bit masked +#define CONCRETE(X) \ + X("NOP", 0xD503201F) \ + X("RET", 0xD65F03C0) \ + X("BR X16", 0xD61F0200) \ + X("BLR X8", 0xD63F0100) \ + X("B #0x100", 0x14000040) \ + X("B #-0x8", 0x17FFFFFE) \ + X("BL #0x1000", 0x94000400) \ + X("BL #-0x40", 0x97FFFFF0) \ + X("ADR X0, #0x10", 0x10000080) \ + X("ADR X3, #-0x4", 0x10FFFFE3) \ + X("ADR X5, #0x5", 0x30000025) \ + X("ADRP X0, #0x1000", 0xB0000000) \ + X("ADRP X8, #0x7000", 0xF0000028) \ + X("ADRP X1, #-0x2000", 0xD0FFFFE1) \ + X("ADRP X2, #0x123000", 0xF0000902) \ + X("LDRH W0, [X1,#0x10]", 0x79402020) \ + X("LDRH W0, [X1]", 0x79400020) \ + X("LDRH W2, [X3,#2]!", 0x78402C62) \ + X("LDRH W2, [X3],#2", 0x78402462) \ + X("LDRH W4, [X5,X6]", 0x786668A4) \ + X("LDRH W4, [X5,X6,LSL#1]", 0x786678A4) \ + X("LDR W0, [X1,#0x18]", 0xB9401820) \ + X("LDR X0, [X1,#0x18]", 0xF9400C20) \ + X("LDR X0, [X1]", 0xF9400020) \ + X("LDR W9, [SP,#0x24]", 0xB94027E9) \ + X("LDR X0, [X1,#8]!", 0xF8408C20) \ + X("LDR X0, [X1],#8", 0xF8408420) \ + X("LDR W0, [X1,#-4]!", 0xB85FCC20) \ + X("LDR X2, [X3,X4]", 0xF8646862) \ + X("LDR X2, [X3,X4,LSL#3]", 0xF8647862) \ + X("LDR W2, [X3,W4,UXTW#2]", 0xB8645862) \ + X("LDR X2, [X3,W4,SXTW]", 0xF864C862) \ + X("LDR X0, #0x100", 0x58000800) \ + X("LDR W1, #-0x10", 0x18FFFF81) \ + X("MOVZ W0, #0x1234", 0x52824680) \ + X("MOVZ X0, #0x1234, LSL#16", 0xD2A24680) \ + X("MOVZ X3, #0xFFFF, LSL#48", 0xD2FFFFE3) \ + X("MOVN W1, #0", 0x12800001) \ + X("MOVK X2, #0xBEEF, LSL#32", 0xF2D7DDE2) \ + X("ORR X0, X1, #0xFF", 0xB2401C20) \ + X("ORR W0, W1, #0x1", 0x32000020) \ + X("ORR X0, X1, #0xFFFF0000FFFF0000", 0xB2103C20) \ + X("ORR X0, X1, X2", 0xAA020020) \ + X("ORR W0, W1, W2, LSL#4", 0x2A021020) \ + X("ORR X0, X1, X2, ROR#7", 0xAAC21C20) \ + X("LDP X29, X30, [SP,#0x10]", 0xA9417BFD) \ + X("LDP W0, W1, [X2,#8]", 0x29410440) \ + X("LDP X29, X30, [SP],#0x20", 0xA8C27BFD) \ + X("LDP X0, X1, [X2,#-0x10]!", 0xA9FF0440) \ + X("ADD X0, X1, #0x10", 0x91004020) \ + X("ADD W0, W1, #0x10", 0x11004020) \ + X("ADD X0, X1, #1, LSL#12", 0x91400420) \ + X("ADD X0, SP, #0x20", 0x910083E0) \ + X("ADD X0, X1, X2", 0x8B020020) \ + X("ADD X0, X1, X2, LSL#3", 0x8B020C20) \ + X("ADD X0, X1, W2, UXTW", 0x8B224020) \ + X("ADD X0, X1, W2, SXTW#2", 0x8B22C820) \ + X("ADDS X0, X1, #4", 0xB1001020) \ + X("SUB SP, SP, #0x40", 0xD10103FF) \ + X("SUB X0, X1, X2", 0xCB020020) \ + X("SUBS W0, W1, #1", 0x71000420) \ + X("STR W0, [SP,#0x1C]", 0xB9001FE0) \ + X("STR X0, [X1,#0x18]", 0xF9000C20) \ + X("STR X0, [X1]", 0xF9000020) \ + X("STR X0, [SP,#-0x10]!", 0xF81F0FE0) \ + X("STR X0, [X1],#8", 0xF8008420) \ + X("STR W0, [X1,X2,LSL#2]", 0xB8227820) \ + X("CBZ X0, #0x40", 0xB4000200) \ + X("CBZ W0, #-0x8", 0x34FFFFC0) \ + X("CBNZ X1, #0x100", 0xB5000801) + +// Wildcard pattern, an instruction, its encoding, and whether the pattern must match it +#define WILDCARD(X) \ + X("LDR W?, [X?,#0x18]", "LDR W3, [X9,#0x18]", 0xB9401923, 1) \ + X("LDR W?, [X?,#0x18]", "LDR W3, [X9,#0x1C]", 0xB9401D23, 0) \ + X("LDR W?, [X?,#0x18]", "LDR X3, [X9,#0x18]", 0xF9400D23, 0) \ + X("LDR W?, [X?,#0x18]", "STR W3, [X9,#0x18]", 0xB9001923, 0) \ + X("LDR ?, [X?,#?]", "LDR X3, [X9,#0x18]", 0xF9400D23, 1) \ + X("LDR ?, [X?,#?]", "LDR W3, [SP,#0x40]", 0xB94043E3, 1) \ + X("LDR X?, [X?,#?]", "LDR X0, [X1,#8]!", 0xF8408C20, 0) \ + X("LDR X?, [X?],#?", "LDR X0, [X1],#8", 0xF8408420, 1) \ + X("LDR X?, [X?,#?]!", "LDR X0, [X1,#-8]!", 0xF85F8C20, 1) \ + X("LDR X?, [X?,X?,LSL#3]", "LDR X0, [X1,X2,LSL#3]", 0xF8627820, 1) \ + X("LDR X?, #?", "LDR X0, #0x100", 0x58000800, 1) \ + X("LDR X?, #?", "LDR X0, [X1]", 0xF9400020, 0) \ + X("LDR W0, [X1, #0x18]", "LDR W0, [X1,#0x18]", 0xB9401820, 1) \ + X("LDRH W?, [X?,#?]", "LDRH W1, [X2,#0x10]", 0x79402041, 1) \ + X("LDRH W?, [X?,#?]", "STRH W1, [X2,#0x10]", 0x79002041, 0) \ + X("LDRH W?, [X?,#?]", "LDRB W1, [X2,#0x10]", 0x39404041, 0) \ + X("STR X?, [SP,#?]", "STR X5, [SP,#0x30]", 0xF9001BE5, 1) \ + X("STR X?, [SP,#?]", "STR X5, [X1,#0x30]", 0xF9001825, 0) \ + X("STR ?, [X?,#?]", "STR W5, [X1,#0x30]", 0xB9003025, 1) \ + X("STR W?, [X?,#0x1C]", "STR W5, [X1,#0x1C]", 0xB9001C25, 1) \ + X("STR X?, [X?,#?]!", "STR X0, [SP,#-0x10]!", 0xF81F0FE0, 1) \ + X("ADRP X?, #?", "ADRP X8, #0x5000", 0xB0000028, 1) \ + X("ADRP X?, #?", "ADR X8, #0x50", 0x10000288, 0) \ + X("ADRP X8, #?", "ADRP X9, #0x5000", 0xB0000029, 0) \ + X("ADR X?, #?", "ADR X1, #-0x40", 0x10FFFE01, 1) \ + X("ADD X?, X?, #?", "ADD X0, X1, #0x3A0", 0x910E8020, 1) \ + X("ADD X?, X?, #?", "SUB X0, X1, #0x3A0", 0xD10E8020, 0) \ + X("ADD X?, X?, #?", "ADD X0, X1, X2", 0x8B020020, 0) \ + X("ADD X?, X?, #?", "ADDS X0, X1, #1", 0xB1000420, 0) \ + X("ADD W?, W?, #?", "ADD W0, W1, #1", 0x11000420, 1) \ + X("ADD X?, X?, #?, LSL#12", "ADD X0, X1, #1, LSL#12", 0x91400420, 1) \ + X("ADD X?, X?, X?", "ADD X0, X1, X2", 0x8B020020, 1) \ + X("ADD X?, X?, X?", "ADD X0, X1, #2", 0x91000820, 0) \ + X("SUB SP, SP, #?", "SUB SP, SP, #0x80", 0xD10203FF, 1) \ + X("SUB X?, X?, #?", "ADD X0, X1, #1", 0x91000420, 0) \ + X("SUBS W?, W?, #?", "SUBS W0, W1, #1", 0x71000420, 1) \ + X("SUBS W?, W?, #?", "SUB W0, W1, #1", 0x51000420, 0) \ + X("ADD X0, X1, #1, LSL #12", "ADD X0, X1, #1, LSL#12", 0x91400420, 1) \ + X("ADD X0, X1, #1, LSL #12", "ADD X0, X1, #1", 0x91000420, 0) \ + X("BL ?", "BL #0x2000", 0x94000800, 1) \ + X("BL #?", "BL #0x2000", 0x94000800, 1) \ + X("BL #?", "B #0x2000", 0x14000800, 0) \ + X("B #?", "B #-0x10", 0x17FFFFFC, 1) \ + X("B ?", "BL #0x10", 0x94000004, 0) \ + X("CBZ ?, #?", "CBZ W0, #0x20", 0x34000100, 1) \ + X("CBZ ?, #?", "CBNZ W0, #0x20", 0x35000100, 0) \ + X("CBNZ X?, #?", "CBNZ X1, #-0x20", 0xB5FFFF01, 1) \ + X("CBZ X?, #?", "CBZ W1, #0x20", 0x34000101, 0) \ + X("MOVZ W?, #?", "MOVZ W0, #0x10", 0x52800200, 1) \ + X("MOVZ W?, #?", "MOVK W0, #0x10", 0x72800200, 0) \ + X("MOVK X?, #?, LSL#?", "MOVK X0, #0x10, LSL#16", 0xF2A00200, 1) \ + X("MOVZ X?, #0x10", "MOVZ X0, #0x10, LSL#16", 0xD2A00200, 0) \ + X("ORR X?, X?, X?", "ORR X0, X1, X2", 0xAA020020, 1) \ + X("ORR X?, X?, #?", "ORR X0, X1, #0xFF", 0xB2401C20, 1) \ + X("ORR X?, X?, X?", "AND X0, X1, X2", 0x8A020020, 0) \ + X("ORR X0, XZR, X1", "MOV X0, X1", 0xAA0103E0, 1) \ + X("LDP X?, X?, [SP,#?]", "LDP X29, X30, [SP,#0x10]", 0xA9417BFD, 1) \ + X("LDP X?, X?, [SP,#?]", "STP X29, X30, [SP,#0x10]", 0xA9017BFD, 0) \ + X("LDP X?, X?, [SP],#?", "LDP X29, X30, [SP],#0x30", 0xA8C37BFD, 1) \ + X("LDP X?, X?, [SP,#?]!", "LDP X29, X30, [SP,#-0x30]!", 0xA9FD7BFD, 1) \ + X("BR X?", "BR X16", 0xD61F0200, 1) \ + X("BR X?", "BLR X16", 0xD63F0200, 0) \ + X("BLR X?", "BLR X8", 0xD63F0100, 1) \ + X("RET", "RET X1", 0xD65F0020, 0) \ + X("NOP", "NOP", 0xD503201F, 1) + +struct Word { uint32_t pat, mask; }; +template static Word word(const P& p, size_t idx = 0) { + Word w; + memcpy(&w.pat, p.pattern() + idx * 4, 4); + memcpy(&w.mask, p.mask() + idx * 4, 4); + return w; +} + +static void concrete() { + int n = 0; +#define CASE(text, enc) { \ + const auto w = word(ARM_PATTERN(text)); \ + if (w.pat != (enc) || w.mask != 0xFFFFFFFF) { \ + printf("FAIL %s: pat=%08X mask=%08X, want %08X/FFFFFFFF\n", text, w.pat, w.mask, (enc)); exit(1); \ + } ++n; } + CONCRETE(CASE) +#undef CASE + printf(" concrete encodings ok (%d)\n", n); +} + +static void wildcards() { + int n = 0; +#define CASE(ptext, itext, enc, want) { \ + const auto w = word(ARM_PATTERN(ptext)); \ + if ((((enc) & w.mask) == w.pat) != (want)) { \ + printf("FAIL %s vs %s: should%s match (pat=%08X mask=%08X enc=%08X)\n", ptext, itext, (want) ? "" : " not", w.pat, w.mask, (enc)); exit(1); \ + } ++n; } + WILDCARD(CASE) +#undef CASE + printf(" wildcard matching ok (%d)\n", n); +} + +template static bool same(const A& a, const B& b) { + return a.length() == b.length() && a.offset() == b.offset() && a.deref() == b.deref() && a.aligned() == b.aligned() + && !memcmp(a.pattern(), b.pattern(), a.length()) && !memcmp(a.mask(), b.mask(), a.length()); +} + +// Each line is parsed on its own: no instruction may turn into a wildcard because of what follows it +static void line_formats() { + constexpr auto canon = ARM_PATTERN("NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"); + CHECK(canon.length() == 12 && canon.offset() == 4 && canon.deref() && canon.aligned()); + CHECK(word(canon, 0).pat == 0xD503201F && word(canon, 1).pat == 0xB9401908 && word(canon, 2).pat == 0xD65F03C0); + for (size_t k = 0; k < 3; ++k) CHECK(word(canon, k).mask == 0xFFFFFFFF); + CHECK(same(canon, ARM_PATTERN("\tNOP\r\n\tX LDR W8, [X8,#0x18]\r\n\tRET\r\n/d"))); // tabs, CRLF + CHECK(same(canon, ARM_PATTERN(" NOP \n\n X LDR W8, [X8, #0x18] \n\n RET\n/d"))); // padding, blank lines + CHECK(same(canon, ARM_PATTERN("nop\nx ldr w8, [x8, #0x18]\nret\n/d"))); // lowercase, as lldb prints it + CHECK(same(ARM_PATTERN("ADD X0, X1, #1, LSL#12"), ARM_PATTERN("ADD X0, X1, #1, LSL #12"))); + CHECK(same(ARM_PATTERN("BL #?"), ARM_PATTERN("BL ?"))); + CHECK(same(ARM_PATTERN("NOP\nRET"), ARM_PATTERN("NOP\nRET\n"))); + CHECK(ARM_PATTERN("NOP\nRET\n/a").length() == 8); + puts(" line formats ok"); +} + +#ifdef __arm64__ +// /d on hand-built code: ADRP beyond 2GB, and ADRP followers bounded by the scanned size +static void deref_synthetic() { + static const uint32_t far_code[] = { 0x528EEEE9, 0x90480000, 0x91004000, 0 }; // MOVZ W9, #0x7777; ADRP X0, #0x90000000; ADD X0, X0, #0x10 + const auto far_page = reinterpret_cast(&far_code[1]) & ~uintptr_t(0xFFF); + constexpr auto adrl = ARM_PATTERN("MOVZ W9, #0x7777\nX ADRP X?, #?\n/d"); + CHECK(adrl.find(reinterpret_cast(far_code), sizeof far_code) == far_page + 0x90000000 + 0x10); + + static const uint32_t neg_code[] = { 0x528EEEE9, 0x90B80000, 0 }; // ADRP X0, #-0x90000000 + const auto neg_page = reinterpret_cast(&neg_code[1]) & ~uintptr_t(0xFFF); + CHECK(adrl.find(reinterpret_cast(neg_code), sizeof neg_code) == neg_page - 0x90000000); + + static const uint32_t adr_back[] = { 0x528EEEE9, 0x10FFFFE3, 0 }; // ADR X3, #-0x4 + CHECK(ARM_PATTERN("MOVZ W9, #0x7777\nX ADR X?, #?\n/d").find(reinterpret_cast(adr_back), sizeof adr_back) == adr_back); + + // ADRP X0, #0x1000; ADD X1, X0, #0x10; LDR X2, [X1,#8]: ADD into another register, then a load through it + static const uint32_t chain[] = { 0x528EEEE9, 0xB0000000, 0x91004001, 0xF9400422, 0 }; + const auto page = reinterpret_cast(&chain[1]) & ~uintptr_t(0xFFF); + const auto bytes = reinterpret_cast(chain); + CHECK(adrl.find(bytes, sizeof chain) == page + 0x1000 + 0x10 + 8); + CHECK(adrl.find(bytes, 12) == page + 0x1000 + 0x10); // the LDR is past the end: not read + CHECK(adrl.find(bytes, 8) == page + 0x1000); // ADRP is the last word scanned + puts(" /d on synthetic code ok"); +} +#endif + +#if defined(__arm64__) && defined(__APPLE__) +#define SYM(x) "_" x +#define ASM_LINE(text, ...) text "\n" +#define ASM_WILD(ptext, itext, ...) itext "\n" +__asm__(".data\n.p2align 2\n.globl " SYM("asm_concrete") "\n" SYM("asm_concrete") ":\n" CONCRETE(ASM_LINE) + ".globl " SYM("asm_wildcard") "\n" SYM("asm_wildcard") ":\n" WILDCARD(ASM_WILD)); +extern "C" const uint32_t asm_concrete[], asm_wildcard[]; + +// The tables' encodings must still be what the assembler produces +static void assembler_agrees() { + size_t i = 0; +#define CASE(text, enc) if (asm_concrete[i++] != (enc)) { printf("FAIL table: %s assembles to %08X\n", text, asm_concrete[i - 1]); exit(1); } + CONCRETE(CASE) +#undef CASE + i = 0; +#define CASE(ptext, itext, enc, want) if (asm_wildcard[i++] != (enc)) { printf("FAIL table: %s assembles to %08X\n", itext, asm_wildcard[i - 1]); exit(1); } + WILDCARD(CASE) +#undef CASE + puts(" tables match the assembler ok"); +} + +// /d against code whose ADRP/ADD/LDR/BL operands were filled in by the linker +extern "C" int arm_target_data; +int arm_target_data = 0x1234; +extern "C" void arm_target_func() {} +extern "C" const uint32_t arm_reloc_code[]; +__asm__(".text\n.p2align 2\n.globl " SYM("arm_reloc_code") "\n" SYM("arm_reloc_code") ":\n" + " movz w9, #0x7777\n" + " adrp x0, " SYM("arm_target_data") "@PAGE\n" + " add x0, x0, " SYM("arm_target_data") "@PAGEOFF\n" + " movz w9, #0x6666\n" + " adrp x8, " SYM("arm_target_data") "@PAGE\n" + " add x1, x8, " SYM("arm_target_data") "@PAGEOFF\n" // different destination register + " movz w9, #0x5555\n" + " adrp x8, " SYM("arm_target_data") "@PAGE\n" + " ldr w2, [x8, " SYM("arm_target_data") "@PAGEOFF]\n" + " movz w9, #0x4444\n" + " bl " SYM("arm_target_func") "\n" + " movz w9, #0x3333\n" + " b " SYM("arm_target_func") "\n" + " ret\n"); + +static void deref_relocated() { + const auto code = reinterpret_cast(arm_reloc_code); + const size_t n = 14 * 4; + const void* data = &arm_target_data; + const void* func = reinterpret_cast(&arm_target_func); + CHECK(ARM_PATTERN("MOVZ W9, #0x7777\nX ADRP X?, #?\nADD X?, X?, #?\n/d").find(code, n) == data); + CHECK(ARM_PATTERN("MOVZ W9, #0x6666\nX ADRP X?, #?\nADD X?, X?, #?\n/d").find(code, n) == data); + CHECK(ARM_PATTERN("MOVZ W9, #0x5555\nX ADRP X?, #?\nLDR W?, [X?,#?]\n/d").find(code, n) == data); + CHECK(ARM_PATTERN("MOVZ W9, #0x4444\nX BL ?\n/d").find(code, n) == func); + CHECK(ARM_PATTERN("MOVZ W9, #0x3333\nX B #?\n/d").find(code, n) == func); + CHECK(ARM_PATTERN("MOVZ W9, #0x4444\nX BL ?").find(code, n) == code + 40); // no /d: the X address + puts(" /d on relocated code ok"); +} +#endif + +int main() { + concrete(); + wildcards(); + line_formats(); +#ifdef __arm64__ + deref_synthetic(); +#endif +#if defined(__arm64__) && defined(__APPLE__) + assembler_agrees(); + deref_relocated(); +#endif + puts("all arm tests passed"); +} From 7906eb7638da554ef90a4a8909ba9ad777affb3b Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:39:08 -0700 Subject: [PATCH 05/19] Keep XORPattern plaintext out of optimized builds With a constexpr pattern, clang -O3 inlined find() and folded pattern ^ keys into a constant, emitting the plaintext in __const (caught by test_patterns in a CMake Release build). Reading the keys through volatile keeps the XOR at runtime. Co-Authored-By: Claude Opus 5.5 --- XORPattern.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/XORPattern.hpp b/XORPattern.hpp index d14021f..44c6345 100644 --- a/XORPattern.hpp +++ b/XORPattern.hpp @@ -71,8 +71,11 @@ namespace patterns { virtual void* find(const uint8_t* bytes, size_t size) const override { // Decoded only on the stack for the duration of the scan; the binary still holds just the XOR'd bytes std::array plain{}; + // Keys read through volatile: otherwise, with a constexpr pattern, the optimizer folds the XOR at compile + // time and emits the plaintext as a constant (seen with clang -O3) + const volatile uint8_t* keys = keys_.data(); for (size_t j = 0; j < narr; ++j) - plain[j] = pattern_[j] ^ keys_[j]; + plain[j] = pattern_[j] ^ keys[j]; const auto hit = scan(bytes, size, plain.data(), mask_.data()); return hit ? get_result(const_cast(hit), bytes + size) : nullptr; } From 28665856f5989037bb5ddc8a6684bdebca8f6473 Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Fri, 2 Oct 2026 18:39:12 -0700 Subject: [PATCH 06/19] Detect arm64 from __aarch64__ and _M_ARM64, not only __arm64__ __arm64__ is only defined by Apple's toolchain, so GCC/clang on Linux and MSVC on Windows ARM64 silently got x86 /d semantics and alignment. Pattern.hpp now defines PATTERNS_ARM64 from any of the three. Co-Authored-By: Claude Opus 5.5 --- CompileTimePattern.hpp | 2 +- Pattern.hpp | 22 ++++++++++++++-------- RuntimePattern.hpp | 2 +- XORPattern.hpp | 2 +- tests/example_pointer_table.cpp | 2 +- tests/test_arm.cpp | 8 ++++---- tests/test_patterns.cpp | 12 ++++++------ 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/CompileTimePattern.hpp b/CompileTimePattern.hpp index f3164ac..9da9d20 100644 --- a/CompileTimePattern.hpp +++ b/CompileTimePattern.hpp @@ -50,7 +50,7 @@ namespace patterns { else if (*ptr == 'X' || *ptr == 'x') { offset_ = n; if (i + 1 < nstr && p[i + 1] != ' ') { -#ifndef __arm64__ +#ifndef PATTERNS_ARM64 insn_len_ = get_inst_len_opt(&p[++i]); #endif while (i + 1 < nstr && p[i + 1] != ' ') diff --git a/Pattern.hpp b/Pattern.hpp index 5291120..9380a5f 100644 --- a/Pattern.hpp +++ b/Pattern.hpp @@ -14,6 +14,12 @@ #define PATTERNS_NEON #endif +// arm64 instruction semantics (/d decoding, 4-byte alignment). __arm64__ alone is Apple only: GCC and clang +// elsewhere define __aarch64__, MSVC _M_ARM64 +#if defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) +#define PATTERNS_ARM64 +#endif + #ifndef _MSC_VER #define __forceinline inline __attribute__((always_inline)) #endif @@ -116,7 +122,7 @@ namespace patterns { if ((at[j] ^ pat[j]) & msk[j]) return false; return true; } -#ifdef __arm64__ +#ifdef PATTERNS_ARM64 template T extract_bitfield(uint32_t insn, unsigned width, unsigned offset) { constexpr size_t int_width = sizeof(int32_t) * 8; @@ -305,7 +311,7 @@ namespace patterns { uint32_t length_ = 0; uint32_t offset_ = 0; bool deref_ = false; -#ifndef __arm64__ +#ifndef PATTERNS_ARM64 // Remaining bytes from the offset when used in dereferencing uint32_t insn_len_ = 0; // Since all arm64 instructions are 32 bit and encoded, just doing deref instead of both deref and relative @@ -314,7 +320,7 @@ namespace patterns { bool rel_ = false; #endif bool align_ = false; -#ifdef __arm64__ +#ifdef PATTERNS_ARM64 // arm64 instructions are all 32 bits, so the align scanning will be at the instruction level static constexpr size_t align_size_ = 4; #else @@ -332,7 +338,7 @@ namespace patterns { constexpr bool deref() const { return deref_; } -#ifndef __arm64__ +#ifndef PATTERNS_ARM64 constexpr bool relative() const { return rel_; } @@ -448,7 +454,7 @@ namespace patterns { } constexpr void handle_options(const char* ptr) { while (*ptr) { -#ifdef __arm64__ +#ifdef PATTERNS_ARM64 if (*ptr == 'd') deref_ = true; #else @@ -463,7 +469,7 @@ namespace patterns { #endif else if (*ptr == 'a') align_ = true; -#ifndef __arm64__ +#ifndef PATTERNS_ARM64 // Check the next character to see what size we're reading at this relative address else if (*ptr > '0' && (sizeof(void*) == 0x8 ? *ptr < '9' : *ptr < '5')) { size_ = *ptr - '0'; @@ -474,7 +480,7 @@ namespace patterns { } } void* get_result(uint8_t* address, const uint8_t* end) const { -#ifdef __arm64__ +#ifdef PATTERNS_ARM64 if (deref_) { auto insn = reinterpret_cast(address + offset_); int64_t offset = 0; @@ -546,7 +552,7 @@ namespace patterns { } return nullptr; } -#ifndef __arm64__ +#ifndef PATTERNS_ARM64 const intptr_t relative_value(uint8_t* ptr) const { switch (size_) { case 1: diff --git a/RuntimePattern.hpp b/RuntimePattern.hpp index e54327e..ca24dad 100644 --- a/RuntimePattern.hpp +++ b/RuntimePattern.hpp @@ -23,7 +23,7 @@ namespace patterns { else if (*ptr == 'X' || *ptr == 'x') { offset_ = n; if (i + 1 < len && p[i + 1] != ' ') { -#ifndef __arm64__ +#ifndef PATTERNS_ARM64 insn_len_ = get_inst_len_opt(&p[++i]); #endif while (i + 1 < len && p[i + 1] != ' ') diff --git a/XORPattern.hpp b/XORPattern.hpp index 44c6345..7e649fc 100644 --- a/XORPattern.hpp +++ b/XORPattern.hpp @@ -37,7 +37,7 @@ namespace patterns { else if (*ptr == 'X' || *ptr == 'x') { offset_ = n; if (i + 1 < nstr && p[i + 1] != ' ') { -#ifndef __arm64__ +#ifndef PATTERNS_ARM64 insn_len_ = get_inst_len_opt(&p[++i]); #endif while (i + 1 < nstr && p[i + 1] != ' ') diff --git a/tests/example_pointer_table.cpp b/tests/example_pointer_table.cpp index 2778003..e256845 100644 --- a/tests/example_pointer_table.cpp +++ b/tests/example_pointer_table.cpp @@ -53,7 +53,7 @@ inline size_t resolve_pointers(const uint8_t* start, size_t size, unsigned threa constinit const PointerEntry* table_is_constant = g_pointers; int main() { -#ifdef __arm64__ +#ifdef PATTERNS_ARM64 puts("x86 signatures with x86 /d semantics; skipped on arm64"); return 0; #endif diff --git a/tests/test_arm.cpp b/tests/test_arm.cpp index 097ecd9..46611b5 100644 --- a/tests/test_arm.cpp +++ b/tests/test_arm.cpp @@ -201,7 +201,7 @@ static void line_formats() { puts(" line formats ok"); } -#ifdef __arm64__ +#ifdef PATTERNS_ARM64 // /d on hand-built code: ADRP beyond 2GB, and ADRP followers bounded by the scanned size static void deref_synthetic() { static const uint32_t far_code[] = { 0x528EEEE9, 0x90480000, 0x91004000, 0 }; // MOVZ W9, #0x7777; ADRP X0, #0x90000000; ADD X0, X0, #0x10 @@ -227,7 +227,7 @@ static void deref_synthetic() { } #endif -#if defined(__arm64__) && defined(__APPLE__) +#if defined(PATTERNS_ARM64) && defined(__APPLE__) #define SYM(x) "_" x #define ASM_LINE(text, ...) text "\n" #define ASM_WILD(ptext, itext, ...) itext "\n" @@ -288,10 +288,10 @@ int main() { concrete(); wildcards(); line_formats(); -#ifdef __arm64__ +#ifdef PATTERNS_ARM64 deref_synthetic(); #endif -#if defined(__arm64__) && defined(__APPLE__) +#if defined(PATTERNS_ARM64) && defined(__APPLE__) assembler_agrees(); deref_relocated(); #endif diff --git a/tests/test_patterns.cpp b/tests/test_patterns.cpp index 5cf5d47..0c1960a 100644 --- a/tests/test_patterns.cpp +++ b/tests/test_patterns.cpp @@ -2,7 +2,7 @@ #include "RuntimePattern.hpp" #include "CompileTimePattern.hpp" #include "XORPattern.hpp" -#ifdef __arm64__ +#ifdef PATTERNS_ARM64 #include "ArmPattern.hpp" #endif #include "multi_scan.hpp" @@ -52,7 +52,7 @@ static void edge_cases() { // X offset, /r (relative value) and /d (rip-relative deref) on a call: E8 alignas(8) uint8_t code[16] = { 0x90, 0xE8, 0x10, 0x00, 0x00, 0x00, 0xC3 }; CHECK("E8 X ? ? ? ? C3"_rtpattern.find(code, 16) == code + 2); -#ifndef __arm64__ // arm64 has no /r, and its /d decodes arm instructions +#ifndef PATTERNS_ARM64 // arm64 has no /r, and its /d decodes arm instructions CHECK("E8 X4 ? ? ? ? C3 /r"_rtpattern.find(code, 16) == 0x10); CHECK("E8 X4 ? ? ? ? C3 /d"_rtpattern.find(code, 16) == code + 6 + 0x10); #endif @@ -132,7 +132,7 @@ static void fuzz(unsigned rounds) { const bool align = rng() % 4 == 0; MaskPattern mp(p, m, align); const auto got = mp.find(buf.data() + base, n); -#ifdef __arm64__ +#ifdef PATTERNS_ARM64 constexpr size_t align_step = 4; // instruction size, not pointer size (see Pattern::align_size_) #else constexpr size_t align_step = sizeof(void*); @@ -163,7 +163,7 @@ static void multi_strategies() { if (j >= 4 && j < 8) s += "? "; else { snprintf(hex, sizeof hex, "%02X ", mod[off + j]); s += hex; } } -#ifndef __arm64__ // x86-style /d on random bytes; arm64 /d would try to decode them as instructions +#ifndef PATTERNS_ARM64 // x86-style /d on random bytes; arm64 /d would try to decode them as instructions s += k % 2 ? "/d" : ""; #endif owned.emplace_back(s.c_str()); @@ -183,7 +183,7 @@ static void multi_strategies() { puts(" multi-pattern strategies agree ok"); } -#ifdef __arm64__ +#ifdef PATTERNS_ARM64 static void arm_patterns() { constexpr auto adrl = "LDR W?, [X?,#0x18]\n" "X ADRP X?, #?\n" @@ -213,7 +213,7 @@ int main(int, char** argv) { no_plaintext_in_binary(argv[0]); fuzz(300000); multi_strategies(); -#ifdef __arm64__ +#ifdef PATTERNS_ARM64 arm_patterns(); #endif puts("all tests passed"); From a6dcae883ea98af28f7eb5189aca52ed55601744 Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 09:19:01 -0700 Subject: [PATCH 07/19] Add find_all, ArmXORPattern and XOR'd masks - Pattern: public match()/resolve(), find_all() for many patterns over one range - XORPattern: masks stored XOR'd with their own key stream; unmask() for originals - ArmXORPattern: ArmPattern with XOR'd instruction words and masks - Protected Pattern destructor, size_t loop indices - tests: example_usage replaces example_pointer_table, test_tls (Windows), multi_scan uses match()/resolve(), ArmXORPattern tests Co-Authored-By: Claude Opus 5.5 --- ArmPattern.hpp | 5 ++ ArmXORPattern.hpp | 81 +++++++++++++++++ CompileTimePattern.hpp | 4 +- Pattern.hpp | 49 ++++++++-- README.md | 62 +++++++++++-- RuntimePattern.hpp | 2 +- XORPattern.hpp | 41 +++++---- tests/CMakeLists.txt | 10 ++- tests/bench_multi.cpp | 2 +- tests/example_pointer_table.cpp | 81 ----------------- tests/example_usage.cpp | 153 ++++++++++++++++++++++++++++++++ tests/multi_scan.hpp | 39 ++------ tests/test_arm.cpp | 58 +++++++++++- tests/test_patterns.cpp | 7 ++ tests/test_tls.cpp | 51 +++++++++++ 15 files changed, 493 insertions(+), 152 deletions(-) create mode 100644 ArmXORPattern.hpp delete mode 100644 tests/example_pointer_table.cpp create mode 100644 tests/example_usage.cpp create mode 100644 tests/test_tls.cpp diff --git a/ArmPattern.hpp b/ArmPattern.hpp index 695f3eb..94353e8 100644 --- a/ArmPattern.hpp +++ b/ArmPattern.hpp @@ -658,10 +658,15 @@ namespace patterns { } } + template + class ArmXORPattern; + template class ArmPattern final : public Pattern { std::array m_instructions{}; std::array m_masks{}; + // Builds from an ArmPattern at compile time, then XORs the instructions + friend class ArmXORPattern; public: PATTERNS_CONSTEVAL ArmPattern(const char* p) : m_instructions{}, m_masks{} diff --git a/ArmXORPattern.hpp b/ArmXORPattern.hpp new file mode 100644 index 0000000..9ee890f --- /dev/null +++ b/ArmXORPattern.hpp @@ -0,0 +1,81 @@ +#pragma once +#include "ArmPattern.hpp" +#include "XORPattern.hpp" +#include + +namespace patterns { + + // ArmPattern with the instruction words and masks stored XOR'd, like XORPattern + template + class ArmXORPattern final : public Pattern { + std::array m_instructions{}; + std::array m_masks{}; + std::array m_keys{}; + std::array m_mask_keys{}; + public: + PATTERNS_CONSTEVAL ArmXORPattern(const char* p, uint32_t seed) : + m_instructions{}, m_masks{}, m_keys{}, m_mask_keys{} + { + // Encoded by ArmPattern here at compile time; only the XOR'd words make it into the binary + const ArmPattern plain(p); + length_ = plain.length(); + offset_ = plain.offset(); + // Flags parsed the same way ArmPattern does + size_t options = detail::skip_past(p, 0, '/'); + if (p[options - 1] == '/') handle_options(&p[options]); + align_ = true; + for (size_t j = 0; j < narr; ++j) { + for (uint32_t b = 0; b < 4; ++b) { + const auto at = seed + static_cast(j * 4 + b); + m_keys[j] |= static_cast(detail::generate_key(at)) << (b * 8); + m_mask_keys[j] |= static_cast(detail::generate_key(at, detail::mask_keys)) << (b * 8); + } + m_instructions[j] = plain.m_instructions[j] ^ m_keys[j]; + m_masks[j] = plain.m_masks[j] ^ m_mask_keys[j]; + } + } + virtual const uint8_t* pattern() const override { + return reinterpret_cast(m_instructions.data()); + } + // XOR'd like pattern(); unmask() gives the original + virtual const uint8_t* mask() const override { + return reinterpret_cast(m_masks.data()); + } + uint8_t unmask(size_t idx) const { + return byte(m_masks[idx / 4] ^ m_mask_keys[idx / 4], idx); + } + // Override on [] operator to perform XOR for original byte + virtual uint8_t operator[](size_t idx) const override { + return byte(m_instructions[idx / 4] ^ m_keys[idx / 4], idx); + } + virtual const uint8_t* match(const uint8_t* bytes, size_t size) const override { + // Decoded only on the stack for the duration of the scan; the binary still holds just the XOR'd words + std::array plain{}, plain_mask{}; + // Keys read through volatile so the optimizer can't fold the XOR and emit the plaintext (see XORPattern) + const volatile uint32_t* keys = m_keys.data(); + const volatile uint32_t* mask_keys = m_mask_keys.data(); + for (size_t j = 0; j < narr; ++j) { + plain[j] = m_instructions[j] ^ keys[j]; + plain_mask[j] = m_masks[j] ^ mask_keys[j]; + } + return scan(bytes, size, reinterpret_cast(plain.data()), + reinterpret_cast(plain_mask.data())); + } + private: + // Byte idx % 4 of a word, in memory order like pattern() and mask() + static uint8_t byte(uint32_t word, size_t idx) { + uint8_t bytes[4]; + memcpy(bytes, &word, 4); + return bytes[idx % 4]; + } + }; +} + +template +PATTERNS_CONSTEVAL auto operator""_armxorpattern() { + return patterns::ArmXORPattern(str.data, str.seed); +} + +#ifndef ARM_XOR_PATTERN +#define ARM_XOR_PATTERN(x) ([] { constexpr auto p = patterns::ArmXORPattern(x, patterns::detail::site_seed(__FILE__, __LINE__, __COUNTER__)); return p; }()) +#endif diff --git a/CompileTimePattern.hpp b/CompileTimePattern.hpp index 9da9d20..d619b62 100644 --- a/CompileTimePattern.hpp +++ b/CompileTimePattern.hpp @@ -7,7 +7,7 @@ namespace patterns { namespace detail { constexpr size_t pattern_length(const char* s, size_t nstr) { size_t res = 0; - for (auto i = 0; i < nstr - 1; i += 2) { + for (size_t i = 0; i < nstr - 1; i += 2) { auto c = s[i]; if (c == 'X' || c == 'x') { while (i + 1 < nstr && s[i + 1] != ' ') @@ -39,7 +39,7 @@ namespace patterns { { length_ = narr; auto n = 0; - for (auto i = 0; i < nstr; i += 2) { + for (size_t i = 0; i < nstr; i += 2) { auto ptr = &p[i]; if (p[i] == '?') { m_pattern[n] = 0; diff --git a/Pattern.hpp b/Pattern.hpp index 9380a5f..00ae2d3 100644 --- a/Pattern.hpp +++ b/Pattern.hpp @@ -326,9 +326,11 @@ namespace patterns { #else static constexpr size_t align_size_ = sizeof(void*); #endif + // Patterns are used through Pattern* but never deleted through one; protected keeps the destructor + // trivial (constexpr patterns in C++17) while making `delete` through a Pattern* a compile error + ~Pattern() = default; public: Pattern() = default; - ~Pattern() = default; constexpr uint32_t length() const { return length_; } @@ -352,9 +354,17 @@ namespace patterns { T find(const uint8_t* bytes, size_t size) const { return reinterpret_cast(find(bytes, size)); } - virtual void* find(const uint8_t* bytes, size_t size) const { - const auto hit = scan(bytes, size, pattern(), mask()); - return hit ? get_result(const_cast(hit), bytes + size) : nullptr; + void* find(const uint8_t* bytes, size_t size) const { + const auto hit = match(bytes, size); + return hit ? resolve(hit, bytes + size) : nullptr; + } + // Address of the first match, before the X offset and /d /r are applied + virtual const uint8_t* match(const uint8_t* bytes, size_t size) const { + return scan(bytes, size, pattern(), mask()); + } + // What find() returns for a match from match(). end bounds how far /d may read (arm64 ADRP sequences) + void* resolve(const uint8_t* match, const uint8_t* end) const { + return get_result(const_cast(match), end); } virtual uint8_t operator[](size_t idx) const { return pattern()[idx]; @@ -479,7 +489,7 @@ namespace patterns { ++ptr; } } - void* get_result(uint8_t* address, const uint8_t* end) const { + void* get_result(uint8_t* address, [[maybe_unused]] const uint8_t* end) const { #ifdef PATTERNS_ARM64 if (deref_) { auto insn = reinterpret_cast(address + offset_); @@ -553,7 +563,7 @@ namespace patterns { return nullptr; } #ifndef PATTERNS_ARM64 - const intptr_t relative_value(uint8_t* ptr) const { + intptr_t relative_value(uint8_t* ptr) const { switch (size_) { case 1: return static_cast(*reinterpret_cast(ptr)); @@ -576,4 +586,31 @@ namespace patterns { return detail::stoi(ptr); } }; + + // Resolves many patterns over one range: out[i] = pats[i]->find(start, size). Returns how many matched. + // Faster than a find() per pattern: the range is walked in cache-sized chunks with every pattern run over a + // chunk while it's hot, so memory is read once instead of once per pattern. + // No allocation, threads, CRT state or static initialization, so it's safe from TLS callbacks and DllMain. + inline size_t find_all(const uint8_t* start, size_t size, const Pattern* const* pats, size_t count, void** out, + size_t chunk = 1 << 20) { + if (!chunk) chunk = 1 << 20; + size_t found = 0; + for (size_t i = 0; i < count; ++i) out[i] = nullptr; + // out[i] holds the raw match until the end. Chunks go in order, so the first hit is the lowest address + for (size_t pos = 0; pos < size && found < count; pos += chunk) { + for (size_t i = 0; i < count; ++i) { + if (out[i]) continue; + // Reach length - 1 bytes into the next chunk, so matches straddling the boundary are found, + // but no match starting in the next chunk is + const size_t n = chunk + pats[i]->length() - 1; + if (auto hit = pats[i]->match(start + pos, n < size - pos ? n : size - pos)) { + out[i] = const_cast(hit); + ++found; + } + } + } + for (size_t i = 0; i < count; ++i) + if (out[i]) out[i] = pats[i]->resolve(static_cast(out[i]), start + size); + return found; + } } diff --git a/README.md b/README.md index a28e080..a62a75f 100644 --- a/README.md +++ b/README.md @@ -6,18 +6,21 @@ - CompileTimePattern - XORPattern - ArmPattern +- ArmXORPattern RuntimePattern will allocate the pattern & mask with std::vectors default allocator along with leaving the pattern string in the binary. CompileTimePattern creates the pattern & mask during compile time and stored in a std::array. This means the data is in the binaries module, and no pattern string left. -XORPattern creates the pattern & mask along with a keys array during compile time. Pattern stored is XOR'd bytes, otherwise it acts just the same as the CompileTimePattern. (the pattern() method however will return the XOR'd pattern, access the original bytes with the [] operator) +XORPattern creates the pattern & mask along with two keys arrays during compile time. Pattern and mask are both stored XOR'd (with separate keys), otherwise it acts just the same as the CompileTimePattern. (the pattern() and mask() methods however return the XOR'd bytes; access the original pattern bytes with the [] operator and the original mask bytes with unmask()) ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Instructions not implemented would result in a wildcard portion for those 4 bytes. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). One instruction per line; case, indentation and spacing don't matter, so lines pasted from IDA (`LDR W8, [X8,#0x18]`) or lldb (`ldr w8, [x8, #0x18]`) both work. lldb prints ADRP operands as page counts and branch targets as absolute addresses, so wildcard those (`#?`). Aliases such as `MOV` aren't implemented (they become wildcards); write the underlying `MOVZ`/`ORR` instead. -Every pattern type has a string literal: `_rtpattern` (any standard), and with C++20 `_ctpattern`, `_xorpattern`, `_armpattern`. Each `_xorpattern` gets its own random keys from where it's written (file, line, column) and the build time, so the same pattern written twice is stored differently. Before C++20, use the `COMPILETIME_PATTERN`, `XOR_PATTERN` and `ARM_PATTERN` macros (`XOR_PATTERN` gets per-definition keys from `__FILE__`/`__LINE__`/`__COUNTER__`). +ArmXORPattern (`ArmXORPattern.hpp`) is ArmPattern with the instruction words and masks stored XOR'd, the way XORPattern stores them. Same syntax, flags and results, plus `[]` and unmask() for the originals. Decoded on the stack only while scanning. -Compile-time patterns (`_ctpattern`, `_xorpattern`, `_armpattern` and their macros) are always built at compile time, so the pattern text never ends up in the binary. The class names still do, through RTTI, because `Pattern` is polymorphic. Build with `/GR-` (MSVC) or `-fno-rtti` (clang/gcc) to strip them. +Every pattern type has a string literal: `_rtpattern` (any standard), and with C++20 `_ctpattern`, `_xorpattern`, `_armpattern`, `_armxorpattern`. Each `_xorpattern` and `_armxorpattern` gets its own random keys from where it's written (file, line, column) and the build time, so the same pattern written twice is stored differently. Before C++20, use the `COMPILETIME_PATTERN`, `XOR_PATTERN`, `ARM_PATTERN` and `ARM_XOR_PATTERN` macros (`XOR_PATTERN` and `ARM_XOR_PATTERN` get per-definition keys from `__FILE__`/`__LINE__`/`__COUNTER__`). + +Compile-time patterns (`_ctpattern`, `_xorpattern`, `_armpattern`, `_armxorpattern` and their macros) are always built at compile time, so the pattern text never ends up in the binary. The class names still do, through RTTI, because `Pattern` is polymorphic. Build with `/GR-` (MSVC) or `-fno-rtti` (clang/gcc) to strip them. Scanning uses SSE2 (x86/x64) or NEON (arm64) when available. Define `PATTERNS_NO_SIMD` before including to use a portable 64-bit integer scan instead (e.g. kernel code). @@ -46,6 +49,52 @@ constexpr auto arm_pattern = "/d"_armpattern; ``` +#### Usage +All of these are in `tests/example_usage.cpp` (styles 3-5) and `tests/test_tls.cpp` (style 6), built and checked by the tests. + +**1. One pattern, one scan** +```c++ +constexpr auto p = "48 8B 05 X4 ? ? ? ? 48 85 C0 /d"_ctpattern; +auto ctx = p.find(start, size); +``` + +**2. Many patterns, one pass: `patterns::find_all`** +The range is walked in 1 MB chunks with every pattern run over a chunk while it's in cache, so memory is read once instead of once per pattern (~3.5x faster than a `find()` per pattern). It allocates nothing, starts no threads and needs no CRT state, so it's safe anywhere, including TLS callbacks and DllMain. +```c++ +inline constexpr auto lua_context = "48 8B 05 X4 ? ? ? ? 48 85 C0 /d"_xorpattern; +inline constexpr auto lua_thread = "8B 0D X4 ? ? ? ? 85 C9 /d"_ctpattern; +inline constexpr const patterns::Pattern* list[] = { &lua_context, &lua_thread }; // constant-initialized + +void* out[2]; // out[i] = list[i]->find(start, size) +size_t found = patterns::find_all(start, size, list, 2, out); +``` + +**Where the module comes from.** For globals resolved at static init (3, 4), every scan gets its range from one function whose setup runs once, on first use, from whichever initializer gets there first. That's also the place to decrypt the module, or wait for it. A plain global holding the range doesn't work: the initialization order of globals across files is unspecified. +```c++ +inline std::pair game_module() { + static const auto range = [] { /* find the module, decrypt it... */ return std::pair{ base, size }; }(); + return range; +} +``` + +**3. Per global, at static init.** Simplest; one full pass over the module per global. +```c++ +// Takes the exact pattern type: MSVC rejects a compile-time-only derived temporary bound to const Pattern& (C7595) +template T scan(const P& p) { + const auto [start, size] = game_module(); + return p.template find(start, size); +} +inline lua_State** g_lua_context = scan("48 8B 05 X4 ? ? ? ? 48 85 C0 /d"_ctpattern); +``` + +**4. Batched, at static init.** One list (X-macro); the first of its globals to initialize runs `find_all` for all of them, one pass total. Each global is still one line: `inline T name = static_cast(resolved()[name_index]);` + +**5. Explicit call.** Same list, globals start as `nullptr` and `resolve_pointers(start, size)` fills them when you choose: after decryption, from main, DllMain... + +**6. TLS callback.** Style 5 called from a TLS callback, as the module loads and before the CRT is initialized. The list, patterns and `find_all` need no runtime initialization, so they work there. + +Static init, TLS callbacks and DllMain of a DLL run under the loader lock: never start threads and wait on them there (deadlock). Outside of it, `match()` (raw match address) and `resolve()` (apply the X offset and /d /r) are public for building your own strategies, e.g. spreading chunks over threads (`tests/multi_scan.hpp`, ~10x faster again on 24 threads). + #### Tests & benchmarks `tests/` is a separate CMake project (the library itself stays header-only): ```sh @@ -53,10 +102,11 @@ cmake -S tests -B build -DCMAKE_BUILD_TYPE=Release cmake --build build --config Release ctest --test-dir build -C Release --output-on-failure ./build/bench_scan [module file] # single-pattern speed; with a file, 40 real signatures sampled from it -./build/bench_multi [module file] # many patterns: sequential vs parallel vs cache-chunked +./build/bench_multi [module file] # many patterns: find() per pattern vs find_all vs threaded ``` - `test_patterns`: edge cases, fuzzing against a naive scanner, per-definition XOR keys, and a check that its own binary contains no pattern plaintext. Also built with `PATTERNS_NO_SIMD`. - `test_cpp17`: the macro path. - `test_arm`: ArmPattern encodings checked against clang's assembler (concrete instructions must encode exactly, wildcard patterns must match / not match), line formats, and `/d` resolution. On Apple arm64 it also re-assembles its tables and resolves `/d` on linker-relocated ADRP/ADD/LDR/BL code. -- `example_pointer_table`: globals resolved from one pattern list (X-macro) with a single chunked, multi-threaded scan. See the notes at the top of the file. -- `multi_scan.hpp`: the multi-pattern strategies used by the above. +- `example_usage`: usage styles 3-5 above against a module that has to be decrypted first, logging each result as `[count] name address (rebased address)`. +- `test_tls` (Windows): style 6, `find_all` run from a TLS callback over the executable's own image. +- `multi_scan.hpp`: other multi-pattern strategies (per-pattern threads, chunks over threads) for the benchmark. diff --git a/RuntimePattern.hpp b/RuntimePattern.hpp index ca24dad..d031a9c 100644 --- a/RuntimePattern.hpp +++ b/RuntimePattern.hpp @@ -12,7 +12,7 @@ namespace patterns { public: RuntimePattern(const char* p, size_t len) { int n = 0; - for (int i = 0; i < len; i += 2) { + for (size_t i = 0; i < len; i += 2) { auto ptr = &p[i]; if (*ptr == '?') { m_pattern.push_back(0); diff --git a/XORPattern.hpp b/XORPattern.hpp index 7e649fc..6afaa91 100644 --- a/XORPattern.hpp +++ b/XORPattern.hpp @@ -4,9 +4,11 @@ namespace patterns { namespace detail { - // Key byte from the definition site's seed, mixed with the build time so keys also change every build - constexpr uint8_t generate_key(uint32_t seed) { - uint32_t value = 2166136261u + seed; + // Key byte from the definition site's seed, mixed with the build time so keys also change every build. + // Pattern and mask keys come from different streams: with shared keys, pattern ^ mask would cancel them out + enum key_stream : uint8_t { pattern_keys, mask_keys }; + constexpr uint8_t generate_key(uint32_t seed, key_stream stream = pattern_keys) { + uint32_t value = static_cast(((2166136261u + seed) ^ stream) * 16777619ull); for (char c : __TIME__) value = static_cast((value ^ c) * 16777619ull); return static_cast(value); @@ -18,19 +20,22 @@ namespace patterns { std::array pattern_; std::array mask_; std::array keys_; + std::array mask_keys_; public: PATTERNS_CONSTEVAL XORPattern(const char* p, uint32_t seed) : - pattern_{}, mask_{}, keys_{} + pattern_{}, mask_{}, keys_{}, mask_keys_{} { - for (size_t j = 0; j < narr; ++j) + for (size_t j = 0; j < narr; ++j) { keys_[j] = detail::generate_key(seed + static_cast(j)); + mask_keys_[j] = detail::generate_key(seed + static_cast(j), detail::mask_keys); + } length_ = narr; auto n = 0; - for (auto i = 0; i < nstr; i += 2) { + for (size_t i = 0; i < nstr; i += 2) { auto ptr = &p[i]; if (*ptr == '?') { pattern_[n] = 0 ^ keys_[n]; - mask_[n] = 0; + mask_[n] = 0 ^ mask_keys_[n]; ++n; } // Capture where we have our offset marker 'X' at @@ -52,7 +57,7 @@ namespace patterns { } else if (*ptr != ' ') { pattern_[n] = value(ptr) ^ keys_[n]; - mask_[n] = 0xFF; + mask_[n] = 0xFF ^ mask_keys_[n]; ++n; } else --i; @@ -61,27 +66,29 @@ namespace patterns { virtual const uint8_t* pattern() const override { return pattern_.data(); } + // XOR'd like pattern(); unmask() gives the original virtual const uint8_t* mask() const override { return mask_.data(); } + uint8_t unmask(size_t idx) const { + return mask_[idx] ^ mask_keys_[idx]; + } // Override on [] operator to perform XOR for original byte virtual uint8_t operator[](size_t idx) const override { return pattern_[idx] ^ keys_[idx]; } - virtual void* find(const uint8_t* bytes, size_t size) const override { + virtual const uint8_t* match(const uint8_t* bytes, size_t size) const override { // Decoded only on the stack for the duration of the scan; the binary still holds just the XOR'd bytes - std::array plain{}; + std::array plain{}, plain_mask{}; // Keys read through volatile: otherwise, with a constexpr pattern, the optimizer folds the XOR at compile // time and emits the plaintext as a constant (seen with clang -O3) const volatile uint8_t* keys = keys_.data(); - for (size_t j = 0; j < narr; ++j) + const volatile uint8_t* mask_keys = mask_keys_.data(); + for (size_t j = 0; j < narr; ++j) { plain[j] = pattern_[j] ^ keys[j]; - const auto hit = scan(bytes, size, plain.data(), mask_.data()); - return hit ? get_result(const_cast(hit), bytes + size) : nullptr; - } - template - T find(const uint8_t* bytes, size_t size) const { - return reinterpret_cast(find(bytes, size)); + plain_mask[j] = mask_[j] ^ mask_keys[j]; + } + return scan(bytes, size, plain.data(), plain_mask.data()); } }; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4d8aea7..231da5a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -21,11 +21,11 @@ target_compile_definitions(test_patterns_nosimd PRIVATE PATTERNS_NO_SIMD) add_executable(test_cpp17 test_cpp17.cpp) add_executable(test_arm test_arm.cpp) set_target_properties(test_cpp17 PROPERTIES CXX_STANDARD 17) -add_executable(example_pointer_table example_pointer_table.cpp) +add_executable(example_usage example_usage.cpp) add_executable(bench_scan bench_scan.cpp) add_executable(bench_multi bench_multi.cpp) -foreach(t test_patterns test_patterns_nosimd example_pointer_table bench_multi) +foreach(t test_patterns test_patterns_nosimd bench_multi) target_link_libraries(${t} PRIVATE Threads::Threads) endforeach() @@ -33,4 +33,8 @@ add_test(NAME test_patterns COMMAND test_patterns) add_test(NAME test_patterns_nosimd COMMAND test_patterns_nosimd) add_test(NAME test_cpp17 COMMAND test_cpp17) add_test(NAME test_arm COMMAND test_arm) -add_test(NAME example_pointer_table COMMAND example_pointer_table) +add_test(NAME example_usage COMMAND example_usage) +if(WIN32) + add_executable(test_tls test_tls.cpp) + add_test(NAME test_tls COMMAND test_tls) +endif() diff --git a/tests/bench_multi.cpp b/tests/bench_multi.cpp index cce26f1..b4dd023 100644 --- a/tests/bench_multi.cpp +++ b/tests/bench_multi.cpp @@ -61,7 +61,7 @@ int main(int argc, char** argv) { printf(" %-34s %8.1f ms %s\n", name, best, got == want ? "" : "RESULTS DIFFER"); }; run("sequential (find() per pattern)", [&] { multi::sequential(mod.data(), mod.size(), pats, want); got = want; }); - run("chunked, 1 thread", [&] { multi::chunked(mod.data(), mod.size(), pats, got, 1); }); + run("patterns::find_all (1 thread)", [&] { got.resize(pats.size()); patterns::find_all(mod.data(), mod.size(), pats.data(), pats.size(), got.data()); }); run("parallel per pattern", [&] { multi::per_pattern(mod.data(), mod.size(), pats, got, threads); }); run("chunked, all threads", [&] { multi::chunked(mod.data(), mod.size(), pats, got, threads); }); } diff --git a/tests/example_pointer_table.cpp b/tests/example_pointer_table.cpp deleted file mode 100644 index e256845..0000000 --- a/tests/example_pointer_table.cpp +++ /dev/null @@ -1,81 +0,0 @@ -// Pattern-resolved globals declared in one list, resolved later with one call. -// -// - The list lives in a header next to the globals it fills. -// - Everything generated from it is built at compile time (no static-initializer code), so the table is usable -// from a TLS callback, before the CRT has run any constructors. -// - The module range is only needed at resolve time, so it can run after decryption. -// - Patterns live in read-only data instead of being re-materialized on a function's stack. -// -// Threads: don't start threads and wait on them under the loader lock (DllMain, TLS callbacks, static -// initializers of a DLL). New threads can't start until the lock is released, so waiting on them deadlocks. -// There, resolve with threads = 1 (still ~2x faster than sequential find() calls via chunking), or resolve -// from a thread you start and don't wait on. -#include "CompileTimePattern.hpp" -#include "multi_scan.hpp" -#include -#include - -// ---- pointers.hpp --------------------------------------------------------------------------------------------- -#define GAME_POINTERS(X) \ - X(void*, luaO_nilobject_, "48 83 EC ? E8 ? ? ? ? 48 8D 0D X4 ? ? ? ? 48 3B C1 75 /d") \ - X(void*, dummynode_, "44 8B C3 48 89 47 20 48 8B D7 89 47 40 48 8B CD 48 8D 05 X4 ? /d") \ - X(int*, s_errorHandlerRef, "48 8B CB E8 ? ? ? ? 44 8B 05 X4 ? ? ? ? BA F0 D8 FF FF 48 8B CF E8 /d") - -#define DECLARE_POINTER(type, name, sig) \ - inline type name = nullptr; \ - inline constexpr auto name##_pattern = sig##_ctpattern; -GAME_POINTERS(DECLARE_POINTER) - -struct PointerEntry { - const patterns::Pattern* pattern; - void (*set)(void*); - const char* name; // drop in release builds if the names shouldn't be in the binary -}; -#define POINTER_ENTRY(type, name, sig) { &name##_pattern, [](void* p) { name = static_cast(p); }, #name }, -inline constexpr PointerEntry g_pointers[] = { GAME_POINTERS(POINTER_ENTRY) }; - -// Returns how many resolved -inline size_t resolve_pointers(const uint8_t* start, size_t size, unsigned threads) { - std::vector pats; - for (auto& e : g_pointers) pats.push_back(e.pattern); - std::vector out; - multi::chunked(start, size, pats, out, threads); - size_t found = 0; - for (size_t i = 0; i < out.size(); ++i) { - g_pointers[i].set(out[i]); - found += out[i] != nullptr; - } - return found; -} -// --------------------------------------------------------------------------------------------------------------- - -// Proves the table needs no runtime initialization -constinit const PointerEntry* table_is_constant = g_pointers; - -int main() { -#ifdef PATTERNS_ARM64 - puts("x86 signatures with x86 /d semantics; skipped on arm64"); - return 0; -#endif - // Synthetic 32 MB "module" with the three code sequences planted at known places - std::vector mod(32 << 20); - std::mt19937 rng(9); - for (auto& b : mod) b = static_cast(rng()); - auto plant = [&](size_t at, std::initializer_list bytes) { std::copy(bytes.begin(), bytes.end(), mod.begin() + at); }; - plant(0x0123450, { 0x48, 0x83, 0xEC, 0x28, 0xE8, 1, 2, 3, 4, 0x48, 0x8D, 0x0D, 0x00, 0x10, 0x00, 0x00, 0x48, 0x3B, 0xC1, 0x75 }); - plant(0x0ABCDE0, { 0x44, 0x8B, 0xC3, 0x48, 0x89, 0x47, 0x20, 0x48, 0x8B, 0xD7, 0x89, 0x47, 0x40, 0x48, 0x8B, 0xCD, 0x48, 0x8D, 0x05, - 0x00, 0x20, 0x00, 0x00 }); - plant(0x1F00010, { 0x48, 0x8B, 0xCB, 0xE8, 1, 2, 3, 4, 0x44, 0x8B, 0x05, 0xF0, 0xFF, 0xFF, 0xFF, 0xBA, 0xF0, 0xD8, 0xFF, 0xFF, 0x48, 0x8B, 0xCF, 0xE8 }); - const uint8_t* base = mod.data(); - - const size_t found = resolve_pointers(base, mod.size(), std::thread::hardware_concurrency()); - // /d result = match + X offset + 4 (rel32 size) + rel32 - const bool ok = found == 3 - && luaO_nilobject_ == base + 0x0123450 + 12 + 4 + 0x1000 - && dummynode_ == base + 0x0ABCDE0 + 19 + 4 + 0x2000 - && reinterpret_cast(s_errorHandlerRef) == base + 0x1F00010 + 11 + 4 - 0x10; - printf("%-20s %p\n%-20s %p\n%-20s %p\n", "luaO_nilobject_", luaO_nilobject_, "dummynode_", dummynode_, - "s_errorHandlerRef", static_cast(s_errorHandlerRef)); - puts(ok ? "pointer table ok" : "FAIL pointer table"); - return !ok; -} diff --git a/tests/example_usage.cpp b/tests/example_usage.cpp new file mode 100644 index 0000000..dd37ad4 --- /dev/null +++ b/tests/example_usage.cpp @@ -0,0 +1,153 @@ +// Every way to fill pattern-resolved globals, against a synthetic "module" that must be decrypted before it can be +// scanned. Each result is logged as [count] name address (address rebased to the disassembler's image base). +// +// 1. Per global, at static init: inline T name = scan("name", "..."_ctpattern); +// Simplest. One full pass over the module per global. +// 2. Batched, at static init: one list; the first of its globals to initialize runs find_all for all of them. +// One pass for the whole list. +// 3. Explicit call: same list; resolve_pointers(start, size) whenever the module is ready (main, a TLS +// callback, DllMain, after decryption...). test_tls.cpp runs find_all from a real TLS callback. +// +// The module and anything that must happen before scanning (decryption) live in game_module(), which runs once on +// first use, from whichever initializer gets there first; the initialization order of globals across files is +// unspecified, so a plain global holding the range could still be unset. +// In a DLL, static init, TLS callbacks and DllMain run under the loader lock: don't wait on other threads there. +#include "CompileTimePattern.hpp" +#include "XORPattern.hpp" +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#define BASEADDR 0x100000000 +#else +#define BASEADDR 0x140000000 +#endif + +// ---- the module ------------------------------------------------------------------------------------------------ +// Where the code sequences the patterns look for are planted +constexpr size_t at_one = 0x012340, at_two = 0x0ABCD0, at_four = 0x5F0010, at_five = 0xC00050; +constexpr uint8_t module_key = 0x5A; + +static std::vector load_encrypted_module() { + std::vector m(16 << 20); + std::mt19937 rng(9); + for (auto& b : m) b = static_cast(rng()); + auto plant = [&](size_t at, std::initializer_list bytes) { std::copy(bytes.begin(), bytes.end(), m.begin() + at); }; + plant(at_one, { 0x48, 0x8B, 0x05, 0x11, 0x22, 0x33, 0x44, 0x48, 0x85, 0xC0, 0x74, 0x05, 0xE8 }); + plant(at_two, { 0x40, 0x53, 0x48, 0x83, 0xEC, 0x20, 0x8B, 0x0D, 1, 2, 3, 4, 0x85, 0xC9, 0x0F, 0x84 }); + plant(at_four, { 0x4C, 0x8D, 0x05, 9, 8, 7, 6, 0x48, 0x8B, 0xD3, 0x48, 0x8B, 0xCF, 0xE8 }); + plant(at_five, { 0xF3, 0x0F, 0x10, 0x05, 5, 6, 7, 8, 0x0F, 0x2F, 0xC1, 0x76 }); + for (auto& b : m) b ^= module_key; + return m; +} + +// Every scan gets its range from here. A real one would find the module (GetModuleHandle, dlopen, dyld...) and +// decrypt it or wait until it's decrypted +inline std::pair game_module() { + static const std::vector image = [] { + auto m = load_encrypted_module(); + for (auto& b : m) b ^= module_key; + return m; + }(); + return { image.data(), image.size() }; +} + +// ---- logging --------------------------------------------------------------------------------------------------- +inline std::atomic g_scan_count{0}, g_found_count{0}; + +inline void log_result(const char* name, const void* p) { + const int n = ++g_scan_count; + if (!p) { + printf("[%2d]\t%40s\tnot found\n", n, name); + return; + } + ++g_found_count; + const auto rebased = reinterpret_cast(p) - reinterpret_cast(game_module().first) + BASEADDR; + printf("[%2d]\t%40s\t%p (%p)\n", n, name, p, reinterpret_cast(rebased)); +} + +// ---- 1. per global, at static init ----------------------------------------------------------------------------- +// Takes the exact pattern type: MSVC rejects a compile-time-only derived temporary bound to const Pattern& (C7595) +template T scan(const char* name, const P& pattern) { + const auto [start, size] = game_module(); + const T result = pattern.template find(start, size); + log_result(name, reinterpret_cast(result)); + return result; +} + +inline void* g_one = scan("g_one", "48 8B 05 X ? ? ? ? 48 85 C0 74 ? E8"_ctpattern); +inline uint32_t* g_two = scan("g_two", "40 53 48 83 EC 20 8B 0D X ? ? ? ? 85 C9 0F 84"_xorpattern); +inline void* g_three = scan("g_three", "DE AD BE EF 13 37 C0 DE 00 11"_ctpattern); + +// ---- one list for 2 and 3 -------------------------------------------------------------------------------------- +#define POINTERS(X) \ + X(void*, g_four, "4C 8D 05 X ? ? ? ? 48 8B D3 48 8B CF E8") \ + X(float*, g_five, "F3 0F 10 05 X ? ? ? ? 0F 2F C1 76") \ + X(void*, g_missing, "DE AD BE EF 13 37 C0 DE 00 11 22") + +// All built at compile time (no initializer code), so usable before static init too, e.g. from a TLS callback +#define PATTERN_OF(type, name, sig) inline constexpr auto name##_pattern = sig##_ctpattern; +POINTERS(PATTERN_OF) +#define INDEX_OF(type, name, sig) name##_index, +enum : size_t { POINTERS(INDEX_OF) pointer_count }; +#define ADDRESS_OF(type, name, sig) &name##_pattern, +inline constexpr const patterns::Pattern* pointer_patterns[] = { POINTERS(ADDRESS_OF) }; +#define NAME_OF(type, name, sig) #name, // drop in release builds if the names shouldn't be in the binary +inline constexpr const char* pointer_names[] = { POINTERS(NAME_OF) }; + +// ---- 2. batched, at static init -------------------------------------------------------------------------------- +inline void* const* resolved() { + static const auto out = [] { + std::array r{}; + const auto [start, size] = game_module(); + patterns::find_all(start, size, pointer_patterns, pointer_count, r.data()); + for (size_t i = 0; i < pointer_count; ++i) log_result(pointer_names[i], r[i]); + return r; + }(); + return out.data(); +} +#define BATCHED_GLOBAL(type, name, sig) inline type name = static_cast(resolved()[name##_index]); +POINTERS(BATCHED_GLOBAL) + +// ---- 3. explicit call ------------------------------------------------------------------------------------------ +#define LATE_GLOBAL(type, name, sig) inline type late_##name = nullptr; +POINTERS(LATE_GLOBAL) +#define SETTER_OF(type, name, sig) [](void* p) { late_##name = static_cast(p); }, +inline constexpr void (*pointer_setters[])(void*) = { POINTERS(SETTER_OF) }; + +// No allocation or threads: safe under the loader lock. Returns how many resolved +inline size_t resolve_pointers(const uint8_t* start, size_t size) { + void* out[pointer_count]; + const size_t found = patterns::find_all(start, size, pointer_patterns, pointer_count, out); + for (size_t i = 0; i < pointer_count; ++i) { + pointer_setters[i](out[i]); + log_result(pointer_names[i], out[i]); + } + return found; +} + +// Proves the list needs no runtime initialization +constinit const patterns::Pattern* const* patterns_are_constant = pointer_patterns; +constinit void (* const* setters_are_constant)(void*) = pointer_setters; + +int main() { + puts("-- main: explicit resolve_pointers --"); + const auto [base, size] = game_module(); + const size_t late_found = resolve_pointers(base, size); + printf("%d of %d found\n", g_found_count.load(), g_scan_count.load()); + + const bool ok = g_one == base + at_one + 3 + && g_two == reinterpret_cast(base + at_two + 8) + && !g_three + && g_four == base + at_four + 3 + && g_five == reinterpret_cast(base + at_five + 4) + && !g_missing + && late_found == 2 && late_g_four == g_four && late_g_five == g_five && !late_g_missing + && g_scan_count == 9 && g_found_count == 6; + puts(ok ? "usage ok" : "FAIL usage"); + return !ok; +} diff --git a/tests/multi_scan.hpp b/tests/multi_scan.hpp index 35b3278..3e9497c 100644 --- a/tests/multi_scan.hpp +++ b/tests/multi_scan.hpp @@ -1,6 +1,6 @@ #pragma once -// Strategies for resolving many patterns over one module. Test/benchmark code, not part of the library. -// Built only on the public Pattern API, so any pattern type works (XOR included). +// Strategies for resolving many patterns over one module, for comparison with patterns::find_all. +// Test/benchmark code, not part of the library. Built only on the public Pattern API (match / resolve). #include "Pattern.hpp" #include #include @@ -8,29 +8,6 @@ #include namespace multi { - // Plain copy of a pattern that exposes the raw match address (before /d /r handling), - // needed to merge results from different chunks by lowest address - struct Raw : patterns::Pattern { - std::vector p, m; - explicit Raw(const patterns::Pattern& src) { - length_ = src.length(); - align_ = src.aligned(); - for (uint32_t i = 0; i < length_; ++i) { - p.push_back(src[i]); // operator[] decodes XOR patterns - m.push_back(src.mask()[i]); - } - } - const uint8_t* pattern() const override { return p.data(); } - const uint8_t* mask() const override { return m.data(); } - const uint8_t* match(const uint8_t* bytes, size_t size) const { return scan(bytes, size, p.data(), m.data()); } - }; - - // Applies the pattern's own result handling (/d, /r, X offset) to a known match: a scan of exactly - // length() bytes can only match at that address - inline void* finish(const patterns::Pattern& p, const uint8_t* match) { - return match ? p.find(match, p.length()) : nullptr; - } - // One full scan per pattern, one after another (what a list of find() calls does) inline void sequential(const uint8_t* start, size_t size, const std::vector& pats, std::vector& out) { out.assign(pats.size(), nullptr); @@ -56,12 +33,8 @@ namespace multi { // is read from RAM once instead of once per pattern. Chunks are spread over threads; threads = 1 runs inline. inline void chunked(const uint8_t* start, size_t size, const std::vector& pats, std::vector& out, unsigned threads = std::thread::hardware_concurrency(), size_t chunk = 1 << 20) { - std::vector raw; size_t overlap = 0; - for (auto p : pats) { - raw.emplace_back(*p); - overlap = std::max(overlap, p->length()); - } + for (auto p : pats) overlap = std::max(overlap, p->length()); // Lowest match per pattern; chunks finish out of order, so keep the minimum rather than the first found std::vector> best(pats.size()); for (auto& b : best) b = UINTPTR_MAX; @@ -72,10 +45,10 @@ namespace multi { const uint8_t* s = start + c * chunk; // Overlap into the next chunk so matches straddling the boundary are found const size_t n = std::min(chunk + overlap - 1, size - c * chunk); - for (size_t i = 0; i < raw.size(); ++i) { + for (size_t i = 0; i < pats.size(); ++i) { if (best[i].load(std::memory_order_relaxed) < reinterpret_cast(s)) continue; // an earlier chunk already matched - if (auto hit = raw[i].match(s, n)) { + if (auto hit = pats[i]->match(s, n)) { uintptr_t v = reinterpret_cast(hit), cur = best[i]; while (v < cur && !best[i].compare_exchange_weak(cur, v)) {} } @@ -92,6 +65,6 @@ namespace multi { out.assign(pats.size(), nullptr); for (size_t i = 0; i < pats.size(); ++i) if (best[i] != UINTPTR_MAX) - out[i] = finish(*pats[i], reinterpret_cast(best[i].load())); + out[i] = pats[i]->resolve(reinterpret_cast(best[i].load()), start + size); } } diff --git a/tests/test_arm.cpp b/tests/test_arm.cpp index 46611b5..3399b7c 100644 --- a/tests/test_arm.cpp +++ b/tests/test_arm.cpp @@ -1,9 +1,11 @@ // ArmPattern tests. Exit code 0 = pass. Build with tests/CMakeLists.txt. // The encoder is checked on every platform against encodings produced by clang's assembler. On Apple arm64 the // test also re-assembles every table line to confirm the tables, and resolves /d on linker-relocated code. -#include "ArmPattern.hpp" +#include "ArmXORPattern.hpp" +#include #include #include +#include #define CHECK(x) do { if (!(x)) { printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); exit(1); } } while (0) @@ -201,6 +203,56 @@ static void line_formats() { puts(" line formats ok"); } +// ArmXORPattern: the same encoding and results as ArmPattern, different stored words per definition site +static void xor_patterns() { + constexpr auto plain = ARM_PATTERN("NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"); + constexpr auto a = "NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"_armxorpattern; + constexpr auto b = "NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"_armxorpattern; + auto m = ARM_XOR_PATTERN("NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"); // deliberately not constexpr: still built at compile time + static_assert(a.length() == 12 && a.offset() == 4 && a.deref() && a.aligned()); + CHECK(memcmp(a.pattern(), b.pattern(), 12) != 0 && memcmp(a.pattern(), m.pattern(), 12) != 0); + CHECK(memcmp(a.mask(), b.mask(), 12) != 0 && memcmp(a.mask(), plain.mask(), 12) != 0); + for (size_t k = 0; k < 12; ++k) CHECK(a[k] == plain[k] && m[k] == plain[k] && a.unmask(k) == plain.mask()[k]); + + alignas(4) static const uint32_t code[] = { 0x52800000, 0xD503201F, 0xB9401908, 0xD65F03C0, 0 }; + const auto bytes = reinterpret_cast(code); + CHECK(a.match(bytes, sizeof code) == bytes + 4); + CHECK(b.match(bytes, sizeof code) == bytes + 4); + CHECK(m.match(bytes, sizeof code) == plain.match(bytes, sizeof code)); + CHECK(ARM_XOR_PATTERN("NOP\nX LDR W?, [X?,#?]").find(bytes, sizeof code) == bytes + 8); + puts(" xor patterns ok"); +} + +// The ArmXORPattern's instruction words, masks and source text must not be in our own executable. The needles +// are stored scrambled and rebuilt at runtime, otherwise the test itself would put them in the binary. +static void xor_no_plaintext_in_binary(const char* exe) { + // MOVZ W3, #0x5A3C; MOVK W?, #0x9E1D, LSL #16; LDR W?, [X3,#?] as encoded (wildcard bits zero), then their masks + static const uint32_t scrambled[] = { 0x528B4783 ^ 0xA5A5A5A5, 0x72B3C3A0 ^ 0xA5A5A5A5, 0xB9400060 ^ 0xA5A5A5A5, + 0xFFFFFFFF ^ 0xA5A5A5A5, 0xFFFFFFE0 ^ 0xA5A5A5A5, 0xFFC003E0 ^ 0xA5A5A5A5 }; + volatile uint32_t key = 0xA5A5A5A5; // volatile, or the optimizer folds the descramble back into the plaintext + uint32_t words[3], masks[3]; + for (int i = 0; i < 3; ++i) { words[i] = scrambled[i] ^ key; masks[i] = scrambled[i + 3] ^ key; } + auto xp = ARM_XOR_PATTERN("MOVZ W3, #0x5A3C\nMOVK W?, #0x9E1D, LSL #16\nLDR W?, [X3,#?]"); + CHECK(xp.find(reinterpret_cast(words), sizeof words) == words); + for (size_t k = 0; k < 12; ++k) CHECK(xp.unmask(k) == reinterpret_cast(masks)[k]); + + FILE* f = fopen(exe, "rb"); + if (!f) { printf(" binary check skipped (can't open %s)\n", exe); return; } + std::vector bin; + uint8_t chunk[65536]; + for (size_t n; (n = fread(chunk, 1, sizeof chunk, f)) > 0;) bin.insert(bin.end(), chunk, chunk + n); + fclose(f); + auto contains = [&](const void* needle, size_t len) { + return std::search(bin.begin(), bin.end(), (const uint8_t*)needle, (const uint8_t*)needle + len) != bin.end(); + }; + char text[] = "MOVZ W3, #0x5A3B"; // last digit bumped so this literal isn't the needle + text[sizeof text - 2] = 'C'; + CHECK(!contains(words, sizeof words)); + CHECK(!contains(masks, sizeof masks)); + CHECK(!contains(text, sizeof text - 1)); + puts(" xor: no plaintext in binary ok"); +} + #ifdef PATTERNS_ARM64 // /d on hand-built code: ADRP beyond 2GB, and ADRP followers bounded by the scanned size static void deref_synthetic() { @@ -284,10 +336,12 @@ static void deref_relocated() { } #endif -int main() { +int main(int, char** argv) { concrete(); wildcards(); line_formats(); + xor_patterns(); + xor_no_plaintext_in_binary(argv[0]); #ifdef PATTERNS_ARM64 deref_synthetic(); #endif diff --git a/tests/test_patterns.cpp b/tests/test_patterns.cpp index 0c1960a..83d3aa3 100644 --- a/tests/test_patterns.cpp +++ b/tests/test_patterns.cpp @@ -68,11 +68,15 @@ static void literals_and_macros() { constexpr auto c = "DE AD BE EF 13 37 C0 DE"_xorpattern; constexpr auto d = "DE AD BE EF 13 37 C0 DE"_xorpattern; CHECK(memcmp(a.pattern(), b.pattern(), 8) != 0); CHECK(memcmp(c.pattern(), d.pattern(), 8) != 0); + CHECK(memcmp(a.mask(), b.mask(), 8) != 0); // masks are XOR'd too CHECK(a.find(buf, sizeof buf) == buf + 2); CHECK(d.find(buf, sizeof buf) == buf + 2); constexpr auto ct = "DE AD ? EF X 13"_ctpattern; static_assert(ct.length() == 5 && ct.offset() == 4); CHECK(ct.find(buf, sizeof buf) == buf + 6); + constexpr auto xw = "DE AD ? EF X 13"_xorpattern; + for (size_t k = 0; k < 5; ++k) CHECK(xw.unmask(k) == ct.mask()[k] && xw[k] == ct.pattern()[k]); + CHECK(xw.find(buf, sizeof buf) == buf + 6); #endif auto m1 = XOR_PATTERN("DE AD BE EF 13 37 C0 DE"); // deliberately not constexpr: still built at compile time auto m2 = XOR_PATTERN("DE AD BE EF 13 37 C0 DE"); @@ -175,6 +179,9 @@ static void multi_strategies() { multi::per_pattern(mod.data(), mod.size(), pats, got); CHECK(got == want); for (size_t chunk : { 7, 64, 4096, 1 << 20 }) { + got.assign(pats.size(), &got); // garbage, find_all must overwrite every entry + CHECK(patterns::find_all(mod.data(), mod.size(), pats.data(), pats.size(), got.data(), chunk) == pats.size() - 1); + CHECK(got == want); multi::chunked(mod.data(), mod.size(), pats, got, 1, chunk); CHECK(got == want); multi::chunked(mod.data(), mod.size(), pats, got, 4, chunk); diff --git a/tests/test_tls.cpp b/tests/test_tls.cpp new file mode 100644 index 0000000..650fd80 --- /dev/null +++ b/tests/test_tls.cpp @@ -0,0 +1,51 @@ +// patterns::find_all from a TLS callback: runs as the module is loaded, before the CRT is initialized and before any +// static constructor or main. Scans this executable's own image. Windows only. +#include "XORPattern.hpp" +#include +#ifdef _MSC_VER +#include + +// Data to find in our own image. The patterns are XOR'd, so their stored bytes can't match themselves +extern "C" const uint8_t tls_target[] = { 0x5E, 0xC7, 0x13, 0x9A, 0x40, 0xD2, 0x7B, 0xE1, 0x36, 0xAF, 0x08, 0x95 }; + +inline constexpr auto whole = "5E C7 13 9A 40 D2 7B E1 36 AF 08 95"_xorpattern; +inline constexpr auto with_offset = "5E C7 ? 9A X 40 D2 7B E1"_xorpattern; +inline constexpr auto absent = "5E C7 13 9A 40 D2 7B E1 36 AF 08 96"_xorpattern; +inline constexpr const patterns::Pattern* tls_patterns[] = { &whole, &with_offset, &absent }; + +static void* tls_results[3]; +static size_t tls_found; +static bool callback_ran, callback_saw_crt; + +// Dynamic initializer (a call the compiler can't fold into static data): nonzero once the CRT is up +static const DWORD crt_initialized = GetCurrentProcessId(); + +static void NTAPI on_tls(PVOID module, DWORD reason, PVOID) { + if (reason != DLL_PROCESS_ATTACH) return; + callback_ran = true; + callback_saw_crt = crt_initialized != 0; + const auto base = static_cast(module); + const auto nt = reinterpret_cast(base + reinterpret_cast(base)->e_lfanew); + tls_found = patterns::find_all(base, nt->OptionalHeader.SizeOfImage, tls_patterns, 3, tls_results); +} + +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:_tls_used") +#pragma comment(linker, "/INCLUDE:tls_callback") +#else +#pragma comment(linker, "/INCLUDE:__tls_used") +#pragma comment(linker, "/INCLUDE:_tls_callback") +#endif +#pragma section(".CRT$XLB", read) +extern "C" __declspec(allocate(".CRT$XLB")) const PIMAGE_TLS_CALLBACK tls_callback = on_tls; + +int main() { + const bool ok = callback_ran && !callback_saw_crt && crt_initialized && tls_found == 2 + && tls_results[0] == tls_target && tls_results[1] == tls_target + 4 && !tls_results[2]; + printf("callback ran: %d, before CRT init: %d, found: %zu\n", callback_ran, !callback_saw_crt, tls_found); + puts(ok ? "tls ok" : "FAIL tls"); + return !ok; +} +#else +int main() { puts("TLS callbacks registered via MSVC sections; skipped"); } +#endif From 74e71230f47a47dbdd01ff58a84bde80c71933fd Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 09:19:22 -0700 Subject: [PATCH 08/19] Move headers to include/PatternScanner for header-only use Consumers add include/ to their include path and use #include . Tests include the same way. Co-Authored-By: Claude Opus 5.5 --- README.md | 9 ++++++++- ArmPattern.hpp => include/PatternScanner/ArmPattern.hpp | 0 .../PatternScanner/ArmXORPattern.hpp | 0 .../PatternScanner/CompileTimePattern.hpp | 0 Pattern.hpp => include/PatternScanner/Pattern.hpp | 0 .../PatternScanner/RuntimePattern.hpp | 0 XORPattern.hpp => include/PatternScanner/XORPattern.hpp | 0 tests/CMakeLists.txt | 2 +- tests/bench_multi.cpp | 2 +- tests/bench_scan.cpp | 2 +- tests/example_usage.cpp | 4 ++-- tests/multi_scan.hpp | 2 +- tests/test_arm.cpp | 2 +- tests/test_cpp17.cpp | 6 +++--- tests/test_patterns.cpp | 8 ++++---- tests/test_tls.cpp | 2 +- 16 files changed, 23 insertions(+), 16 deletions(-) rename ArmPattern.hpp => include/PatternScanner/ArmPattern.hpp (100%) rename ArmXORPattern.hpp => include/PatternScanner/ArmXORPattern.hpp (100%) rename CompileTimePattern.hpp => include/PatternScanner/CompileTimePattern.hpp (100%) rename Pattern.hpp => include/PatternScanner/Pattern.hpp (100%) rename RuntimePattern.hpp => include/PatternScanner/RuntimePattern.hpp (100%) rename XORPattern.hpp => include/PatternScanner/XORPattern.hpp (100%) diff --git a/README.md b/README.md index a62a75f..fb120a0 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ # Pattern Scanning +Header-only. Add `include/` to your include path and include the headers you need: +```c++ +#include +#include +``` +(CMake: `target_include_directories(app PRIVATE path/to/PatternScanner/include)`) + #### Classes - Pattern (base class) - RuntimePattern @@ -16,7 +23,7 @@ XORPattern creates the pattern & mask along with two keys arrays during compile ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Instructions not implemented would result in a wildcard portion for those 4 bytes. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). One instruction per line; case, indentation and spacing don't matter, so lines pasted from IDA (`LDR W8, [X8,#0x18]`) or lldb (`ldr w8, [x8, #0x18]`) both work. lldb prints ADRP operands as page counts and branch targets as absolute addresses, so wildcard those (`#?`). Aliases such as `MOV` aren't implemented (they become wildcards); write the underlying `MOVZ`/`ORR` instead. -ArmXORPattern (`ArmXORPattern.hpp`) is ArmPattern with the instruction words and masks stored XOR'd, the way XORPattern stores them. Same syntax, flags and results, plus `[]` and unmask() for the originals. Decoded on the stack only while scanning. +ArmXORPattern (`PatternScanner/ArmXORPattern.hpp`) is ArmPattern with the instruction words and masks stored XOR'd, the way XORPattern stores them. Same syntax, flags and results, plus `[]` and unmask() for the originals. Decoded on the stack only while scanning. Every pattern type has a string literal: `_rtpattern` (any standard), and with C++20 `_ctpattern`, `_xorpattern`, `_armpattern`, `_armxorpattern`. Each `_xorpattern` and `_armxorpattern` gets its own random keys from where it's written (file, line, column) and the build time, so the same pattern written twice is stored differently. Before C++20, use the `COMPILETIME_PATTERN`, `XOR_PATTERN`, `ARM_PATTERN` and `ARM_XOR_PATTERN` macros (`XOR_PATTERN` and `ARM_XOR_PATTERN` get per-definition keys from `__FILE__`/`__LINE__`/`__COUNTER__`). diff --git a/ArmPattern.hpp b/include/PatternScanner/ArmPattern.hpp similarity index 100% rename from ArmPattern.hpp rename to include/PatternScanner/ArmPattern.hpp diff --git a/ArmXORPattern.hpp b/include/PatternScanner/ArmXORPattern.hpp similarity index 100% rename from ArmXORPattern.hpp rename to include/PatternScanner/ArmXORPattern.hpp diff --git a/CompileTimePattern.hpp b/include/PatternScanner/CompileTimePattern.hpp similarity index 100% rename from CompileTimePattern.hpp rename to include/PatternScanner/CompileTimePattern.hpp diff --git a/Pattern.hpp b/include/PatternScanner/Pattern.hpp similarity index 100% rename from Pattern.hpp rename to include/PatternScanner/Pattern.hpp diff --git a/RuntimePattern.hpp b/include/PatternScanner/RuntimePattern.hpp similarity index 100% rename from RuntimePattern.hpp rename to include/PatternScanner/RuntimePattern.hpp diff --git a/XORPattern.hpp b/include/PatternScanner/XORPattern.hpp similarity index 100% rename from XORPattern.hpp rename to include/PatternScanner/XORPattern.hpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 231da5a..4c75b51 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -11,7 +11,7 @@ endif() set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) find_package(Threads REQUIRED) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/..) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../include) add_compile_definitions(_CRT_SECURE_NO_WARNINGS) # fopen in the test code enable_testing() diff --git a/tests/bench_multi.cpp b/tests/bench_multi.cpp index b4dd023..5c91aa1 100644 --- a/tests/bench_multi.cpp +++ b/tests/bench_multi.cpp @@ -1,7 +1,7 @@ // Resolving many patterns over one module: sequential vs parallel per pattern vs cache-chunked. // bench_multi synthetic 96 MB module with 60 planted signatures // bench_multi 60 unique signatures sampled from that file -#include "RuntimePattern.hpp" +#include #include "multi_scan.hpp" #include #include diff --git a/tests/bench_scan.cpp b/tests/bench_scan.cpp index 8665824..bb042ea 100644 --- a/tests/bench_scan.cpp +++ b/tests/bench_scan.cpp @@ -1,7 +1,7 @@ // Single-pattern scan speed. // bench_scan synthetic 64 MB of random bytes // bench_scan also 40 real signatures sampled from that file (e.g. a game exe or dylib) -#include "RuntimePattern.hpp" +#include #include #include #include diff --git a/tests/example_usage.cpp b/tests/example_usage.cpp index dd37ad4..8eb7cc2 100644 --- a/tests/example_usage.cpp +++ b/tests/example_usage.cpp @@ -12,8 +12,8 @@ // first use, from whichever initializer gets there first; the initialization order of globals across files is // unspecified, so a plain global holding the range could still be unset. // In a DLL, static init, TLS callbacks and DllMain run under the loader lock: don't wait on other threads there. -#include "CompileTimePattern.hpp" -#include "XORPattern.hpp" +#include +#include #include #include #include diff --git a/tests/multi_scan.hpp b/tests/multi_scan.hpp index 3e9497c..6da5c0b 100644 --- a/tests/multi_scan.hpp +++ b/tests/multi_scan.hpp @@ -1,7 +1,7 @@ #pragma once // Strategies for resolving many patterns over one module, for comparison with patterns::find_all. // Test/benchmark code, not part of the library. Built only on the public Pattern API (match / resolve). -#include "Pattern.hpp" +#include #include #include #include diff --git a/tests/test_arm.cpp b/tests/test_arm.cpp index 3399b7c..fa6e4b2 100644 --- a/tests/test_arm.cpp +++ b/tests/test_arm.cpp @@ -1,7 +1,7 @@ // ArmPattern tests. Exit code 0 = pass. Build with tests/CMakeLists.txt. // The encoder is checked on every platform against encodings produced by clang's assembler. On Apple arm64 the // test also re-assembles every table line to confirm the tables, and resolves /d on linker-relocated code. -#include "ArmXORPattern.hpp" +#include #include #include #include diff --git a/tests/test_cpp17.cpp b/tests/test_cpp17.cpp index 2e99388..3101037 100644 --- a/tests/test_cpp17.cpp +++ b/tests/test_cpp17.cpp @@ -1,7 +1,7 @@ // The pre-C++20 path: macros instead of literals. Built as C++17 by tests/CMakeLists.txt. -#include "RuntimePattern.hpp" -#include "CompileTimePattern.hpp" -#include "XORPattern.hpp" +#include +#include +#include #include int main() { diff --git a/tests/test_patterns.cpp b/tests/test_patterns.cpp index 83d3aa3..b72a2de 100644 --- a/tests/test_patterns.cpp +++ b/tests/test_patterns.cpp @@ -1,9 +1,9 @@ // Correctness tests. Exit code 0 = pass. Build with tests/CMakeLists.txt. -#include "RuntimePattern.hpp" -#include "CompileTimePattern.hpp" -#include "XORPattern.hpp" +#include +#include +#include #ifdef PATTERNS_ARM64 -#include "ArmPattern.hpp" +#include #endif #include "multi_scan.hpp" #include diff --git a/tests/test_tls.cpp b/tests/test_tls.cpp index 650fd80..2381470 100644 --- a/tests/test_tls.cpp +++ b/tests/test_tls.cpp @@ -1,6 +1,6 @@ // patterns::find_all from a TLS callback: runs as the module is loaded, before the CRT is initialized and before any // static constructor or main. Scans this executable's own image. Windows only. -#include "XORPattern.hpp" +#include #include #ifdef _MSC_VER #include From f7d4350fb2a6c2ab669fad8b69ed9f70cce359eb Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 09:21:33 -0700 Subject: [PATCH 09/19] Add PatternScanner::PatternScanner interface target Root CMakeLists.txt for add_subdirectory / FetchContent users. The tests link the target instead of setting the include path themselves. Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 9 +++++++++ README.md | 6 +++++- tests/CMakeLists.txt | 3 ++- 3 files changed, 16 insertions(+), 2 deletions(-) create mode 100644 CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..c72d9e3 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,9 @@ +# Header-only: for add_subdirectory / FetchContent users. Links as PatternScanner::PatternScanner. +# Tests are a separate project in tests/. +cmake_minimum_required(VERSION 3.16) +project(PatternScanner CXX) + +add_library(PatternScanner INTERFACE) +add_library(PatternScanner::PatternScanner ALIAS PatternScanner) +target_include_directories(PatternScanner INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include) +target_compile_features(PatternScanner INTERFACE cxx_std_17) diff --git a/README.md b/README.md index fb120a0..b0b35d8 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,11 @@ Header-only. Add `include/` to your include path and include the headers you nee #include #include ``` -(CMake: `target_include_directories(app PRIVATE path/to/PatternScanner/include)`) +With CMake (C++17 or newer): +```cmake +add_subdirectory(path/to/PatternScanner) # or FetchContent_Declare + FetchContent_MakeAvailable(PatternScanner) +target_link_libraries(app PRIVATE PatternScanner::PatternScanner) +``` #### Classes - Pattern (base class) diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4c75b51..54a29a2 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -11,7 +11,8 @@ endif() set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) find_package(Threads REQUIRED) -include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../include) +add_subdirectory(.. PatternScanner) +link_libraries(PatternScanner::PatternScanner) add_compile_definitions(_CRT_SECURE_NO_WARNINGS) # fopen in the test code enable_testing() From 07939e426427fbdd7f0a42ccf2deb43871ca33d4 Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 09:46:09 -0700 Subject: [PATCH 10/19] Per-pattern target architecture, RuntimeArmPattern, C++17 ArmPattern - patterns::arch (x86/x64/arm64) decides /d /r, X and /a per pattern instead of by #ifdef; defaults to the compile target, Arm patterns are always arm64. RuntimePattern takes it as an argument, for scanning another architecture's code. - resolve() can report whether the result is an address or a value. - RuntimeArmPattern: ArmPattern from text known at runtime. - ArmPattern/ArmXORPattern compile as C++17 (no std::countr_zero, literals guarded), covered by test_cpp17. - Arch-specific tests now run on every host. Co-Authored-By: Claude Opus 5.5 --- README.md | 12 ++- include/PatternScanner/ArmPattern.hpp | 23 +++++- include/PatternScanner/ArmXORPattern.hpp | 3 + include/PatternScanner/CompileTimePattern.hpp | 5 +- include/PatternScanner/Pattern.hpp | 78 ++++++++++--------- include/PatternScanner/RuntimeArmPattern.hpp | 47 +++++++++++ include/PatternScanner/RuntimePattern.hpp | 13 ++-- include/PatternScanner/XORPattern.hpp | 5 +- tests/test_arm.cpp | 38 +++++++-- tests/test_cpp17.cpp | 10 +++ tests/test_patterns.cpp | 44 ++++++----- 11 files changed, 197 insertions(+), 81 deletions(-) create mode 100644 include/PatternScanner/RuntimeArmPattern.hpp diff --git a/README.md b/README.md index b0b35d8..44c39be 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ target_link_libraries(app PRIVATE PatternScanner::PatternScanner) - XORPattern - ArmPattern - ArmXORPattern +- RuntimeArmPattern RuntimePattern will allocate the pattern & mask with std::vectors default allocator along with leaving the pattern string in the binary. @@ -33,12 +34,17 @@ Every pattern type has a string literal: `_rtpattern` (any standard), and with C Compile-time patterns (`_ctpattern`, `_xorpattern`, `_armpattern`, `_armxorpattern` and their macros) are always built at compile time, so the pattern text never ends up in the binary. The class names still do, through RTTI, because `Pattern` is polymorphic. Build with `/GR-` (MSVC) or `-fno-rtti` (clang/gcc) to strip them. +RuntimeArmPattern (`PatternScanner/RuntimeArmPattern.hpp`) is ArmPattern built at runtime, for assembly only known then (user input, config files). + +#### Target architecture +/d, /r, `X` and /a depend on the architecture of the code being scanned (`patterns::arch::x86`, `x64` or `arm64`). By default that's the one you compile for (`patterns::host_arch`); ArmPattern, ArmXORPattern and RuntimeArmPattern are always arm64. To scan code of another architecture, e.g. from a disassembler plugin, pass it to RuntimePattern: `patterns::RuntimePattern("E8 X4 ? ? ? ? /d", patterns::arch::x86)`. `resolve(match, end, &is_value)` reports whether the result is an address or a value read from the code (/r; arm64 /d on MOVZ, ADD, or a lone LDR/STR offset). + Scanning uses SSE2 (x86/x64) or NEON (arm64) when available. Define `PATTERNS_NO_SIMD` before including to use a portable 64-bit integer scan instead (e.g. kernel code). #### Pattern Flags: - d: Dereference from the found address, or found address + offset ( addr + size + relative address ) -- r: Read the relative value at the address, or at the address + offset (not used on arm) -- a: Only match at aligned addresses. 4 byte alignment x32 & arm64, 8 byte alignment x64 +- r: Read the relative value at the address, or at the address + offset (not used on arm64) +- a: Only match at aligned addresses. 4 byte alignment x86 & arm64, 8 byte alignment x64 - x: This one is used inside of the pattern itself, to show where the offset is at (start address of what we want to capture). This can be followed by a 1/2/4 or 8 for the size of the operand when doing /r or /d @@ -117,7 +123,7 @@ ctest --test-dir build -C Release --output-on-failure ``` - `test_patterns`: edge cases, fuzzing against a naive scanner, per-definition XOR keys, and a check that its own binary contains no pattern plaintext. Also built with `PATTERNS_NO_SIMD`. - `test_cpp17`: the macro path. -- `test_arm`: ArmPattern encodings checked against clang's assembler (concrete instructions must encode exactly, wildcard patterns must match / not match), line formats, and `/d` resolution. On Apple arm64 it also re-assembles its tables and resolves `/d` on linker-relocated ADRP/ADD/LDR/BL code. +- `test_arm`: ArmPattern encodings checked against clang's assembler (concrete instructions must encode exactly, wildcard patterns must match / not match), line formats, RuntimeArmPattern parsing the same, and `/d` resolution. On Apple arm64 it also re-assembles its tables and resolves `/d` on linker-relocated ADRP/ADD/LDR/BL code. - `example_usage`: usage styles 3-5 above against a module that has to be decrypted first, logging each result as `[count] name address (rebased address)`. - `test_tls` (Windows): style 6, `find_all` run from a TLS callback over the executable's own image. - `multi_scan.hpp`: other multi-pattern strategies (per-pattern threads, chunks over threads) for the benchmark. diff --git a/include/PatternScanner/ArmPattern.hpp b/include/PatternScanner/ArmPattern.hpp index 94353e8..aab0904 100644 --- a/include/PatternScanner/ArmPattern.hpp +++ b/include/PatternScanner/ArmPattern.hpp @@ -1,8 +1,8 @@ #pragma once #include #include -#include #include +#include #include "Pattern.hpp" @@ -148,6 +148,18 @@ namespace patterns { return (v >> (n & 63)) | (v << (-n & 63)); } + // std::countr_zero / countl_zero are C++20; the macros need C++17 + constexpr int countr_zero(uint64_t n) { + int c = 0; + for (; c < 64 && !(n >> c & 1); ++c) {} + return c; + } + constexpr int countl_zero(uint64_t n) { + int c = 0; + for (; c < 64 && !(n >> (63 - c) & 1); ++c) {} + return c; + } + constexpr uint64_t clear_trailing_ones(uint64_t n) { return n & (n + 1); } @@ -161,11 +173,11 @@ namespace patterns { if (val == 0 || ~val == 0) return { false, 0, 0, 0 }; - int rotation = std::countr_zero(clear_trailing_ones(val)); + int rotation = countr_zero(clear_trailing_ones(val)); uint64_t normalized = rotate_right(val, rotation & 63); - int zeroes = std::countl_zero(normalized); - int ones = std::countr_zero(~normalized); + int zeroes = countl_zero(normalized); + int ones = countr_zero(~normalized); int size = zeroes + ones; if (rotate_right(val, size & 63) != val) @@ -671,6 +683,7 @@ namespace patterns { PATTERNS_CONSTEVAL ArmPattern(const char* p) : m_instructions{}, m_masks{} { + arch_ = arch::arm64; length_ = narr * 4; uint32_t idx = 0; // Find the options @@ -698,10 +711,12 @@ namespace patterns { }; } +#ifdef PATTERNS_LITERALS template PATTERNS_CONSTEVAL auto operator""_armpattern() { return patterns::ArmPattern(str.data); } +#endif #ifndef ARM_PATTERN #define ARM_PATTERN(x) ([] { constexpr auto p = patterns::ArmPattern(x); return p; }()) diff --git a/include/PatternScanner/ArmXORPattern.hpp b/include/PatternScanner/ArmXORPattern.hpp index 9ee890f..f99952b 100644 --- a/include/PatternScanner/ArmXORPattern.hpp +++ b/include/PatternScanner/ArmXORPattern.hpp @@ -18,6 +18,7 @@ namespace patterns { { // Encoded by ArmPattern here at compile time; only the XOR'd words make it into the binary const ArmPattern plain(p); + arch_ = arch::arm64; length_ = plain.length(); offset_ = plain.offset(); // Flags parsed the same way ArmPattern does @@ -71,10 +72,12 @@ namespace patterns { }; } +#ifdef PATTERNS_LITERALS template PATTERNS_CONSTEVAL auto operator""_armxorpattern() { return patterns::ArmXORPattern(str.data, str.seed); } +#endif #ifndef ARM_XOR_PATTERN #define ARM_XOR_PATTERN(x) ([] { constexpr auto p = patterns::ArmXORPattern(x, patterns::detail::site_seed(__FILE__, __LINE__, __COUNTER__)); return p; }()) diff --git a/include/PatternScanner/CompileTimePattern.hpp b/include/PatternScanner/CompileTimePattern.hpp index d619b62..04f0129 100644 --- a/include/PatternScanner/CompileTimePattern.hpp +++ b/include/PatternScanner/CompileTimePattern.hpp @@ -50,9 +50,8 @@ namespace patterns { else if (*ptr == 'X' || *ptr == 'x') { offset_ = n; if (i + 1 < nstr && p[i + 1] != ' ') { -#ifndef PATTERNS_ARM64 - insn_len_ = get_inst_len_opt(&p[++i]); -#endif + if (arch_ != arch::arm64) + insn_len_ = get_inst_len_opt(&p[i + 1]); while (i + 1 < nstr && p[i + 1] != ' ') ++i; } diff --git a/include/PatternScanner/Pattern.hpp b/include/PatternScanner/Pattern.hpp index 00ae2d3..9e060bb 100644 --- a/include/PatternScanner/Pattern.hpp +++ b/include/PatternScanner/Pattern.hpp @@ -14,8 +14,8 @@ #define PATTERNS_NEON #endif -// arm64 instruction semantics (/d decoding, 4-byte alignment). __arm64__ alone is Apple only: GCC and clang -// elsewhere define __aarch64__, MSVC _M_ARM64 +// Compiling for arm64: patterns default to arm64 semantics (/d decoding, 4-byte alignment). __arm64__ alone is +// Apple only: GCC and clang elsewhere define __aarch64__, MSVC _M_ARM64 #if defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) #define PATTERNS_ARM64 #endif @@ -122,7 +122,6 @@ namespace patterns { if ((at[j] ^ pat[j]) & msk[j]) return false; return true; } -#ifdef PATTERNS_ARM64 template T extract_bitfield(uint32_t insn, unsigned width, unsigned offset) { constexpr size_t int_width = sizeof(int32_t) * 8; @@ -303,29 +302,35 @@ namespace patterns { } return false; } -#endif } + // Instruction set a pattern's /d /r, X and /a follow. Defaults to the one being compiled for; RuntimePattern + // takes another, to scan code of a different architecture (e.g. from a disassembler plugin) + enum class arch : uint8_t { x86, x64, arm64 }; +#ifdef PATTERNS_ARM64 + constexpr arch host_arch = arch::arm64; +#else + constexpr arch host_arch = sizeof(void*) == 8 ? arch::x64 : arch::x86; +#endif + class Pattern { protected: uint32_t length_ = 0; uint32_t offset_ = 0; bool deref_ = false; -#ifndef PATTERNS_ARM64 - // Remaining bytes from the offset when used in dereferencing + // x86/x64 only. Remaining bytes from the offset when used in dereferencing uint32_t insn_len_ = 0; - // Since all arm64 instructions are 32 bit and encoded, just doing deref instead of both deref and relative - // Defaulting 4 byte relative address reading + // x86/x64 only (arm64 instructions are 32 bit and encoded, so arm64 has just deref). Defaulting 4 byte + // relative address reading uint32_t size_ = 4; bool rel_ = false; -#endif bool align_ = false; -#ifdef PATTERNS_ARM64 + // Set before parsing: the flags and X are read per architecture + arch arch_ = host_arch; // arm64 instructions are all 32 bits, so the align scanning will be at the instruction level - static constexpr size_t align_size_ = 4; -#else - static constexpr size_t align_size_ = sizeof(void*); -#endif + constexpr size_t align_size() const { + return arch_ == arch::x64 ? 8 : 4; + } // Patterns are used through Pattern* but never deleted through one; protected keeps the destructor // trivial (constexpr patterns in C++17) while making `delete` through a Pattern* a compile error ~Pattern() = default; @@ -340,14 +345,15 @@ namespace patterns { constexpr bool deref() const { return deref_; } -#ifndef PATTERNS_ARM64 constexpr bool relative() const { return rel_; } -#endif constexpr bool aligned() const { return align_; } + constexpr arch target() const { + return arch_; + } virtual const uint8_t* mask() const = 0; virtual const uint8_t* pattern() const = 0; template @@ -362,9 +368,11 @@ namespace patterns { virtual const uint8_t* match(const uint8_t* bytes, size_t size) const { return scan(bytes, size, pattern(), mask()); } - // What find() returns for a match from match(). end bounds how far /d may read (arm64 ADRP sequences) - void* resolve(const uint8_t* match, const uint8_t* end) const { - return get_result(const_cast(match), end); + // What find() returns for a match from match(). end bounds how far /d may read (arm64 ADRP sequences). + // *is_value is set when the result is a value read from the code (/r; arm64 /d on LDR/STR offsets, MOVZ, + // ADD) rather than an address + void* resolve(const uint8_t* match, const uint8_t* end, bool* is_value = nullptr) const { + return get_result(const_cast(match), end, is_value); } virtual uint8_t operator[](size_t idx) const { return pattern()[idx]; @@ -376,7 +384,7 @@ namespace patterns { if (size < length_) return nullptr; if (!length_) return bytes; const size_t last = size - length_; - const size_t step = align_ ? align_size_ : 1; + const size_t step = align_ ? align_size() : 1; // Aligned scans align to the absolute address, not to wherever the caller's buffer starts size_t i = (step - reinterpret_cast(bytes) % step) % step; // Anchors: first and last of the most specific bytes (most mask bits set). Far apart, so random @@ -463,35 +471,29 @@ namespace patterns { return (detail::get_bits(c[0]) << 4 | detail::get_bits(c[1])); } constexpr void handle_options(const char* ptr) { + const bool arm64 = arch_ == arch::arm64; while (*ptr) { -#ifdef PATTERNS_ARM64 - if (*ptr == 'd') - deref_ = true; -#else if (*ptr == 'd') { if (rel_) throw std::logic_error("Cannot use relative and deref together!"); deref_ = true; } - else if (*ptr == 'r') { + else if (*ptr == 'r' && !arm64) { if (deref_) throw std::logic_error("Cannot use relative and deref together!"); rel_ = true; } -#endif else if (*ptr == 'a') align_ = true; -#ifndef PATTERNS_ARM64 // Check the next character to see what size we're reading at this relative address - else if (*ptr > '0' && (sizeof(void*) == 0x8 ? *ptr < '9' : *ptr < '5')) { + else if (!arm64 && *ptr > '0' && (arch_ == arch::x64 ? *ptr < '9' : *ptr < '5')) { size_ = *ptr - '0'; if ((size_ & (size_ - 1)) != 0) throw std::logic_error("Size is not a valid data type size!"); } -#endif ++ptr; } } - void* get_result(uint8_t* address, [[maybe_unused]] const uint8_t* end) const { -#ifdef PATTERNS_ARM64 - if (deref_) { + void* get_result(uint8_t* address, const uint8_t* end, bool* is_value = nullptr) const { + if (is_value) *is_value = false; + if (arch_ == arch::arm64 && deref_) { auto insn = reinterpret_cast(address + offset_); int64_t offset = 0; bool is_sub = false, is_adrp = false; @@ -537,32 +539,33 @@ namespace patterns { else if (detail::a64_decode_ldr(*insn, nullptr, nullptr, &offset) || detail::a64_decode_ldrh(*insn, nullptr, nullptr, &offset) || detail::a64_decode_str(*insn, nullptr, nullptr, &offset)) { + if (is_value) *is_value = true; return reinterpret_cast(offset); } else if (detail::a64_decode_movz(*insn, &sf, nullptr, &offset) || detail::a64_decode_arithmetic(*insn, nullptr, &sf, nullptr, nullptr, &offset)) { + if (is_value) *is_value = true; return reinterpret_cast((sf ? offset : static_cast(offset))); } else throw std::logic_error("Failed to decode instruction with defined functions"); -#else - if (deref_ || rel_) { + } + else if (arch_ != arch::arm64 && (deref_ || rel_)) { const auto relative_address = relative_value(address + offset_); if (deref_) { return reinterpret_cast(reinterpret_cast(address) + offset_ + insn_len_ + relative_address); } else { + if (is_value) *is_value = true; return reinterpret_cast(relative_address); } -#endif } else { return reinterpret_cast(reinterpret_cast(address) + offset_); } return nullptr; } -#ifndef PATTERNS_ARM64 intptr_t relative_value(uint8_t* ptr) const { switch (size_) { case 1: @@ -577,9 +580,8 @@ namespace patterns { #endif } // Should never hit here - return NULL; + return 0; } -#endif constexpr int32_t get_inst_len_opt(const char* ptr) const { if (*ptr > '9' || *ptr < '0') throw std::logic_error("Invalid data for calculating remaining instruction size!"); diff --git a/include/PatternScanner/RuntimeArmPattern.hpp b/include/PatternScanner/RuntimeArmPattern.hpp new file mode 100644 index 0000000..76e0efe --- /dev/null +++ b/include/PatternScanner/RuntimeArmPattern.hpp @@ -0,0 +1,47 @@ +#pragma once +#include "ArmPattern.hpp" +#include +#include + +namespace patterns { + + // ArmPattern built at runtime, for text only known then (user input, config files). Same syntax and results; + // like RuntimePattern, the text is in the binary if it's a literal. + class RuntimeArmPattern final : public Pattern { + std::vector m_instructions; + std::vector m_masks; + public: + RuntimeArmPattern(const char* text, size_t len) { + arch_ = arch::arm64; + // The parser reads up to a terminator, so work on a terminated copy + const std::string s(text, len); + const char* p = s.c_str(); + // Same parsing as the ArmPattern constructor + size_t options = detail::skip_past(p, 0, '/'); + if (options && p[options - 1] == '/') handle_options(&p[options]); + align_ = true; + for (size_t i = 0; p[i];) { + char line[detail::max_line] = {}; + i = detail::read_line(p, i, line); + if (line[0] == '/') break; + if (!line[0]) continue; + size_t start = 0; + if (line[0] == 'X') { offset_ = static_cast(m_instructions.size() * 4); start = detail::skip_whitespace(line, 1); } + auto encoded = detail::a64_encode_instruction(&line[start]); + m_instructions.push_back(encoded.pattern); + m_masks.push_back(encoded.mask); + } + length_ = static_cast(m_instructions.size() * 4); + } + explicit RuntimeArmPattern(const char* text) : + RuntimeArmPattern(text, strlen(text)) + { + } + virtual const uint8_t* pattern() const override { + return reinterpret_cast(m_instructions.data()); + } + virtual const uint8_t* mask() const override { + return reinterpret_cast(m_masks.data()); + } + }; +} diff --git a/include/PatternScanner/RuntimePattern.hpp b/include/PatternScanner/RuntimePattern.hpp index d031a9c..c089278 100644 --- a/include/PatternScanner/RuntimePattern.hpp +++ b/include/PatternScanner/RuntimePattern.hpp @@ -10,7 +10,9 @@ namespace patterns { std::vector m_pattern; std::vector m_mask; public: - RuntimePattern(const char* p, size_t len) { + // target: the architecture of the code being scanned, for /d /r, X and /a (see patterns::arch) + RuntimePattern(const char* p, size_t len, arch target = host_arch) { + arch_ = target; int n = 0; for (size_t i = 0; i < len; i += 2) { auto ptr = &p[i]; @@ -23,9 +25,8 @@ namespace patterns { else if (*ptr == 'X' || *ptr == 'x') { offset_ = n; if (i + 1 < len && p[i + 1] != ' ') { -#ifndef PATTERNS_ARM64 - insn_len_ = get_inst_len_opt(&p[++i]); -#endif + if (arch_ != arch::arm64) + insn_len_ = get_inst_len_opt(&p[i + 1]); while (i + 1 < len && p[i + 1] != ' ') ++i; } @@ -45,8 +46,8 @@ namespace patterns { } length_ = n; } - RuntimePattern(const char* p) : - RuntimePattern(p, strlen(p)) + RuntimePattern(const char* p, arch target = host_arch) : + RuntimePattern(p, strlen(p), target) { } ~RuntimePattern() = default; diff --git a/include/PatternScanner/XORPattern.hpp b/include/PatternScanner/XORPattern.hpp index 6afaa91..a21d1cf 100644 --- a/include/PatternScanner/XORPattern.hpp +++ b/include/PatternScanner/XORPattern.hpp @@ -42,9 +42,8 @@ namespace patterns { else if (*ptr == 'X' || *ptr == 'x') { offset_ = n; if (i + 1 < nstr && p[i + 1] != ' ') { -#ifndef PATTERNS_ARM64 - insn_len_ = get_inst_len_opt(&p[++i]); -#endif + if (arch_ != arch::arm64) + insn_len_ = get_inst_len_opt(&p[i + 1]); while (i + 1 < nstr && p[i + 1] != ' ') ++i; } diff --git a/tests/test_arm.cpp b/tests/test_arm.cpp index fa6e4b2..3ba9393 100644 --- a/tests/test_arm.cpp +++ b/tests/test_arm.cpp @@ -1,7 +1,8 @@ // ArmPattern tests. Exit code 0 = pass. Build with tests/CMakeLists.txt. -// The encoder is checked on every platform against encodings produced by clang's assembler. On Apple arm64 the -// test also re-assembles every table line to confirm the tables, and resolves /d on linker-relocated code. +// The encoder and /d are checked on every platform against encodings produced by clang's assembler. On Apple arm64 +// the test also re-assembles every table line to confirm the tables, and resolves /d on linker-relocated code. #include +#include #include #include #include @@ -203,6 +204,24 @@ static void line_formats() { puts(" line formats ok"); } +// RuntimeArmPattern: the same parse as ArmPattern, from text only known at runtime +static void runtime_patterns() { + int n = 0; +#define CASE(text, ...) { CHECK(same(ARM_PATTERN(text), patterns::RuntimeArmPattern(text))); ++n; } + CONCRETE(CASE) +#undef CASE +#define CASE(ptext, ...) { CHECK(same(ARM_PATTERN(ptext), patterns::RuntimeArmPattern(ptext))); ++n; } + WILDCARD(CASE) +#undef CASE + constexpr auto canon = ARM_PATTERN("NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"); + CHECK(same(canon, patterns::RuntimeArmPattern("nop\r\n x ldr w8, [x8, #0x18]\n\nret\n/d"))); + CHECK(same(ARM_PATTERN("NOP\nRET\n/a"), patterns::RuntimeArmPattern("NOP\nRET\n/a"))); + const char unterminated[] = { 'N', 'O', 'P', '\n', 'R', 'E', 'T', 'X' }; // the length bounds the text + CHECK(same(ARM_PATTERN("NOP\nRET"), patterns::RuntimeArmPattern(unterminated, 7))); + CHECK(patterns::RuntimeArmPattern("").length() == 0 && patterns::RuntimeArmPattern("/d").length() == 0); + printf(" runtime patterns ok (%d)\n", n); +} + // ArmXORPattern: the same encoding and results as ArmPattern, different stored words per definition site static void xor_patterns() { constexpr auto plain = ARM_PATTERN("NOP\nX LDR W8, [X8,#0x18]\nRET\n/d"); @@ -253,7 +272,6 @@ static void xor_no_plaintext_in_binary(const char* exe) { puts(" xor: no plaintext in binary ok"); } -#ifdef PATTERNS_ARM64 // /d on hand-built code: ADRP beyond 2GB, and ADRP followers bounded by the scanned size static void deref_synthetic() { static const uint32_t far_code[] = { 0x528EEEE9, 0x90480000, 0x91004000, 0 }; // MOVZ W9, #0x7777; ADRP X0, #0x90000000; ADD X0, X0, #0x10 @@ -275,9 +293,18 @@ static void deref_synthetic() { CHECK(adrl.find(bytes, sizeof chain) == page + 0x1000 + 0x10 + 8); CHECK(adrl.find(bytes, 12) == page + 0x1000 + 0x10); // the LDR is past the end: not read CHECK(adrl.find(bytes, 8) == page + 0x1000); // ADRP is the last word scanned + const patterns::RuntimeArmPattern rt("MOVZ W9, #0x7777\nX ADRP X?, #?\n/d"); + bool is_value = true; + CHECK(rt.resolve(rt.match(bytes, sizeof chain), bytes + sizeof chain, &is_value) == reinterpret_cast(page + 0x1000 + 0x10 + 8)); + CHECK(!is_value); + + // Values rather than addresses: the immediate of MOVZ, the offset of a lone LDR + const auto movz = ARM_PATTERN("X MOVZ W9, #0x7777\n/d"); + CHECK(movz.resolve(movz.match(bytes, sizeof chain), bytes + sizeof chain, &is_value) == reinterpret_cast(0x7777) && is_value); + const auto ldr = ARM_PATTERN("ADD X1, X0, #0x10\nX LDR X2, [X1,#?]\n/d"); + CHECK(ldr.resolve(ldr.match(bytes, sizeof chain), bytes + sizeof chain, &is_value) == reinterpret_cast(8) && is_value); puts(" /d on synthetic code ok"); } -#endif #if defined(PATTERNS_ARM64) && defined(__APPLE__) #define SYM(x) "_" x @@ -340,11 +367,10 @@ int main(int, char** argv) { concrete(); wildcards(); line_formats(); + runtime_patterns(); xor_patterns(); xor_no_plaintext_in_binary(argv[0]); -#ifdef PATTERNS_ARM64 deref_synthetic(); -#endif #if defined(PATTERNS_ARM64) && defined(__APPLE__) assembler_agrees(); deref_relocated(); diff --git a/tests/test_cpp17.cpp b/tests/test_cpp17.cpp index 3101037..ec073dd 100644 --- a/tests/test_cpp17.cpp +++ b/tests/test_cpp17.cpp @@ -2,6 +2,8 @@ #include #include #include +#include +#include #include int main() { @@ -10,10 +12,18 @@ int main() { auto x1 = XOR_PATTERN("48 8B ? ? 07"); auto x2 = XOR_PATTERN("48 8B ? ? 07"); patterns::RuntimePattern rp("48 8B ? ? 07"); + alignas(4) static const uint32_t code[] = { 0xD503201F, 0xD65F03C0 }; // NOP; RET + const auto code_bytes = reinterpret_cast(code); + auto arm = ARM_PATTERN("NOP\nRET"); + auto arm_xor = ARM_XOR_PATTERN("NOP\nRET"); + patterns::RuntimeArmPattern arm_rt("NOP\nRET"); const bool ok = ct.find(buf, 8) == buf + 2 && x1.find(buf, 8) == buf + 2 && x2.find(buf, 8) == buf + 2 && rp.find(buf, 8) == buf + 2 + && arm.find(code_bytes, 8) == code_bytes + && arm_xor.find(code_bytes, 8) == code_bytes + && arm_rt.find(code_bytes, 8) == code_bytes && memcmp(x1.pattern(), x2.pattern(), 5) != 0; // per-definition keys puts(ok ? "c++17 macros ok" : "FAIL c++17 macros"); return !ok; diff --git a/tests/test_patterns.cpp b/tests/test_patterns.cpp index b72a2de..1762efd 100644 --- a/tests/test_patterns.cpp +++ b/tests/test_patterns.cpp @@ -2,9 +2,7 @@ #include #include #include -#ifdef PATTERNS_ARM64 #include -#endif #include "multi_scan.hpp" #include #include @@ -52,10 +50,29 @@ static void edge_cases() { // X offset, /r (relative value) and /d (rip-relative deref) on a call: E8 alignas(8) uint8_t code[16] = { 0x90, 0xE8, 0x10, 0x00, 0x00, 0x00, 0xC3 }; CHECK("E8 X ? ? ? ? C3"_rtpattern.find(code, 16) == code + 2); -#ifndef PATTERNS_ARM64 // arm64 has no /r, and its /d decodes arm instructions - CHECK("E8 X4 ? ? ? ? C3 /r"_rtpattern.find(code, 16) == 0x10); - CHECK("E8 X4 ? ? ? ? C3 /d"_rtpattern.find(code, 16) == code + 6 + 0x10); -#endif + // x86 semantics on any host, as a disassembler plugin scanning x86 code needs (arm64 has no /r) + for (auto a : { patterns::arch::x86, patterns::arch::x64 }) { + const patterns::RuntimePattern rel("E8 X4 ? ? ? ? C3 /r", a), deref("E8 X4 ? ? ? ? C3 /d", a); + CHECK(rel.find(code, 16) == 0x10); + CHECK(deref.find(code, 16) == code + 6 + 0x10); + bool is_value = false; + CHECK(rel.resolve(rel.match(code, 16), code + 16, &is_value) == reinterpret_cast(0x10) && is_value); + CHECK(deref.resolve(deref.match(code, 16), code + 16, &is_value) == code + 6 + 0x10 && !is_value); + CHECK(rel.target() == a); + } + // arm64 semantics for byte patterns: /d decodes the instruction at X (BL #0x1000), X isn't parsed + alignas(4) const uint32_t bl[] = { 0xD503201F, 0x94000400 }; + const auto bl_bytes = reinterpret_cast(bl); + CHECK(patterns::RuntimePattern("1F 20 03 D5 X ? ? ? 94 /d", patterns::arch::arm64).find(bl_bytes, 8) == bl_bytes + 4 + 0x1000); + // /a steps by the target's alignment: 8 for x64, 4 for x86 and arm64 + alignas(8) uint8_t al[24] = {}; + al[12] = 0xAB; + CHECK(patterns::RuntimePattern("AB /a", patterns::arch::x64).find(al, 24) == nullptr); + CHECK(patterns::RuntimePattern("AB /a", patterns::arch::x86).find(al, 24) == al + 12); + CHECK(patterns::RuntimePattern("AB /a", patterns::arch::arm64).find(al, 24) == al + 12); + threw = false; + try { patterns::RuntimePattern("E8 X4 ? ? ? ? /r8", patterns::arch::x86); } catch (const std::logic_error&) { threw = true; } + CHECK(!threw); // 8 isn't a size on x86, so it's ignored like any other unknown flag puts(" edge cases ok"); } @@ -136,11 +153,7 @@ static void fuzz(unsigned rounds) { const bool align = rng() % 4 == 0; MaskPattern mp(p, m, align); const auto got = mp.find(buf.data() + base, n); -#ifdef PATTERNS_ARM64 - constexpr size_t align_step = 4; // instruction size, not pointer size (see Pattern::align_size_) -#else - constexpr size_t align_step = sizeof(void*); -#endif + const size_t align_step = mp.target() == patterns::arch::x64 ? 8 : 4; // see Pattern::align_size() const auto want = naive(buf.data() + base, n, p.data(), m.data(), len, align ? align_step : 1); if (got != want) { printf("FAIL fuzz round %u: len=%zu n=%zu base=%zu align=%d\n", r, len, n, base, align); @@ -167,10 +180,9 @@ static void multi_strategies() { if (j >= 4 && j < 8) s += "? "; else { snprintf(hex, sizeof hex, "%02X ", mod[off + j]); s += hex; } } -#ifndef PATTERNS_ARM64 // x86-style /d on random bytes; arm64 /d would try to decode them as instructions s += k % 2 ? "/d" : ""; -#endif - owned.emplace_back(s.c_str()); + // x86-style /d on random bytes; arm64 /d would try to decode them as instructions + owned.emplace_back(s.c_str(), patterns::arch::x64); } owned.emplace_back("01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F"); // absent for (auto& p : owned) pats.push_back(&p); @@ -190,7 +202,6 @@ static void multi_strategies() { puts(" multi-pattern strategies agree ok"); } -#ifdef PATTERNS_ARM64 static void arm_patterns() { constexpr auto adrl = "LDR W?, [X?,#0x18]\n" "X ADRP X?, #?\n" @@ -203,7 +214,6 @@ static void arm_patterns() { CHECK(ret.find(reinterpret_cast(code), sizeof code) == code + 1); puts(" arm64 patterns ok"); } -#endif int main(int, char** argv) { #ifdef PATTERNS_NO_SIMD @@ -220,8 +230,6 @@ int main(int, char** argv) { no_plaintext_in_binary(argv[0]); fuzz(300000); multi_strategies(); -#ifdef PATTERNS_ARM64 arm_patterns(); -#endif puts("all tests passed"); } From 61da333997eb548bd044c9e820711a854614d69b Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 09:46:09 -0700 Subject: [PATCH 11/19] Add IDA plugin: search databases with PatternScanner patterns plugins/ida: find next/previous or all matches, with /d /r results, following the database's architecture. Builds against the user's IDA SDK via ida-cmake; optional idalib test on x86/x64/arm64 ELF fixtures. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 + plugins/ida/CMakeLists.txt | 67 +++++++++++ plugins/ida/README.md | 46 ++++++++ plugins/ida/pattern_scanner.cpp | 196 +++++++++++++++++++++++++++++++ plugins/ida/search.hpp | 110 +++++++++++++++++ plugins/ida/test/arm64.S | 18 +++ plugins/ida/test/test_search.cpp | 112 ++++++++++++++++++ plugins/ida/test/x86.S | 30 +++++ 8 files changed, 582 insertions(+) create mode 100644 plugins/ida/CMakeLists.txt create mode 100644 plugins/ida/README.md create mode 100644 plugins/ida/pattern_scanner.cpp create mode 100644 plugins/ida/search.hpp create mode 100644 plugins/ida/test/arm64.S create mode 100644 plugins/ida/test/test_search.cpp create mode 100644 plugins/ida/test/x86.S diff --git a/README.md b/README.md index 44c39be..23a658a 100644 --- a/README.md +++ b/README.md @@ -127,3 +127,6 @@ ctest --test-dir build -C Release --output-on-failure - `example_usage`: usage styles 3-5 above against a module that has to be decrypted first, logging each result as `[count] name address (rebased address)`. - `test_tls` (Windows): style 6, `find_all` run from a TLS callback over the executable's own image. - `multi_scan.hpp`: other multi-pattern strategies (per-pattern threads, chunks over threads) for the benchmark. + +#### Plugins +- [`plugins/ida`](plugins/ida/README.md): search IDA databases with these patterns (IDA 9.x, Windows/macOS/Linux). diff --git a/plugins/ida/CMakeLists.txt b/plugins/ida/CMakeLists.txt new file mode 100644 index 0000000..04d5e69 --- /dev/null +++ b/plugins/ida/CMakeLists.txt @@ -0,0 +1,67 @@ +# PatternScanner IDA plugin (IDA 9.x). Needs the IDA SDK with ida-cmake, which isn't part of this repo: +# cmake -S plugins/ida -B build-ida -DIDASDK=/path/to/ida-sdk [-DIDABIN=/path/to/ida] +# cmake --build build-ida --config Release +# The plugin is copied to $IDABIN/plugins (IDABIN defaults to the SDK's bin folder). +# +# Optional idalib test of the search, against small ELF files assembled with clang and lld: +# -DIDA_INSTALL_DIR=/path/to/ida then ctest --test-dir build-ida -C Release --output-on-failure +cmake_minimum_required(VERSION 3.27) +project(PatternScannerIDA CXX) +set(CMAKE_CXX_STANDARD 17) + +set(IDASDK "$ENV{IDASDK}" CACHE PATH "IDA SDK: a GitHub checkout (ida-cmake in src/cmake) or a zip SDK with ida-cmake cloned into it") +if(NOT IDASDK) + message(FATAL_ERROR "Set IDASDK (-DIDASDK=... or the environment variable) to your IDA SDK") +endif() +# ida-cmake reads the environment variable +set(ENV{IDASDK} "${IDASDK}") +find_file(IDA_CMAKE_BOOTSTRAP bootstrap.cmake + PATHS "${IDASDK}/src/cmake" "${IDASDK}/cmake" "${IDASDK}/ida-cmake" NO_DEFAULT_PATH NO_CACHE) +if(NOT IDA_CMAKE_BOOTSTRAP) + message(FATAL_ERROR "ida-cmake not found in ${IDASDK}. GitHub SDK: git submodule update --init src/cmake. " + "Zip SDK: git clone https://github.com/allthingsida/ida-cmake ${IDASDK}/ida-cmake") +endif() +include(${IDA_CMAKE_BOOTSTRAP}) +# The bootstrap points the environment variable at src/ for GitHub checkouts; idasdk reads this variable first +set(IDASDK "$ENV{IDASDK}") +find_package(idasdk REQUIRED) + +add_subdirectory(../.. PatternScanner) + +ida_add_plugin(pattern_scanner + SOURCES pattern_scanner.cpp + LIBRARIES PatternScanner::PatternScanner +) + +set(IDA_INSTALL_DIR "" CACHE PATH "IDA install with idalib and a license; builds the search test") +if(IDA_INSTALL_DIR) + find_program(CLANG NAMES clang) + find_program(LLD NAMES ld.lld ld.lld-21 ld.lld-20 ld.lld-19 ld.lld-18) + if(NOT CLANG OR NOT LLD) + message(FATAL_ERROR "The search test assembles its fixtures with clang and ld.lld") + endif() + set(fixtures) + foreach(f x64:x86_64:x86.S x86:i386:x86.S arm64:aarch64:arm64.S) + string(REPLACE ":" ";" f "${f}") + list(GET f 0 name) + list(GET f 1 triple) + list(GET f 2 src) + set(out ${CMAKE_CURRENT_BINARY_DIR}/fixture_${name}.elf) + add_custom_command(OUTPUT ${out} + COMMAND ${CLANG} --target=${triple}-linux-gnu -nostdlib -static --ld-path=${LLD} + -o ${out} ${CMAKE_CURRENT_SOURCE_DIR}/test/${src} + DEPENDS test/${src}) + list(APPEND fixtures ${out}) + endforeach() + add_custom_target(search_fixtures ALL DEPENDS ${fixtures}) + + ida_add_idalib(test_search TYPE EXECUTABLE SOURCES test/test_search.cpp LIBRARIES PatternScanner::PatternScanner) + if(APPLE OR UNIX) + set_target_properties(test_search PROPERTIES BUILD_RPATH "${IDA_INSTALL_DIR}") + endif() + enable_testing() + add_test(NAME test_search COMMAND test_search ${fixtures}) + if(WIN32) + set_tests_properties(test_search PROPERTIES ENVIRONMENT "PATH=${IDA_INSTALL_DIR};$ENV{PATH}") + endif() +endif() diff --git a/plugins/ida/README.md b/plugins/ida/README.md new file mode 100644 index 0000000..8390af9 --- /dev/null +++ b/plugins/ida/README.md @@ -0,0 +1,46 @@ +# PatternScanner for IDA + +Search an IDA database with PatternScanner patterns, the way Search > Sequence of bytes works. IDA 9.x on Windows, +macOS and Linux. + +- **Shift-Alt-B** (also Search > PatternScanner pattern...): enter a byte pattern (`48 8B 05 X4 ? ? ? ? /d`) or + arm64 assembly (one instruction per line, `X` before the one to resolve, flags last). Options: + - **Find all occurrences**: lists every match with its result, segment, function and instruction. Enter or + double-click jumps to the match. + - **Search up**: the previous match instead of the next one. + - **From cursor**: start at the cursor rather than at the start (or end, searching up) of the database. +- **Ctrl-Shift-B**: repeat the last search from the cursor. + +A single search jumps to the match and prints the result (after X and /d /r) to the output window. + +/d, /r, `X` and /a follow the database's architecture, not the machine running IDA: byte patterns in an +arm64 database decode /d as arm64 instructions, and /a steps 4 bytes in 32-bit databases. Matches never span +segments or uninitialized bytes. + +## Build + +Needs the [IDA SDK](https://github.com/HexRaysSA/ida-sdk) with [ida-cmake](https://github.com/allthingsida/ida-cmake) +(neither is part of this repo): +- GitHub checkout of the SDK: `git submodule update --init src/cmake` +- Zip SDK: `git clone https://github.com/allthingsida/ida-cmake /ida-cmake` + +```sh +cmake -S plugins/ida -B build-ida -DIDASDK=/path/to/ida-sdk +cmake --build build-ida --config Release +``` +The plugin (`pattern_scanner.dll` / `.so` / `.dylib`) is written to `$IDABIN/plugins`; IDABIN defaults to the SDK's +`bin` folder. Copy it to your IDA user plugins folder (`%APPDATA%\Hex-Rays\IDA Pro\plugins` on Windows, +`~/.idapro/plugins` on macOS and Linux), or pass `-DIDABIN=/path/to/ida` to build straight into IDA. + +The SDK ships link libraries for MSVC on Windows, clang on macOS and **gcc on Linux**: on Linux, configure with +`-DCMAKE_CXX_COMPILER=g++` if your default compiler is clang. + +## Test + +`test/test_search.cpp` runs the plugin's search (`search.hpp`) through idalib on small x86, x64 and arm64 ELF files +assembled from `test/*.S` with clang and lld. It needs an IDA install with idalib and a license: +```sh +cmake -S plugins/ida -B build-ida -DIDASDK=/path/to/ida-sdk -DIDA_INSTALL_DIR=/path/to/ida +cmake --build build-ida --config Release +ctest --test-dir build-ida -C Release --output-on-failure +``` diff --git a/plugins/ida/pattern_scanner.cpp b/plugins/ida/pattern_scanner.cpp new file mode 100644 index 0000000..309b6a0 --- /dev/null +++ b/plugins/ida/pattern_scanner.cpp @@ -0,0 +1,196 @@ +// PatternScanner for IDA: search the database with PatternScanner byte or arm64 patterns, like Search > Sequence of +// bytes. Find the next/previous match from the cursor, or list every match with what /d /r resolve it to. +#include "search.hpp" +#include + +#include +#include +#include +#include +#include + +namespace { + // Find all: one row per match. Enter / double-click jumps to the match + struct hits_chooser_t : public chooser_t { + static constexpr int widths_[] = { CHCOL_HEX | 16, CHCOL_PLAIN | 32, 8, 24, 48 }; + static constexpr const char* const header_[] = { "Match", "Result", "Segment", "Function", "Instruction" }; + std::vector hits; + qstring title_; + + hits_chooser_t(qstring t, std::vector h) : + chooser_t(0, qnumber(widths_), widths_, header_), hits(std::move(h)), title_(std::move(t)) { + title = title_.c_str(); + } + size_t idaapi get_count() const override { return hits.size(); } + ea_t idaapi get_ea(size_t n) const override { return hits[n].match; } + cbret_t idaapi enter(size_t n) override { + jumpto(hits[n].match); + return cbret_t(); + } + void idaapi get_row(qstrvec_t* cols, int*, chooser_item_attrs_t*, size_t n) const override { + const auto& h = hits[n]; + (*cols)[0].sprnt("%" FMT_EA "X", h.match); + if (!h.resolved) + (*cols)[1] = "can't decode the instruction at X"; + else if (h.is_value) + (*cols)[1].sprnt("value 0x%llX", static_cast(h.result)); + else { + (*cols)[1].sprnt("%" FMT_EA "X", h.result); + qstring name = get_name(h.result); + if (!name.empty()) (*cols)[1].cat_sprnt(" %s", name.c_str()); + } + get_segm_name(&(*cols)[2], getseg(h.match)); + if (func_t* f = get_func(h.match)) get_func_name(&(*cols)[3], f->start_ea); + generate_disasm_line(&(*cols)[4], h.match, GENDSM_REMOVE_TAGS); + } + }; + + struct plugin_ctx_t; + + struct handler_t : public action_handler_t { + plugin_ctx_t& ctx; + bool repeat; + handler_t(plugin_ctx_t& c, bool r) : ctx(c), repeat(r) {} + int idaapi activate(action_activation_ctx_t*) override; + action_state_t idaapi update(action_update_ctx_t*) override { return AST_ENABLE_ALWAYS; } + }; + + constexpr const char search_action[] = "patternscanner:search"; + constexpr const char next_action[] = "patternscanner:next"; + + struct plugin_ctx_t : public plugmod_t { + handler_t search_handler{ *this, false }; + handler_t next_handler{ *this, true }; + // Dialog state, kept between searches + qstring text; + ushort kind = pscan::db_arch() == patterns::arch::arm64 ? pscan::arm64_pattern : pscan::bytes_pattern; + enum : ushort { opt_all = 1, opt_up = 2, opt_cursor = 4 }; + ushort opts = opt_cursor; + // The last pattern, for Search next + std::optional last; + + plugin_ctx_t() { + register_action(ACTION_DESC_LITERAL_PLUGMOD(search_action, "PatternScanner pattern...", &search_handler, + this, "Shift-Alt-B", "Search with a PatternScanner byte or arm64 pattern", -1)); + register_action(ACTION_DESC_LITERAL_PLUGMOD(next_action, "PatternScanner next", &next_handler, + this, "Ctrl-Shift-B", "Repeat the last PatternScanner search from the cursor", -1)); + attach_action_to_menu("Search/", search_action, SETMENU_APP); + attach_action_to_menu("Search/", next_action, SETMENU_APP); + } + + bool idaapi run(size_t) override { + search(); + return true; + } + + void search() { + static const char form[] = + "STARTITEM 0\n" + "BUTTON YES* Search\n" + "PatternScanner\n" + "\n" + "Bytes: 48 8B 05 X4 ? ? ? ? 48 85 C0 /d\n" + "arm64: one instruction per line, X before the one to resolve, flags last (e.g. /d)\n" + "<~P~attern:t:0:64:::>\n" + "<~B~ytes:R>\n" + "<~A~RM64 assembly:R>>\n" + "<~F~ind all occurrences:C>\n" + "\n" + ">\n"; + textctrl_info_t ti; + ti.text = text; + if (ask_form(form, &ti, &kind, &opts) != ASKBTN_YES) return; + text = ti.text; + try { + last = pscan::parse(text, static_cast(kind)); + } + catch (const std::logic_error& e) { + last.reset(); + warning("PatternScanner: %s", e.what()); + return; + } + const auto& p = pscan::base(*last); + if (!p.length()) { + last.reset(); + warning("PatternScanner: the pattern is empty"); + return; + } + if (opts & opt_all) { + show_wait_box("PatternScanner: searching..."); + auto hits = pscan::find_all(p); + hide_wait_box(); + if (hits.empty()) { + msg("PatternScanner: no matches\n"); + return; + } + qstring title; + title.sprnt("PatternScanner: %" FMT_Z " matches for %s", hits.size(), first_line().c_str()); + msg("%s\n", title.c_str()); + (new hits_chooser_t(title, std::move(hits)))->choose(); + return; + } + const bool up = opts & opt_up; + ea_t from = up ? inf_get_max_ea() : inf_get_min_ea(); + if (opts & opt_cursor) from = get_screen_ea(); + next(from); + } + + // Search next: from just past the cursor, in the last direction + void repeat() { + if (!last) { + search(); + return; + } + const ea_t here = get_screen_ea(); + next(opts & opt_up ? here : here + 1); + } + + void next(ea_t from) { + const bool up = opts & opt_up; + pscan::hit_t h; + show_wait_box("PatternScanner: searching..."); + const bool found = pscan::find_next(pscan::base(*last), from, up, &h); + hide_wait_box(); + if (!found) { + msg("PatternScanner: no match %s %" FMT_EA "X\n", up ? "before" : "from", from); + return; + } + jumpto(h.match); + if (!h.resolved) + msg("PatternScanner: match at %" FMT_EA "X, can't decode the instruction at X\n", h.match); + else if (h.is_value) + msg("PatternScanner: match at %" FMT_EA "X, value 0x%llX\n", h.match, static_cast(h.result)); + else + msg("PatternScanner: match at %" FMT_EA "X -> %" FMT_EA "X\n", h.match, h.result); + } + + qstring first_line() const { + qstring s = text; + const size_t nl = s.find('\n'); + if (nl != qstring::npos) s.resize(nl); + return s; + } + }; + + int idaapi handler_t::activate(action_activation_ctx_t*) { + if (repeat) ctx.repeat(); + else ctx.search(); + return 1; + } + + plugmod_t* idaapi init() { + return new plugin_ctx_t; + } +} + +plugin_t PLUGIN = { + IDP_INTERFACE_VERSION, + PLUGIN_MULTI, + init, + nullptr, + nullptr, + "Search the database with PatternScanner patterns", + "Shift-Alt-B: search (next/previous from the cursor, or find all)\nCtrl-Shift-B: search next", + "PatternScanner", + nullptr, +}; diff --git a/plugins/ida/search.hpp b/plugins/ida/search.hpp new file mode 100644 index 0000000..2a09568 --- /dev/null +++ b/plugins/ida/search.hpp @@ -0,0 +1,110 @@ +#pragma once +// Database search with PatternScanner patterns. No UI, so the idalib test runs the same code as the plugin. +// The library headers go first: the SDK's pro.h redefines some C functions the standard headers use. +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace pscan { + enum kind_t : ushort { bytes_pattern, arm64_pattern }; + + // Architecture of the database being searched, not of the IDA running the plugin: decides /d /r, X and /a + inline patterns::arch db_arch() { + if (inf_is_64bit()) + return inf_get_procname() == "ARM" ? patterns::arch::arm64 : patterns::arch::x64; + return patterns::arch::x86; + } + + // Pattern has no virtual destructor (patterns aren't deleted through it), so the concrete type is kept + using pattern_t = std::variant; + + inline const patterns::Pattern& base(const pattern_t& p) { + return std::visit([](const auto& x) -> const patterns::Pattern& { return x; }, p); + } + + // Throws std::logic_error on invalid text + inline pattern_t parse(qstring text, kind_t kind) { + if (kind == arm64_pattern) + return patterns::RuntimeArmPattern(text.c_str(), text.length()); + // Byte patterns pasted over several lines + for (auto& c : text) + if (c == '\n' || c == '\r' || c == '\t') c = ' '; + return patterns::RuntimePattern(text.c_str(), text.length(), db_arch()); + } + + struct hit_t { + ea_t match = BADADDR; // where the pattern matched + ea_t result = BADADDR; // what find() gives: the X offset and /d /r applied + bool is_value = false; // result is a value read from the code (/r, arm64 /d on MOVZ/ADD/LDR offsets) + bool resolved = false; // false when /d couldn't decode the instruction at X + }; + + // Calls cb(hit) for every match starting in [start, end), in address order, until cb returns false. + // Matches don't span segments or uninitialized bytes. Returns false if the user cancelled. + template + bool for_each_match(const patterns::Pattern& p, ea_t start, ea_t end, F&& cb) { + segment_t* s = getseg(start); + if (!s) s = get_next_seg(start); + for (; s && s->start_ea < end; s = get_next_seg(s->start_ea)) { + if (user_cancelled()) return false; + const ea_t lo = s->start_ea > start ? s->start_ea : start; + const size_t n = s->end_ea - lo; + // The buffer sits at the same offset in a 4KB page as lo: /a and arm64 ADRP page math use its address + std::vector raw(n + 0x1000); + uint8_t* buf = raw.data() + ((lo - reinterpret_cast(raw.data())) & 0xFFF); + bytevec_t loaded; + loaded.resize((n + 7) / 8); + if (get_bytes(buf, n, lo, GMB_READALL, loaded.begin()) < 0) return false; + // Scan each run of initialized bytes on its own + for (size_t a = 0; a < n;) { + if (!loaded.test_bit(a)) { ++a; continue; } + size_t b = a; + while (b < n && loaded.test_bit(b)) ++b; + const uint8_t* run_end = buf + b; + for (const uint8_t* cur = buf + a; cur < run_end;) { + const uint8_t* m = p.match(cur, run_end - cur); + if (!m) break; + hit_t h; + h.match = lo + (m - buf); + if (h.match >= end) return true; + try { + const auto r = reinterpret_cast(p.resolve(m, run_end, &h.is_value)); + // Addresses come back in buffer terms; the arithmetic wraps, so targets outside it work too + h.result = h.is_value ? ea_t(r) : ea_t(lo + (r - reinterpret_cast(buf))); + if (!inf_is_64bit()) h.result &= 0xFFFFFFFF; + h.resolved = true; + } + catch (const std::logic_error&) {} + if (!cb(h)) return true; + cur = m + 1; + } + a = b; + } + } + return true; + } + + // Down: the first match at or after from. Up: the last match before from + inline bool find_next(const patterns::Pattern& p, ea_t from, bool up, hit_t* out) { + bool found = false; + if (up) + for_each_match(p, inf_get_min_ea(), from, [&](const hit_t& h) { *out = h; found = true; return true; }); + else + for_each_match(p, from, BADADDR, [&](const hit_t& h) { *out = h; found = true; return false; }); + return found; + } + + // Empty if cancelled + inline std::vector find_all(const patterns::Pattern& p) { + std::vector hits; + if (!for_each_match(p, inf_get_min_ea(), BADADDR, [&](const hit_t& h) { hits.push_back(h); return true; })) + hits.clear(); + return hits; + } +} diff --git a/plugins/ida/test/arm64.S b/plugins/ida/test/arm64.S new file mode 100644 index 0000000..1f4618a --- /dev/null +++ b/plugins/ida/test/arm64.S @@ -0,0 +1,18 @@ +.text +.globl _start +_start: + ret +code_adrp: + movz w9, #0x7777 + adrp x0, target_data + add x0, x0, :lo12:target_data +code_bl: + movz w9, #0x6666 + bl target_func +target_func: + ret + +.data +.balign 16 +.zero 0x2340 // target_data in a later page, at a page offset +target_data: .quad 0x1122334455667788 diff --git a/plugins/ida/test/test_search.cpp b/plugins/ida/test/test_search.cpp new file mode 100644 index 0000000..3053870 --- /dev/null +++ b/plugins/ida/test/test_search.cpp @@ -0,0 +1,112 @@ +// Runs the plugin's search (search.hpp) through idalib on the fixture binaries. Exit code 0 = pass. +// Usage: test_search +#include "../search.hpp" +#include +#include + +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); exit(1); } } while (0) + +static ea_t sym(const char* name) { + const ea_t ea = get_name_ea(BADADDR, name); + if (ea == BADADDR) { printf("FAIL: no symbol %s\n", name); exit(1); } + return ea; +} + +static pscan::hit_t next(const char* text, pscan::kind_t kind, ea_t from, bool up, bool* found = nullptr) { + const auto p = pscan::parse(text, kind); + pscan::hit_t h; + const bool f = pscan::find_next(pscan::base(p), from, up, &h); + if (found) *found = f; + else CHECK(f); + return h; +} + +static std::vector all(const char* text, pscan::kind_t kind = pscan::bytes_pattern) { + return pscan::find_all(pscan::base(pscan::parse(text, kind))); +} + +static void open(const char* path) { + if (open_database(path, true) != 0) { printf("FAIL: can't open %s\n", path); exit(1); } + auto_wait(); +} + +// x86 and x64: find next/previous, find all, /d /r, uninitialized bytes, /a by the database's pointer size +static void x86_family(const char* path, bool x64) { + open(path); + CHECK(pscan::db_arch() == (x64 ? patterns::arch::x64 : patterns::arch::x86)); + const ea_t m1 = sym("marker1"), m2 = sym("marker2"), m3 = sym("marker3"); + const char* marker = "DE AD BE EF 11 22"; + + const auto hits = all(marker); + CHECK(hits.size() == 3 && hits[0].match == m1 && hits[1].match == m2 && hits[2].match == m3); + CHECK(hits[0].resolved && !hits[0].is_value && hits[0].result == m1); + CHECK(next(marker, pscan::bytes_pattern, m1, false).match == m1); // down: from is included + CHECK(next(marker, pscan::bytes_pattern, m1 + 1, false).match == m2); + CHECK(next(marker, pscan::bytes_pattern, m2, true).match == m1); // up: before from + CHECK(next(marker, pscan::bytes_pattern, inf_get_max_ea(), true).match == m3); + bool found = true; + next(marker, pscan::bytes_pattern, m1, true, &found); + CHECK(!found); + next(marker, pscan::bytes_pattern, m3 + 1, false, &found); + CHECK(!found); + + const ea_t call = sym("code_call"), func = sym("target_func"); + const auto d = next("E8 X4 ? ? ? ? A3 B4 /d", pscan::bytes_pattern, 0, false); + CHECK(d.match == call && d.result == func && !d.is_value); + const auto r = next("E8 X4 ? ? ? ? A3 B4 /r", pscan::bytes_pattern, 0, false); + CHECK(r.is_value && r.result == ((func - (call + 5)) & (x64 ? ~0ull : 0xFFFFFFFFull))); + if (x64) { + const auto lea = next("48 8D 05 X4 ? ? ? ? CC CC A1 B2 /d", pscan::bytes_pattern, 0, false); + CHECK(lea.match == sym("code_lea") && lea.result == sym("target_data")); + } + + const ea_t zeros = sym("zeros"); + for (const auto& h : all("00 00 00 00 00 00 00 00")) + CHECK(h.match + 8 <= zeros || h.match >= zeros + 64); + + const auto aligned = all("C7 C8 C9 CA /a"); + CHECK(aligned.size() == (x64 ? 0u : 1u)); + CHECK(all("C7 C8 C9 CA").size() == 1); + + close_database(false); + printf(" %s ok\n", x64 ? "x64" : "x86"); +} + +// arm64: ADRP page math against database addresses, BL, values, byte patterns with arm64 semantics +static void arm64(const char* path) { + open(path); + CHECK(pscan::db_arch() == patterns::arch::arm64); + const ea_t data = sym("target_data"), func = sym("target_func"); + CHECK((data & 0xFFF) != 0); // the fixture puts it mid-page, so a page-misaligned buffer would show + + const auto adrl = next("MOVZ W9, #0x7777\nX ADRP X?, #?\nADD X?, X?, #?\n/d", pscan::arm64_pattern, 0, false); + CHECK(adrl.match == sym("code_adrp") && adrl.result == data && !adrl.is_value); + const auto bl = next("MOVZ W9, #0x6666\nX BL ?\n/d", pscan::arm64_pattern, 0, false); + CHECK(bl.match == sym("code_bl") && bl.result == func); + const auto movz = next("X MOVZ W9, #0x7777\n/d", pscan::arm64_pattern, 0, false); + CHECK(movz.is_value && movz.result == 0x7777); + // A byte pattern in an arm64 database: /d decodes the ADRP and follows the ADD + const auto bytes = next("E9 EE 8E 52 X ? ? ? ? /d", pscan::bytes_pattern, 0, false); + CHECK(bytes.match == sym("code_adrp") && bytes.result == data); + CHECK(all("NOP\nRET", pscan::arm64_pattern).empty()); + + bool threw = false; + try { pscan::parse("48 8G", pscan::bytes_pattern); } catch (const std::logic_error&) { threw = true; } + CHECK(threw); + + close_database(false); + puts(" arm64 ok"); +} + +int main(int argc, char** argv) { + if (argc != 4) { puts("usage: test_search "); return 2; } + if (init_library() != 0) { puts("FAIL: init_library (idalib missing or no license?)"); return 1; } + x86_family(argv[1], true); + x86_family(argv[2], false); + arm64(argv[3]); + puts("all search tests passed"); +} diff --git a/plugins/ida/test/x86.S b/plugins/ida/test/x86.S new file mode 100644 index 0000000..9b2a78d --- /dev/null +++ b/plugins/ida/test/x86.S @@ -0,0 +1,30 @@ +// Shared by the x86 and x64 fixtures (see CMakeLists.txt): same layout, different databases +.intel_syntax noprefix +.text +.globl _start +_start: + ret +marker1: .byte 0xDE, 0xAD, 0xBE, 0xEF, 0x11, 0x22 +code_call: + call target_func + .byte 0xA3, 0xB4 +marker2: .byte 0xDE, 0xAD, 0xBE, 0xEF, 0x11, 0x22 +.balign 8 + .byte 0x90, 0x90, 0x90, 0x90 +unaligned8: .byte 0xC7, 0xC8, 0xC9, 0xCA // 4 mod 8: found by /a in x86 databases only +#ifdef __x86_64__ +code_lea: + lea rax, [rip + target_data] + .byte 0xCC, 0xCC, 0xA1, 0xB2 +#endif +marker3: .byte 0xDE, 0xAD, 0xBE, 0xEF, 0x11, 0x22 +target_func: + ret + +.data +.balign 16 +target_data: .quad 0x1122334455667788 + +.bss +.balign 16 +zeros: .zero 64 // uninitialized: never matched From 10006c5f39ad8055861508430786f11ead625337 Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:02:03 -0700 Subject: [PATCH 12/19] Share the pattern parsers; test for compile-time leftovers in binaries - One byte pattern parser (detail::parse_bytes / Pattern::parse) behind RuntimePattern, CompileTimePattern, XORPattern and pattern_length; one arm64 parser (detail::parse_arm) behind ArmPattern, RuntimeArmPattern, ArmXORPattern and total_instructions. ArmXORPattern no longer needs friend access to ArmPattern. - test_artifacts: no pattern text for any compile-time type, no XOR plaintext bytes/words/masks, at -O0..-O3/-Os with debug info, C++17 and C++20. Built once per pattern so find() inlines and optimizers get the chance to fold. Replaces the narrower self-scan checks in test_patterns and test_arm. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- include/PatternScanner/ArmPattern.hpp | 54 +++--- include/PatternScanner/ArmXORPattern.hpp | 19 ++- include/PatternScanner/CompileTimePattern.hpp | 48 +----- include/PatternScanner/Pattern.hpp | 42 +++++ include/PatternScanner/RuntimeArmPattern.hpp | 27 ++- include/PatternScanner/RuntimePattern.hpp | 36 +--- include/PatternScanner/XORPattern.hpp | 35 +--- tests/CMakeLists.txt | 31 ++++ tests/test_arm.cpp | 35 +--- tests/test_artifacts.cpp | 161 ++++++++++++++++++ tests/test_patterns.cpp | 29 +--- 12 files changed, 295 insertions(+), 225 deletions(-) create mode 100644 tests/test_artifacts.cpp diff --git a/README.md b/README.md index 23a658a..a912e9d 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,8 @@ ctest --test-dir build -C Release --output-on-failure ./build/bench_scan [module file] # single-pattern speed; with a file, 40 real signatures sampled from it ./build/bench_multi [module file] # many patterns: find() per pattern vs find_all vs threaded ``` -- `test_patterns`: edge cases, fuzzing against a naive scanner, per-definition XOR keys, and a check that its own binary contains no pattern plaintext. Also built with `PATTERNS_NO_SIMD`. +- `test_patterns`: edge cases, fuzzing against a naive scanner, per-definition XOR keys. Also built with `PATTERNS_NO_SIMD`. +- `test_artifacts_*`: every compile-time pattern type (macros, and C++20 literals) must leave no pattern text in its binary, and the XOR ones no plaintext bytes, words or masks. Built once per pattern at each optimization level (`-O0`..`-O3`, `-Os`; MSVC `/Od /O1 /O2`) with debug info, as C++17 and C++20: optimizers fold differently at each, and fold only when a pattern's `find()` is inlined, so each pattern gets its own small function in its own build. - `test_cpp17`: the macro path. - `test_arm`: ArmPattern encodings checked against clang's assembler (concrete instructions must encode exactly, wildcard patterns must match / not match), line formats, RuntimeArmPattern parsing the same, and `/d` resolution. On Apple arm64 it also re-assembles its tables and resolves `/d` on linker-relocated ADRP/ADD/LDR/BL code. - `example_usage`: usage styles 3-5 above against a module that has to be decrypted first, logging each result as `[count] name address (rebased address)`. diff --git a/include/PatternScanner/ArmPattern.hpp b/include/PatternScanner/ArmPattern.hpp index aab0904..5e3f604 100644 --- a/include/PatternScanner/ArmPattern.hpp +++ b/include/PatternScanner/ArmPattern.hpp @@ -652,55 +652,55 @@ namespace patterns { constexpr size_t max_line = 128; - // Non-empty lines up to the options line ("/d" etc.); must agree with the ArmPattern constructor - constexpr size_t total_instructions(const char* str, size_t nstr) { + // The ArmPattern syntax: one instruction per line, X before the one to resolve, flags after '/'. Calls + // line(index, instruction) for each instruction (trimmed and uppercased), x(index) for the X line and + // flags(text) for the text after '/'. Reads up to n chars or a terminator. Returns the number of instructions + template + constexpr size_t parse_arm(const char* p, size_t n, Line&& line, X&& x, Flags&& flags) { + const size_t options = skip_past(p, 0, '/'); + if (options && p[options - 1] == '/') flags(&p[options]); size_t count = 0; - for (size_t i = 0; i < nstr - 1 && str[i];) { - char line[max_line] = {}; - i = read_line(str, i, line); - if (line[0] == '/') break; - if (line[0]) ++count; + for (size_t i = 0; i < n && p[i];) { + char text[max_line] = {}; + i = read_line(p, i, text); + if (text[0] == '/') break; + if (!text[0]) continue; + size_t start = 0; + if (text[0] == 'X') { x(count); start = skip_whitespace(text, 1); } + line(count++, &text[start]); } return count; } + constexpr size_t total_instructions(const char* str, size_t nstr) { + return parse_arm(str, nstr - 1, [](size_t, const char*) {}, [](size_t) {}, [](const char*) {}); + } + template constexpr size_t total_instructions(const char (&str)[N]) { return total_instructions(str, N); } } - template - class ArmXORPattern; - template class ArmPattern final : public Pattern { std::array m_instructions{}; std::array m_masks{}; - // Builds from an ArmPattern at compile time, then XORs the instructions - friend class ArmXORPattern; public: PATTERNS_CONSTEVAL ArmPattern(const char* p) : m_instructions{}, m_masks{} { arch_ = arch::arm64; length_ = narr * 4; - uint32_t idx = 0; - // Find the options - size_t options = detail::skip_past(p, 0, '/'); - if (p[options - 1] == '/') handle_options(&p[options]); align_ = true; - for (size_t i = 0; i < nstr && p[i] && idx < narr;) { - char line[detail::max_line] = {}; - i = detail::read_line(p, i, line); - if (line[0] == '/') break; - if (!line[0]) continue; - size_t start = 0; - if (line[0] == 'X') { offset_ = idx * 4; start = detail::skip_whitespace(line, 1); } - auto encoded = detail::a64_encode_instruction(&line[start]); - m_instructions[idx] = encoded.pattern; - m_masks[idx++] = encoded.mask; - } + detail::parse_arm(p, nstr, + [&](size_t i, const char* instruction) { + const auto encoded = detail::a64_encode_instruction(instruction); + m_instructions[i] = encoded.pattern; + m_masks[i] = encoded.mask; + }, + [&](size_t i) { offset_ = static_cast(i * 4); }, + [&](const char* flags) { handle_options(flags); }); } virtual const uint8_t* pattern() const override { return reinterpret_cast(m_instructions.data()); diff --git a/include/PatternScanner/ArmXORPattern.hpp b/include/PatternScanner/ArmXORPattern.hpp index f99952b..f0d0fa8 100644 --- a/include/PatternScanner/ArmXORPattern.hpp +++ b/include/PatternScanner/ArmXORPattern.hpp @@ -16,14 +16,8 @@ namespace patterns { PATTERNS_CONSTEVAL ArmXORPattern(const char* p, uint32_t seed) : m_instructions{}, m_masks{}, m_keys{}, m_mask_keys{} { - // Encoded by ArmPattern here at compile time; only the XOR'd words make it into the binary - const ArmPattern plain(p); arch_ = arch::arm64; - length_ = plain.length(); - offset_ = plain.offset(); - // Flags parsed the same way ArmPattern does - size_t options = detail::skip_past(p, 0, '/'); - if (p[options - 1] == '/') handle_options(&p[options]); + length_ = narr * 4; align_ = true; for (size_t j = 0; j < narr; ++j) { for (uint32_t b = 0; b < 4; ++b) { @@ -31,9 +25,16 @@ namespace patterns { m_keys[j] |= static_cast(detail::generate_key(at)) << (b * 8); m_mask_keys[j] |= static_cast(detail::generate_key(at, detail::mask_keys)) << (b * 8); } - m_instructions[j] = plain.m_instructions[j] ^ m_keys[j]; - m_masks[j] = plain.m_masks[j] ^ m_mask_keys[j]; } + // Encoded here at compile time; only the XOR'd words make it into the binary + detail::parse_arm(p, nstr, + [&](size_t i, const char* instruction) { + const auto encoded = detail::a64_encode_instruction(instruction); + m_instructions[i] = encoded.pattern ^ m_keys[i]; + m_masks[i] = encoded.mask ^ m_mask_keys[i]; + }, + [&](size_t i) { offset_ = static_cast(i * 4); }, + [&](const char* flags) { handle_options(flags); }); } virtual const uint8_t* pattern() const override { return reinterpret_cast(m_instructions.data()); diff --git a/include/PatternScanner/CompileTimePattern.hpp b/include/PatternScanner/CompileTimePattern.hpp index 04f0129..7e08bf8 100644 --- a/include/PatternScanner/CompileTimePattern.hpp +++ b/include/PatternScanner/CompileTimePattern.hpp @@ -6,21 +6,7 @@ namespace patterns { namespace detail { constexpr size_t pattern_length(const char* s, size_t nstr) { - size_t res = 0; - for (size_t i = 0; i < nstr - 1; i += 2) { - auto c = s[i]; - if (c == 'X' || c == 'x') { - while (i + 1 < nstr && s[i + 1] != ' ') - ++i; - continue; - } - else if (c == '/') - break; - else if (c != ' ') - ++res; - else --i; - } - return res; + return parse_bytes(s, nstr - 1, [](size_t, const char*) {}, [](size_t, const char*) {}, [](const char*) {}); } template @@ -38,37 +24,7 @@ namespace patterns { m_pattern{}, m_mask{} { length_ = narr; - auto n = 0; - for (size_t i = 0; i < nstr; i += 2) { - auto ptr = &p[i]; - if (p[i] == '?') { - m_pattern[n] = 0; - m_mask[n] = 0; - ++n; - } - // Capture where we have our offset marker 'X' at - else if (*ptr == 'X' || *ptr == 'x') { - offset_ = n; - if (i + 1 < nstr && p[i + 1] != ' ') { - if (arch_ != arch::arm64) - insn_len_ = get_inst_len_opt(&p[i + 1]); - while (i + 1 < nstr && p[i + 1] != ' ') - ++i; - } - } - // Break from parsing the pattern, since / at the end starts the flags - else if (*ptr == '/') { - ++ptr; - handle_options(ptr); - break; - } - else if (*ptr != ' ') { - m_pattern[n] = value(ptr); - m_mask[n] = 0xFF; - ++n; - } - else --i; - } + parse(p, nstr, [&](size_t n, uint8_t byte, uint8_t mask) { m_pattern[n] = byte; m_mask[n] = mask; }); } virtual const uint8_t* pattern() const override { return m_pattern.data(); diff --git a/include/PatternScanner/Pattern.hpp b/include/PatternScanner/Pattern.hpp index 9e060bb..3161710 100644 --- a/include/PatternScanner/Pattern.hpp +++ b/include/PatternScanner/Pattern.hpp @@ -122,6 +122,32 @@ namespace patterns { if ((at[j] ^ pat[j]) & msk[j]) return false; return true; } + + // The byte pattern syntax ("48 8B ? X4 ? ? ? ? /d"). Calls byte(index, text) for each byte (text at its two hex + // digits or '?'), x(index, len_text) for the X marker (len_text at the digits after X, or nullptr) and + // flags(text) for the text after '/'. Returns the number of bytes + template + constexpr size_t parse_bytes(const char* p, size_t len, Byte&& byte, X&& x, Flags&& flags) { + size_t n = 0; + for (size_t i = 0; i < len; i += 2) { + const char c = p[i]; + if (c == 'X' || c == 'x') { + x(n, i + 1 < len && p[i + 1] != ' ' ? &p[i + 1] : nullptr); + while (i + 1 < len && p[i + 1] != ' ') + ++i; + } + // The flags end the pattern + else if (c == '/') { + flags(&p[i + 1]); + break; + } + else if (c != ' ') + byte(n++, &p[i]); + else --i; + } + return n; + } + template T extract_bitfield(uint32_t insn, unsigned width, unsigned offset) { constexpr size_t int_width = sizeof(int32_t) * 8; @@ -378,6 +404,22 @@ namespace patterns { return pattern()[idx]; } protected: + // Parses a byte pattern: sets the X offset, X and flags, and calls emit(index, byte, mask) for each byte. + // Set arch_ first. Returns the number of bytes + template + constexpr size_t parse(const char* p, size_t len, Emit&& emit) { + return detail::parse_bytes(p, len, + [&](size_t n, const char* c) { + if (*c == '?') emit(n, uint8_t(0), uint8_t(0)); + else emit(n, value(c), uint8_t(0xFF)); + }, + [&](size_t n, const char* len_text) { + offset_ = static_cast(n); + if (len_text && arch_ != arch::arm64) + insn_len_ = get_inst_len_opt(len_text); + }, + [&](const char* flags) { handle_options(flags); }); + } // First match of pat/msk in [bytes, bytes + size). Two anchor bytes are checked first // (16 positions at a time with SIMD), and only positions passing both get a full compare. const uint8_t* scan(const uint8_t* bytes, size_t size, const uint8_t* pat, const uint8_t* msk) const { diff --git a/include/PatternScanner/RuntimeArmPattern.hpp b/include/PatternScanner/RuntimeArmPattern.hpp index 76e0efe..c7650f1 100644 --- a/include/PatternScanner/RuntimeArmPattern.hpp +++ b/include/PatternScanner/RuntimeArmPattern.hpp @@ -13,25 +13,18 @@ namespace patterns { public: RuntimeArmPattern(const char* text, size_t len) { arch_ = arch::arm64; + align_ = true; // The parser reads up to a terminator, so work on a terminated copy const std::string s(text, len); - const char* p = s.c_str(); - // Same parsing as the ArmPattern constructor - size_t options = detail::skip_past(p, 0, '/'); - if (options && p[options - 1] == '/') handle_options(&p[options]); - align_ = true; - for (size_t i = 0; p[i];) { - char line[detail::max_line] = {}; - i = detail::read_line(p, i, line); - if (line[0] == '/') break; - if (!line[0]) continue; - size_t start = 0; - if (line[0] == 'X') { offset_ = static_cast(m_instructions.size() * 4); start = detail::skip_whitespace(line, 1); } - auto encoded = detail::a64_encode_instruction(&line[start]); - m_instructions.push_back(encoded.pattern); - m_masks.push_back(encoded.mask); - } - length_ = static_cast(m_instructions.size() * 4); + const auto count = detail::parse_arm(s.c_str(), len, + [&](size_t, const char* instruction) { + const auto encoded = detail::a64_encode_instruction(instruction); + m_instructions.push_back(encoded.pattern); + m_masks.push_back(encoded.mask); + }, + [&](size_t i) { offset_ = static_cast(i * 4); }, + [&](const char* flags) { handle_options(flags); }); + length_ = static_cast(count * 4); } explicit RuntimeArmPattern(const char* text) : RuntimeArmPattern(text, strlen(text)) diff --git a/include/PatternScanner/RuntimePattern.hpp b/include/PatternScanner/RuntimePattern.hpp index c089278..f2a33c4 100644 --- a/include/PatternScanner/RuntimePattern.hpp +++ b/include/PatternScanner/RuntimePattern.hpp @@ -13,38 +13,10 @@ namespace patterns { // target: the architecture of the code being scanned, for /d /r, X and /a (see patterns::arch) RuntimePattern(const char* p, size_t len, arch target = host_arch) { arch_ = target; - int n = 0; - for (size_t i = 0; i < len; i += 2) { - auto ptr = &p[i]; - if (*ptr == '?') { - m_pattern.push_back(0); - m_mask.push_back(0); - ++n; - } - // Capture where we have our offset marker 'X' at - else if (*ptr == 'X' || *ptr == 'x') { - offset_ = n; - if (i + 1 < len && p[i + 1] != ' ') { - if (arch_ != arch::arm64) - insn_len_ = get_inst_len_opt(&p[i + 1]); - while (i + 1 < len && p[i + 1] != ' ') - ++i; - } - } - // Break from parsing the pattern, since / at the end starts the flags - else if (*ptr == '/') { - ++ptr; - handle_options(ptr); - break; - } - else if (*ptr != ' ') { - m_pattern.push_back(value(ptr)); - m_mask.push_back(0xFF); - ++n; - } - else i--; - } - length_ = n; + length_ = static_cast(parse(p, len, [&](size_t, uint8_t byte, uint8_t mask) { + m_pattern.push_back(byte); + m_mask.push_back(mask); + })); } RuntimePattern(const char* p, arch target = host_arch) : RuntimePattern(p, strlen(p), target) diff --git a/include/PatternScanner/XORPattern.hpp b/include/PatternScanner/XORPattern.hpp index a21d1cf..2f88d85 100644 --- a/include/PatternScanner/XORPattern.hpp +++ b/include/PatternScanner/XORPattern.hpp @@ -30,37 +30,10 @@ namespace patterns { mask_keys_[j] = detail::generate_key(seed + static_cast(j), detail::mask_keys); } length_ = narr; - auto n = 0; - for (size_t i = 0; i < nstr; i += 2) { - auto ptr = &p[i]; - if (*ptr == '?') { - pattern_[n] = 0 ^ keys_[n]; - mask_[n] = 0 ^ mask_keys_[n]; - ++n; - } - // Capture where we have our offset marker 'X' at - else if (*ptr == 'X' || *ptr == 'x') { - offset_ = n; - if (i + 1 < nstr && p[i + 1] != ' ') { - if (arch_ != arch::arm64) - insn_len_ = get_inst_len_opt(&p[i + 1]); - while (i + 1 < nstr && p[i + 1] != ' ') - ++i; - } - } - // Break from parsing the pattern, since / at the end starts the flags - else if (*ptr == '/') { - ++ptr; - handle_options(ptr); - break; - } - else if (*ptr != ' ') { - pattern_[n] = value(ptr) ^ keys_[n]; - mask_[n] = 0xFF ^ mask_keys_[n]; - ++n; - } - else --i; - } + parse(p, nstr, [&](size_t n, uint8_t byte, uint8_t mask) { + pattern_[n] = byte ^ keys_[n]; + mask_[n] = mask ^ mask_keys_[n]; + }); } virtual const uint8_t* pattern() const override { return pattern_.data(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 54a29a2..c3a988b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -39,3 +39,34 @@ if(WIN32) add_executable(test_tls test_tls.cpp) add_test(NAME test_tls COMMAND test_tls) endif() + +# No pattern text / XOR plaintext left in the binary: one build per pattern (see test_artifacts.cpp for why), at every +# optimization level, with debug info, C++17 (macros) and C++20 (macros and literals) +if(MSVC) + set(artifact_opts Od O1 O2) + set(artifact_flag /) + set(artifact_debug /Zi) +else() + set(artifact_opts O0 O1 O2 O3 Os) + set(artifact_flag -) + set(artifact_debug -g) +endif() +set(artifact_cases ct_macro xor_macro arm_macro armxor_macro ct_literal xor_literal arm_literal armxor_literal) +foreach(std 17 20) + foreach(opt ${artifact_opts}) + set(id 0) + foreach(case ${artifact_cases}) + math(EXPR id "${id} + 1") + if(std EQUAL 17 AND id GREATER 4) # literals need C++20 + break() + endif() + set(t test_artifacts_cpp${std}_${opt}_${case}) + add_executable(${t} test_artifacts.cpp) + set_target_properties(${t} PROPERTIES CXX_STANDARD ${std}) + target_compile_definitions(${t} PRIVATE ONLY_CASE=${id}) + # After the build type's flags, so this level wins + target_compile_options(${t} PRIVATE ${artifact_flag}${opt} ${artifact_debug}) + add_test(NAME ${t} COMMAND ${t}) + endforeach() + endforeach() +endforeach() diff --git a/tests/test_arm.cpp b/tests/test_arm.cpp index 3ba9393..16a16c0 100644 --- a/tests/test_arm.cpp +++ b/tests/test_arm.cpp @@ -3,10 +3,8 @@ // the test also re-assembles every table line to confirm the tables, and resolves /d on linker-relocated code. #include #include -#include #include #include -#include #define CHECK(x) do { if (!(x)) { printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); exit(1); } } while (0) @@ -242,36 +240,6 @@ static void xor_patterns() { puts(" xor patterns ok"); } -// The ArmXORPattern's instruction words, masks and source text must not be in our own executable. The needles -// are stored scrambled and rebuilt at runtime, otherwise the test itself would put them in the binary. -static void xor_no_plaintext_in_binary(const char* exe) { - // MOVZ W3, #0x5A3C; MOVK W?, #0x9E1D, LSL #16; LDR W?, [X3,#?] as encoded (wildcard bits zero), then their masks - static const uint32_t scrambled[] = { 0x528B4783 ^ 0xA5A5A5A5, 0x72B3C3A0 ^ 0xA5A5A5A5, 0xB9400060 ^ 0xA5A5A5A5, - 0xFFFFFFFF ^ 0xA5A5A5A5, 0xFFFFFFE0 ^ 0xA5A5A5A5, 0xFFC003E0 ^ 0xA5A5A5A5 }; - volatile uint32_t key = 0xA5A5A5A5; // volatile, or the optimizer folds the descramble back into the plaintext - uint32_t words[3], masks[3]; - for (int i = 0; i < 3; ++i) { words[i] = scrambled[i] ^ key; masks[i] = scrambled[i + 3] ^ key; } - auto xp = ARM_XOR_PATTERN("MOVZ W3, #0x5A3C\nMOVK W?, #0x9E1D, LSL #16\nLDR W?, [X3,#?]"); - CHECK(xp.find(reinterpret_cast(words), sizeof words) == words); - for (size_t k = 0; k < 12; ++k) CHECK(xp.unmask(k) == reinterpret_cast(masks)[k]); - - FILE* f = fopen(exe, "rb"); - if (!f) { printf(" binary check skipped (can't open %s)\n", exe); return; } - std::vector bin; - uint8_t chunk[65536]; - for (size_t n; (n = fread(chunk, 1, sizeof chunk, f)) > 0;) bin.insert(bin.end(), chunk, chunk + n); - fclose(f); - auto contains = [&](const void* needle, size_t len) { - return std::search(bin.begin(), bin.end(), (const uint8_t*)needle, (const uint8_t*)needle + len) != bin.end(); - }; - char text[] = "MOVZ W3, #0x5A3B"; // last digit bumped so this literal isn't the needle - text[sizeof text - 2] = 'C'; - CHECK(!contains(words, sizeof words)); - CHECK(!contains(masks, sizeof masks)); - CHECK(!contains(text, sizeof text - 1)); - puts(" xor: no plaintext in binary ok"); -} - // /d on hand-built code: ADRP beyond 2GB, and ADRP followers bounded by the scanned size static void deref_synthetic() { static const uint32_t far_code[] = { 0x528EEEE9, 0x90480000, 0x91004000, 0 }; // MOVZ W9, #0x7777; ADRP X0, #0x90000000; ADD X0, X0, #0x10 @@ -363,13 +331,12 @@ static void deref_relocated() { } #endif -int main(int, char** argv) { +int main() { concrete(); wildcards(); line_formats(); runtime_patterns(); xor_patterns(); - xor_no_plaintext_in_binary(argv[0]); deref_synthetic(); #if defined(PATTERNS_ARM64) && defined(__APPLE__) assembler_agrees(); diff --git a/tests/test_artifacts.cpp b/tests/test_artifacts.cpp new file mode 100644 index 0000000..54ceb73 --- /dev/null +++ b/tests/test_artifacts.cpp @@ -0,0 +1,161 @@ +// Compile-time patterns must leave nothing behind in the binary: no pattern text for any of them, and no plaintext +// bytes, words or masks for the XOR ones. tests/CMakeLists.txt builds this once per pattern (ONLY_CASE) at every +// optimization level, with debug info, as C++17 (macros) and C++20 (macros and literals). Exit code 0 = pass. +// Needles are written reversed and turned around at runtime, so this file doesn't put them in the binary itself. +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(x) do { if (!(x)) { printf("FAIL %s:%d: %s\n", __FILE__, __LINE__, #x); exit(1); } } while (0) + +static std::vector bin; + +static bool contains(const void* needle, size_t len) { + const auto n = static_cast(needle); + return std::search(bin.begin(), bin.end(), n, n + len) != bin.end(); +} + +// Reversed text, turned around at runtime. Read through volatile: otherwise the optimizer reverses the literal at +// compile time and puts the needle in the binary itself (seen with clang -O2, C++20) +static std::string forward(const char* reversed) { + const volatile char* v = reversed; + std::string s; + for (size_t i = 0; v[i]; ++i) s.insert(s.begin(), v[i]); + return s; +} + +// Space-separated hex bytes, ? as 0 (how wildcards are stored) +static std::vector hex_bytes(const std::string& text) { + std::vector out; + std::istringstream in(text); + for (std::string tok; in >> tok;) + out.push_back(tok == "?" ? 0 : static_cast(std::stoul(tok, nullptr, 16))); + return out; +} + +// Needle text, and code containing the pattern for it to find +struct Case { + std::string text; + // Fixed size, like most real scans of a known buffer: with a size the optimizer can see, it unrolls and folds more + uint8_t code[64] = {}; + std::vector bytes, masks; // masks: arm64 only + size_t at = 0; +}; + +static Case byte_case(const char* reversed) { + Case c; + c.text = forward(reversed); + c.bytes = hex_bytes(c.text); + memset(c.code, 0x90, sizeof c.code); + c.at = 3; + memcpy(c.code + c.at, c.bytes.data(), c.bytes.size()); + return c; +} + +static Case arm_case(const char* reversed) { + Case c; + c.text = forward(reversed); + const patterns::RuntimeArmPattern rt(c.text.c_str()); + c.bytes.assign(rt.pattern(), rt.pattern() + rt.length()); + c.masks.assign(rt.mask(), rt.mask() + rt.length()); + for (size_t i = 0; i < sizeof c.code; i += 4) memcpy(&c.code[i], "\x1F\x20\x03\xD5", 4); // NOPs around it + c.at = 8; + memcpy(c.code + c.at, c.bytes.data(), c.bytes.size()); + return c; +} + +// No text in the binary (each line on its own for arm64, so partly folded text is caught too); for XOR patterns no +// plaintext bytes or masks either (masks only as a whole: single mask words like FFFFFFFF are everywhere). The +// pattern must still have found its bytes +static void check(const char* name, const Case& c, const uint8_t* found, bool xored) { + if (found != c.code + c.at) { printf("FAIL %s: didn't find its bytes\n", name); exit(1); } + for (size_t a = 0, b; a < c.text.size(); a = b + 1) { + b = c.text.find('\n', a); + if (b == std::string::npos) b = c.text.size(); + if (contains(c.text.data() + a, b - a)) { printf("FAIL %s: pattern text in the binary\n", name); exit(1); } + } + // Folded plaintext may be split into 4-byte immediates (arm64 words always are), so each word on its own + for (size_t i = 0; xored && i + 4 <= c.bytes.size(); i += 4) + if (contains(&c.bytes[i], 4)) { printf("FAIL %s: plaintext bytes %zu..%zu in the binary\n", name, i, i + 3); exit(1); } + if (xored && !c.masks.empty() && contains(c.masks.data(), c.masks.size())) { printf("FAIL %s: plaintext masks in the binary\n", name); exit(1); } +} + +// One pattern per build (ONLY_CASE = its number; 0 runs all). Each scans in its own small function, as a local with +// find() inlined next to it: the shape in which optimizers fold constants, e.g. an XOR pattern's decode back +// into plaintext. With more call sites in the file, or anything else in that function, gcc stops inlining find() and +// such a leak stays hidden +#ifndef ONLY_CASE +#define ONLY_CASE 0 +#endif +#ifdef _MSC_VER +#define NOINLINE __declspec(noinline) +#else +#define NOINLINE __attribute__((noinline)) +#endif +struct Registered { void (*run)(); }; +static std::vector& cases() { + static std::vector v; + return v; +} +#define CASE(fn, name, make_case, def, reversed, xored) \ + NOINLINE static const uint8_t* fn##_find(const uint8_t* code, size_t size) { \ + auto p = def; /* not constexpr: the macros and literals must build at compile time on their own */ \ + return p.find(code, size); \ + } \ + static void fn() { \ + const Case c = make_case(reversed); \ + check(name, c, fn##_find(c.code, sizeof c.code), xored); \ + printf(" %s ok\n", name); \ + } \ + static const bool fn##_registered = (cases().push_back({ fn }), true); + +// Each pattern is unique, and its needle is its text reversed. The XOR arm64 ones use mask shapes no plain +// pattern here has: plain patterns' masks are in the binary by design. The XOR byte patterns have no wildcards, +// so each of their 4-byte words is distinctive +#if ONLY_CASE == 0 || ONLY_CASE == 1 +CASE(ct_macro, "COMPILETIME_PATTERN", byte_case, COMPILETIME_PATTERN("1A 2B ? 4D 5E 6F 71 82"), "28 17 F6 E5 D4 ? B2 A1", false) +#endif +#if ONLY_CASE == 0 || ONLY_CASE == 2 +CASE(xor_macro, "XOR_PATTERN", byte_case, XOR_PATTERN("A1 B2 C3 D4 E5 F6 07 18"), "81 70 6F 5E 4D 3C 2B 1A", true) +#endif +#if ONLY_CASE == 0 || ONLY_CASE == 3 +CASE(arm_macro, "ARM_PATTERN", arm_case, ARM_PATTERN("MOVZ W4, #0x1357\nMOVK W4, #0x2468, LSL #16\nLDR W?, [X4,#?]"), + "]?#,4X[ ,?W RDL\n61# LSL ,8642x0# ,4W KVOM\n7531x0# ,4W ZVOM", false) +#endif +#if ONLY_CASE == 0 || ONLY_CASE == 4 +CASE(armxor_macro, "ARM_XOR_PATTERN", arm_case, ARM_XOR_PATTERN("MOVZ W?, #0x7531\nMOVK W5, #0x8642, LSL #16\nSTR W?, [X5,#0x2C]"), + "]C2x0#,5X[ ,?W RTS\n61# LSL ,2468x0# ,5W KVOM\n1357x0# ,?W ZVOM", true) +#endif +#ifdef PATTERNS_LITERALS +#if ONLY_CASE == 0 || ONLY_CASE == 5 +CASE(ct_literal, "_ctpattern", byte_case, "3C 4D ? 6F 71 82 93 A4"_ctpattern, "4A 39 28 17 F6 ? D4 C3", false) +#endif +#if ONLY_CASE == 0 || ONLY_CASE == 6 +CASE(xor_literal, "_xorpattern", byte_case, "C3 D4 E5 F6 17 28 39 4A"_xorpattern, "A4 93 82 71 6F 5E 4D 3C", true) +#endif +#if ONLY_CASE == 0 || ONLY_CASE == 7 +CASE(arm_literal, "_armpattern", arm_case, "MOVZ W6, #0x1A2B\nMOVK W6, #0x3C4D, LSL #16\nLDR W?, [X6,#?]"_armpattern, + "]?#,6X[ ,?W RDL\n61# LSL ,D4C3x0# ,6W KVOM\nB2A1x0# ,6W ZVOM", false) +#endif +#if ONLY_CASE == 0 || ONLY_CASE == 8 +CASE(armxor_literal, "_armxorpattern", arm_case, "MOVZ W7, #0x2B1A\nMOVK W?, #0x4D3C, LSL #16\nSTR W?, [X7,#0x34]"_armxorpattern, + "]43x0#,7X[ ,?W RTS\n61# LSL ,C3D4x0# ,?W KVOM\nA1B2x0# ,7W ZVOM", true) +#endif +#endif + +int main(int, char** argv) { + FILE* f = fopen(argv[0], "rb"); + if (!f) { printf("FAIL: can't open %s\n", argv[0]); return 1; } + uint8_t chunk[65536]; + for (size_t n; (n = fread(chunk, 1, sizeof chunk, f)) > 0;) bin.insert(bin.end(), chunk, chunk + n); + fclose(f); + CHECK(!cases().empty()); + for (const auto& c : cases()) c.run(); +} diff --git a/tests/test_patterns.cpp b/tests/test_patterns.cpp index 1762efd..9cbd2bc 100644 --- a/tests/test_patterns.cpp +++ b/tests/test_patterns.cpp @@ -105,32 +105,6 @@ static void literals_and_macros() { // The XOR pattern's plaintext and source text must not be in our own executable. The needle is stored // scrambled and rebuilt at runtime, otherwise the test itself would put it in the binary. -static void no_plaintext_in_binary(const char* exe) { - static const uint8_t scrambled[] = { 0x62, 0xBB, 0xFF, 0x56, 0x38, 0x17, 0x43, 0xEA }; - volatile uint8_t key = 0xA5; // volatile, or the optimizer folds the descramble back into the plaintext - uint8_t plain[8]; - for (int i = 0; i < 8; ++i) plain[i] = scrambled[i] ^ key; // C7 1E 5A F3 9D B2 E6 4F - auto xp = XOR_PATTERN("C7 1E 5A F3 9D B2 E6 4F"); - CHECK(xp.find(plain, 8) == plain); - std::string text; - char hex[4]; - for (auto c : plain) { snprintf(hex, sizeof hex, "%02X ", c); text += hex; } - text.pop_back(); - - FILE* f = fopen(exe, "rb"); - if (!f) { printf(" binary check skipped (can't open %s)\n", exe); return; } - std::vector bin; - uint8_t chunk[65536]; - for (size_t n; (n = fread(chunk, 1, sizeof chunk, f)) > 0;) bin.insert(bin.end(), chunk, chunk + n); - fclose(f); - auto contains = [&](const void* needle, size_t len) { - return std::search(bin.begin(), bin.end(), (const uint8_t*)needle, (const uint8_t*)needle + len) != bin.end(); - }; - CHECK(!contains(plain, 8)); - CHECK(!contains(text.data(), text.size())); - puts(" no plaintext in binary ok"); -} - // Random patterns with random bit masks against the naive reference: both scan modes, // sizes around the SIMD block boundaries, odd base addresses static void fuzz(unsigned rounds) { @@ -215,7 +189,7 @@ static void arm_patterns() { puts(" arm64 patterns ok"); } -int main(int, char** argv) { +int main() { #ifdef PATTERNS_NO_SIMD puts("scan path: scalar (PATTERNS_NO_SIMD)"); #elif defined(PATTERNS_SSE2) @@ -227,7 +201,6 @@ int main(int, char** argv) { #endif edge_cases(); literals_and_macros(); - no_plaintext_in_binary(argv[0]); fuzz(300000); multi_strategies(); arm_patterns(); From 480c303d87094b7444dfcbd444f2f5f4dc62558c Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:06:07 -0700 Subject: [PATCH 13/19] test_artifacts: scan every pattern from one function, add LTO builds One build per optimization level instead of one per pattern: all patterns are scanned from a single non-static function, the shape of a real signature file and one in which gcc and clang both inline find() and would fold XOR plaintext back in. LTO variants added. Every leak is reported, not only the first. 14 builds per compiler instead of 60. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- tests/CMakeLists.txt | 38 +++++++------ tests/test_artifacts.cpp | 119 ++++++++++++++++++--------------------- 3 files changed, 77 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index a912e9d..7546c57 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ ctest --test-dir build -C Release --output-on-failure ./build/bench_multi [module file] # many patterns: find() per pattern vs find_all vs threaded ``` - `test_patterns`: edge cases, fuzzing against a naive scanner, per-definition XOR keys. Also built with `PATTERNS_NO_SIMD`. -- `test_artifacts_*`: every compile-time pattern type (macros, and C++20 literals) must leave no pattern text in its binary, and the XOR ones no plaintext bytes, words or masks. Built once per pattern at each optimization level (`-O0`..`-O3`, `-Os`; MSVC `/Od /O1 /O2`) with debug info, as C++17 and C++20: optimizers fold differently at each, and fold only when a pattern's `find()` is inlined, so each pattern gets its own small function in its own build. +- `test_artifacts_*`: every compile-time pattern type (macros, and C++20 literals) must leave no pattern text in its binary, and the XOR ones no plaintext bytes, words or masks. All patterns are scanned from one function, as in a project's signature file, and built at each optimization level (`-O0`..`-O3`, `-Os`; MSVC `/Od /O1 /O2`) plus LTO, with debug info, as C++17 and C++20: optimizers fold constants differently at each. - `test_cpp17`: the macro path. - `test_arm`: ArmPattern encodings checked against clang's assembler (concrete instructions must encode exactly, wildcard patterns must match / not match), line formats, RuntimeArmPattern parsing the same, and `/d` resolution. On Apple arm64 it also re-assembles its tables and resolves `/d` on linker-relocated ADRP/ADD/LDR/BL code. - `example_usage`: usage styles 3-5 above against a module that has to be decrypted first, logging each result as `[count] name address (rebased address)`. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c3a988b..348b9db 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -40,33 +40,37 @@ if(WIN32) add_test(NAME test_tls COMMAND test_tls) endif() -# No pattern text / XOR plaintext left in the binary: one build per pattern (see test_artifacts.cpp for why), at every -# optimization level, with debug info, C++17 (macros) and C++20 (macros and literals) +# No pattern text / XOR plaintext left in the binary, at every optimization level and with LTO (whole-program +# inlining folds differently), with debug info, C++17 (macros) and C++20 (macros and literals) +include(CheckIPOSupported) +check_ipo_supported(RESULT lto_supported LANGUAGES CXX) if(MSVC) set(artifact_opts Od O1 O2) + set(artifact_lto_opts O2) set(artifact_flag /) set(artifact_debug /Zi) else() set(artifact_opts O0 O1 O2 O3 Os) + set(artifact_lto_opts O2 O3) set(artifact_flag -) set(artifact_debug -g) endif() -set(artifact_cases ct_macro xor_macro arm_macro armxor_macro ct_literal xor_literal arm_literal armxor_literal) +if(lto_supported) + foreach(opt ${artifact_lto_opts}) + list(APPEND artifact_opts ${opt}_lto) + endforeach() +endif() foreach(std 17 20) foreach(opt ${artifact_opts}) - set(id 0) - foreach(case ${artifact_cases}) - math(EXPR id "${id} + 1") - if(std EQUAL 17 AND id GREATER 4) # literals need C++20 - break() - endif() - set(t test_artifacts_cpp${std}_${opt}_${case}) - add_executable(${t} test_artifacts.cpp) - set_target_properties(${t} PROPERTIES CXX_STANDARD ${std}) - target_compile_definitions(${t} PRIVATE ONLY_CASE=${id}) - # After the build type's flags, so this level wins - target_compile_options(${t} PRIVATE ${artifact_flag}${opt} ${artifact_debug}) - add_test(NAME ${t} COMMAND ${t}) - endforeach() + set(t test_artifacts_cpp${std}_${opt}) + string(REPLACE "_lto" "" level ${opt}) + add_executable(${t} test_artifacts.cpp) + set_target_properties(${t} PROPERTIES CXX_STANDARD ${std}) + if(NOT level STREQUAL opt) + set_target_properties(${t} PROPERTIES INTERPROCEDURAL_OPTIMIZATION ON) + endif() + # After the build type's flags, so this level wins + target_compile_options(${t} PRIVATE ${artifact_flag}${level} ${artifact_debug}) + add_test(NAME ${t} COMMAND ${t}) endforeach() endforeach() diff --git a/tests/test_artifacts.cpp b/tests/test_artifacts.cpp index 54ceb73..d3daaa2 100644 --- a/tests/test_artifacts.cpp +++ b/tests/test_artifacts.cpp @@ -1,6 +1,6 @@ // Compile-time patterns must leave nothing behind in the binary: no pattern text for any of them, and no plaintext -// bytes, words or masks for the XOR ones. tests/CMakeLists.txt builds this once per pattern (ONLY_CASE) at every -// optimization level, with debug info, as C++17 (macros) and C++20 (macros and literals). Exit code 0 = pass. +// bytes, words or masks for the XOR ones. tests/CMakeLists.txt builds this at every optimization level and with LTO, +// with debug info, as C++17 (macros) and C++20 (macros and literals). Exit code 0 = pass. // Needles are written reversed and turned around at runtime, so this file doesn't put them in the binary itself. #include #include @@ -74,81 +74,59 @@ static Case arm_case(const char* reversed) { // No text in the binary (each line on its own for arm64, so partly folded text is caught too); for XOR patterns no // plaintext bytes or masks either (masks only as a whole: single mask words like FFFFFFFF are everywhere). The // pattern must still have found its bytes -static void check(const char* name, const Case& c, const uint8_t* found, bool xored) { - if (found != c.code + c.at) { printf("FAIL %s: didn't find its bytes\n", name); exit(1); } +static bool check(const char* name, const Case& c, const uint8_t* found, bool xored) { + if (found != c.code + c.at) { printf("FAIL %s: didn't find its bytes\n", name); return false; } for (size_t a = 0, b; a < c.text.size(); a = b + 1) { b = c.text.find('\n', a); if (b == std::string::npos) b = c.text.size(); - if (contains(c.text.data() + a, b - a)) { printf("FAIL %s: pattern text in the binary\n", name); exit(1); } + if (contains(c.text.data() + a, b - a)) { printf("FAIL %s: pattern text in the binary\n", name); return false; } } // Folded plaintext may be split into 4-byte immediates (arm64 words always are), so each word on its own for (size_t i = 0; xored && i + 4 <= c.bytes.size(); i += 4) - if (contains(&c.bytes[i], 4)) { printf("FAIL %s: plaintext bytes %zu..%zu in the binary\n", name, i, i + 3); exit(1); } - if (xored && !c.masks.empty() && contains(c.masks.data(), c.masks.size())) { printf("FAIL %s: plaintext masks in the binary\n", name); exit(1); } + if (contains(&c.bytes[i], 4)) { printf("FAIL %s: plaintext bytes %zu..%zu in the binary\n", name, i, i + 3); return false; } + if (xored && !c.masks.empty() && contains(c.masks.data(), c.masks.size())) { printf("FAIL %s: plaintext masks in the binary\n", name); return false; } + return true; } -// One pattern per build (ONLY_CASE = its number; 0 runs all). Each scans in its own small function, as a local with -// find() inlined next to it: the shape in which optimizers fold constants, e.g. an XOR pattern's decode back -// into plaintext. With more call sites in the file, or anything else in that function, gcc stops inlining find() and -// such a leak stays hidden -#ifndef ONLY_CASE -#define ONLY_CASE 0 +// Each pattern is unique, and its needle is its text reversed. The XOR arm64 ones use mask shapes no plain +// pattern here has: plain patterns' masks are in the binary by design. The XOR byte patterns have no wildcards, +// so each of their 4-byte words is distinctive +#define MACRO_CASES(X) \ + X("COMPILETIME_PATTERN", byte_case, COMPILETIME_PATTERN("1A 2B ? 4D 5E 6F 71 82"), "28 17 F6 E5 D4 ? B2 A1", false) \ + X("XOR_PATTERN", byte_case, XOR_PATTERN("A1 B2 C3 D4 E5 F6 07 18"), "81 70 6F 5E 4D 3C 2B 1A", true) \ + X("ARM_PATTERN", arm_case, ARM_PATTERN("MOVZ W4, #0x1357\nMOVK W4, #0x2468, LSL #16\nLDR W?, [X4,#?]"), \ + "]?#,4X[ ,?W RDL\n61# LSL ,8642x0# ,4W KVOM\n7531x0# ,4W ZVOM", false) \ + X("ARM_XOR_PATTERN", arm_case, ARM_XOR_PATTERN("MOVZ W?, #0x7531\nMOVK W5, #0x8642, LSL #16\nSTR W?, [X5,#0x2C]"), \ + "]C2x0#,5X[ ,?W RTS\n61# LSL ,2468x0# ,5W KVOM\n1357x0# ,?W ZVOM", true) +#ifdef PATTERNS_LITERALS +#define LITERAL_CASES(X) \ + X("_ctpattern", byte_case, "3C 4D ? 6F 71 82 93 A4"_ctpattern, "4A 39 28 17 F6 ? D4 C3", false) \ + X("_xorpattern", byte_case, "C3 D4 E5 F6 17 28 39 4A"_xorpattern, "A4 93 82 71 6F 5E 4D 3C", true) \ + X("_armpattern", arm_case, "MOVZ W6, #0x1A2B\nMOVK W6, #0x3C4D, LSL #16\nLDR W?, [X6,#?]"_armpattern, \ + "]?#,6X[ ,?W RDL\n61# LSL ,D4C3x0# ,6W KVOM\nB2A1x0# ,6W ZVOM", false) \ + X("_armxorpattern", arm_case, "MOVZ W7, #0x2B1A\nMOVK W?, #0x4D3C, LSL #16\nSTR W?, [X7,#0x34]"_armxorpattern, \ + "]43x0#,7X[ ,?W RTS\n61# LSL ,C3D4x0# ,?W KVOM\nA1B2x0# ,7W ZVOM", true) +#else +#define LITERAL_CASES(X) #endif + #ifdef _MSC_VER #define NOINLINE __declspec(noinline) #else #define NOINLINE __attribute__((noinline)) #endif -struct Registered { void (*run)(); }; -static std::vector& cases() { - static std::vector v; - return v; -} -#define CASE(fn, name, make_case, def, reversed, xored) \ - NOINLINE static const uint8_t* fn##_find(const uint8_t* code, size_t size) { \ - auto p = def; /* not constexpr: the macros and literals must build at compile time on their own */ \ - return p.find(code, size); \ - } \ - static void fn() { \ - const Case c = make_case(reversed); \ - check(name, c, fn##_find(c.code, sizeof c.code), xored); \ - printf(" %s ok\n", name); \ - } \ - static const bool fn##_registered = (cases().push_back({ fn }), true); -// Each pattern is unique, and its needle is its text reversed. The XOR arm64 ones use mask shapes no plain -// pattern here has: plain patterns' masks are in the binary by design. The XOR byte patterns have no wildcards, -// so each of their 4-byte words is distinctive -#if ONLY_CASE == 0 || ONLY_CASE == 1 -CASE(ct_macro, "COMPILETIME_PATTERN", byte_case, COMPILETIME_PATTERN("1A 2B ? 4D 5E 6F 71 82"), "28 17 F6 E5 D4 ? B2 A1", false) -#endif -#if ONLY_CASE == 0 || ONLY_CASE == 2 -CASE(xor_macro, "XOR_PATTERN", byte_case, XOR_PATTERN("A1 B2 C3 D4 E5 F6 07 18"), "81 70 6F 5E 4D 3C 2B 1A", true) -#endif -#if ONLY_CASE == 0 || ONLY_CASE == 3 -CASE(arm_macro, "ARM_PATTERN", arm_case, ARM_PATTERN("MOVZ W4, #0x1357\nMOVK W4, #0x2468, LSL #16\nLDR W?, [X4,#?]"), - "]?#,4X[ ,?W RDL\n61# LSL ,8642x0# ,4W KVOM\n7531x0# ,4W ZVOM", false) -#endif -#if ONLY_CASE == 0 || ONLY_CASE == 4 -CASE(armxor_macro, "ARM_XOR_PATTERN", arm_case, ARM_XOR_PATTERN("MOVZ W?, #0x7531\nMOVK W5, #0x8642, LSL #16\nSTR W?, [X5,#0x2C]"), - "]C2x0#,5X[ ,?W RTS\n61# LSL ,2468x0# ,5W KVOM\n1357x0# ,?W ZVOM", true) -#endif -#ifdef PATTERNS_LITERALS -#if ONLY_CASE == 0 || ONLY_CASE == 5 -CASE(ct_literal, "_ctpattern", byte_case, "3C 4D ? 6F 71 82 93 A4"_ctpattern, "4A 39 28 17 F6 ? D4 C3", false) -#endif -#if ONLY_CASE == 0 || ONLY_CASE == 6 -CASE(xor_literal, "_xorpattern", byte_case, "C3 D4 E5 F6 17 28 39 4A"_xorpattern, "A4 93 82 71 6F 5E 4D 3C", true) -#endif -#if ONLY_CASE == 0 || ONLY_CASE == 7 -CASE(arm_literal, "_armpattern", arm_case, "MOVZ W6, #0x1A2B\nMOVK W6, #0x3C4D, LSL #16\nLDR W?, [X6,#?]"_armpattern, - "]?#,6X[ ,?W RDL\n61# LSL ,D4C3x0# ,6W KVOM\nB2A1x0# ,6W ZVOM", false) -#endif -#if ONLY_CASE == 0 || ONLY_CASE == 8 -CASE(armxor_literal, "_armxorpattern", arm_case, "MOVZ W7, #0x2B1A\nMOVK W?, #0x4D3C, LSL #16\nSTR W?, [X7,#0x34]"_armxorpattern, - "]43x0#,7X[ ,?W RTS\n61# LSL ,C3D4x0# ,?W KVOM\nA1B2x0# ,7W ZVOM", true) -#endif -#endif +// All scans in one function with nothing else in it, as a project's signature file does it: the shape in which +// optimizers inline find() and fold constants, e.g. an XOR pattern's decode back into plaintext. Locals, not +// constexpr: the macros and literals must build at compile time on their own. Not static: gcc treats a static +// function only reachable from main() as run once and stops inlining into it, which would hide such a leak +NOINLINE void scan_all(const uint8_t* const* code, size_t size, const uint8_t** found) { + size_t i = 0; +#define SCAN(name, make_case, def, reversed, xored) { auto p = def; found[i] = p.find(code[i], size); ++i; } + MACRO_CASES(SCAN) + LITERAL_CASES(SCAN) +#undef SCAN +} int main(int, char** argv) { FILE* f = fopen(argv[0], "rb"); @@ -156,6 +134,19 @@ int main(int, char** argv) { uint8_t chunk[65536]; for (size_t n; (n = fread(chunk, 1, sizeof chunk, f)) > 0;) bin.insert(bin.end(), chunk, chunk + n); fclose(f); - CHECK(!cases().empty()); - for (const auto& c : cases()) c.run(); + + struct Entry { const char* name; Case c; bool xored; }; + std::vector cases; +#define ADD(name, make_case, def, reversed, xored) cases.push_back({ name, make_case(reversed), xored }); + MACRO_CASES(ADD) + LITERAL_CASES(ADD) +#undef ADD + std::vector code, found(cases.size()); + for (const auto& e : cases) code.push_back(e.c.code); + scan_all(code.data(), sizeof cases[0].c.code, found.data()); + // Every leak reported, not just the first + bool ok = true; + for (size_t i = 0; i < cases.size(); ++i) ok &= check(cases[i].name, cases[i].c, found[i], cases[i].xored); + if (!ok) return 1; + printf("%zu patterns: no pattern text, no XOR plaintext\n", cases.size()); } From 92b19baf967f2b3381d80a46d3114af1807bd228 Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:18:12 -0700 Subject: [PATCH 14/19] Add CI: library tests and IDA plugin build on Windows and macOS Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..d28a3d3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,39 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + workflow_dispatch: + +jobs: + # Library tests (tests/), including the compile-time leftovers matrix + tests: + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - run: cmake -S tests -B build -DCMAKE_BUILD_TYPE=Release + - run: cmake --build build --config Release --parallel + - run: ctest --test-dir build -C Release --output-on-failure + + # IDA plugin build against the public SDK. The idalib search test needs an IDA install and license, so it isn't run + ida-plugin: + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + with: + repository: HexRaysSA/ida-sdk + ref: v9.3.0-release + path: ida-sdk + submodules: true + - run: cmake -S plugins/ida -B build-ida -DCMAKE_BUILD_TYPE=Release -DIDASDK="${{ github.workspace }}/ida-sdk" + - run: cmake --build build-ida --config Release --parallel From 3b2a1b3562e097171444119dff54c83f3376b582 Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:20:22 -0700 Subject: [PATCH 15/19] CI: library tests only Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d28a3d3..b9e4c9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,20 +20,3 @@ jobs: - run: cmake --build build --config Release --parallel - run: ctest --test-dir build -C Release --output-on-failure - # IDA plugin build against the public SDK. The idalib search test needs an IDA install and license, so it isn't run - ida-plugin: - strategy: - fail-fast: false - matrix: - os: [windows-latest, macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: actions/checkout@v4 - with: - repository: HexRaysSA/ida-sdk - ref: v9.3.0-release - path: ida-sdk - submodules: true - - run: cmake -S plugins/ida -B build-ida -DCMAKE_BUILD_TYPE=Release -DIDASDK="${{ github.workspace }}/ida-sdk" - - run: cmake --build build-ida --config Release --parallel From 37710660cfc9385b6af389a5aa83a777e5822a13 Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 10:51:42 -0700 Subject: [PATCH 16/19] Encode arm64 STP and MOV; warn about unsupported lines in the IDA plugin - STP (pre-index, signed offset, post-index): LDP with the load bit clear. - MOV encoded as assemblers pick it: ADD #0 with SP, ORR from the zero register for registers, MOVZ/MOVN/ORR (bitmask) for immediates; MOV Rd, #? matches MOVZ or MOVN. Encodings checked against clang's assembler. - Unsupported instructions still become wildcards, but the IDA plugin now lists them and asks before searching. - Regression test: an IDA-printed prologue (STP/MOV) must skip a near-miss SUB SP prologue, in test_arm and in the idalib search test. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- include/PatternScanner/ArmPattern.hpp | 64 +++++++++++++++++++++++++++ plugins/ida/pattern_scanner.cpp | 6 +++ plugins/ida/search.hpp | 13 ++++++ plugins/ida/test/arm64.S | 15 +++++++ plugins/ida/test/test_search.cpp | 13 ++++++ tests/test_arm.cpp | 45 +++++++++++++++++++ 7 files changed, 157 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7546c57..d38a0fa 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ CompileTimePattern creates the pattern & mask during compile time and stored in XORPattern creates the pattern & mask along with two keys arrays during compile time. Pattern and mask are both stored XOR'd (with separate keys), otherwise it acts just the same as the CompileTimePattern. (the pattern() and mask() methods however return the XOR'd bytes; access the original pattern bytes with the [] operator and the original mask bytes with unmask()) -ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Instructions not implemented would result in a wildcard portion for those 4 bytes. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). One instruction per line; case, indentation and spacing don't matter, so lines pasted from IDA (`LDR W8, [X8,#0x18]`) or lldb (`ldr w8, [x8, #0x18]`) both work. lldb prints ADRP operands as page counts and branch targets as absolute addresses, so wildcard those (`#?`). Aliases such as `MOV` aren't implemented (they become wildcards); write the underlying `MOVZ`/`ORR` instead. +ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Supported: `B`, `BL`, `BR`, `BLR`, `RET`, `CBZ`, `CBNZ`, `ADR`, `ADRP`, `ADD(S)`, `SUB(S)`, `LDR`, `LDRH`, `STR`, `LDP`, `STP`, `MOV`, `MOVZ`, `MOVN`, `MOVK`, `ORR`, `NOP`. Any other instruction becomes a wildcard for those 4 bytes, matching any instruction (the IDA plugin asks before searching with one); write `?` for a deliberate one. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). One instruction per line; case, indentation and spacing don't matter, so lines pasted from IDA (`LDR W8, [X8,#0x18]`) or lldb (`ldr w8, [x8, #0x18]`) both work. lldb prints ADRP operands as page counts and branch targets as absolute addresses, so wildcard those (`#?`). `MOV` is encoded the way assemblers pick it (register, SP, or immediate via `MOVZ`/`MOVN`/`ORR`); `MOV Rd, #?` matches `MOVZ` or `MOVN`. Other aliases aren't implemented. ArmXORPattern (`PatternScanner/ArmXORPattern.hpp`) is ArmPattern with the instruction words and masks stored XOR'd, the way XORPattern stores them. Same syntax, flags and results, plus `[]` and unmask() for the originals. Decoded on the stack only while scanning. diff --git a/include/PatternScanner/ArmPattern.hpp b/include/PatternScanner/ArmPattern.hpp index 5e3f604..c44b526 100644 --- a/include/PatternScanner/ArmPattern.hpp +++ b/include/PatternScanner/ArmPattern.hpp @@ -520,6 +520,67 @@ namespace patterns { return base; } + // STP is LDP with the load bit (22) clear + constexpr ArmOpcode parse_stp(const char* instr, ArmOpcode base) { + base = parse_ldp(instr, base); + base.pattern &= ~(1u << 22); + return base; + } + + // MOV is an alias; encoded the way assemblers pick: ADD #0 when SP is involved, ORR from the zero register for + // registers, and MOVZ, then MOVN, then ORR (bitmask) for immediates. MOV Rd, #? matches MOVZ or MOVN + constexpr ArmOpcode parse_mov(const char* instr, ArmOpcode) { + size_t i = skip_whitespace(instr, skip_mnemonic(instr)); + char rd[5] = {}; + i = skip_whitespace(instr, skip_char(instr, parse_token(instr, i, rd), ',')); + const bool rd_sp = str_eq(rd, "SP") || str_eq(rd, "WSP"); + const bool known_width = rd[0] == 'W' || rd[0] == 'X' || rd_sp; + const bool is64 = rd[0] == 'X' || str_eq(rd, "SP"); + ArmOpcode op{ 0, 0 }; + if (known_width) op.set_bits(is64 ? 1 : 0, 1, 31); + if (auto r = parse_register(rd); r != 0xFFFFFFFF) op.set_bits(r, 0x1F); + + if (instr[i] == '#') { + char imm[24] = {}; + parse_imm_token(instr, i, imm); + if (imm[0] == '?') { + op.set_bits(0x12800000, 0x1F800000); // MOVZ or MOVN: opc (bits 29-30) left open + return op; + } + const unsigned width = is64 ? 64 : 32; + const uint64_t ones = is64 ? ~0ULL : 0xFFFFFFFFULL; + const uint64_t v = stoi(imm) & ones; // unsigned: 64-bit hex fits, and negatives wrap + for (uint32_t hw = 0; hw < width / 16; ++hw) { + if ((v & ~(0xFFFFULL << (hw * 16))) == 0) { + op.set_bits(0x52800000 | hw << 21 | static_cast(v >> (hw * 16)) << 5, 0x7FFFFFE0); + return op; + } + } + for (uint32_t hw = 0; hw < width / 16; ++hw) { + if ((~v & ones & ~(0xFFFFULL << (hw * 16))) == 0) { + op.set_bits(0x12800000 | hw << 21 | static_cast((~v & ones) >> (hw * 16)) << 5, 0x7FFFFFE0); + return op; + } + } + auto [ ok, N, imms, immr ] = encode_bitmask_immediate(v, is64); + if (!ok) return { 0, 0 }; + op.set_bits(0x320003E0 | N << 22 | immr << 16 | imms << 10, 0x7FFFFFE0); + return op; + } + + char rm[5] = {}; + parse_token(instr, i, rm); + const auto rm_val = parse_register(rm); + if (rd_sp || str_eq(rm, "SP") || str_eq(rm, "WSP")) { + op.set_bits(0x11000000, 0x7FFFFC00); // ADD Rd, Rn, #0 + if (rm_val != 0xFFFFFFFF) op.set_bits(rm_val, 0x1F, 5); + } else { + op.set_bits(0x2A0003E0, 0x7FE0FFE0); // ORR Rd, ZR, Rm + if (rm_val != 0xFFFFFFFF) op.set_bits(rm_val, 0x1F, 16); + } + return op; + } + constexpr ArmOpcode parse_str(const char* instr, ArmOpcode base) { size_t i = skip_whitespace(instr, skip_mnemonic(instr)); @@ -569,6 +630,8 @@ namespace patterns { { "MOVK", parse_movX }, { "ORR", parse_orr }, { "LDP", parse_ldp }, + { "STP", parse_stp }, + { "MOV", parse_mov }, { "ADD", parse_arithmetic }, { "ADDS", parse_arithmetic }, { "SUB", parse_arithmetic }, @@ -597,6 +660,7 @@ namespace patterns { { "MOVK", { 0x72800000, 0x7F800000 } }, { "ORR", { 0x22000000, 0x67000000 } }, { "LDP", { 0x28400000, 0x7E400000 } }, + { "STP", { 0x28000000, 0x7E400000 } }, { "BR", { 0xD61F0000, 0xFFFFFC1F } }, { "BLR", { 0xD63F0000, 0xFFFFFC1F } }, { "STR", { 0xB8000000, 0xBEC00000 } }, diff --git a/plugins/ida/pattern_scanner.cpp b/plugins/ida/pattern_scanner.cpp index 309b6a0..061d546 100644 --- a/plugins/ida/pattern_scanner.cpp +++ b/plugins/ida/pattern_scanner.cpp @@ -101,6 +101,12 @@ namespace { ti.text = text; if (ask_form(form, &ti, &kind, &opts) != ASKBTN_YES) return; text = ti.text; + if (kind == pscan::arm64_pattern) { + const qstring unknown = pscan::unsupported_lines(text); + if (!unknown.empty() && ask_yn(ASKBTN_NO, "HIDECANCEL\nPatternScanner doesn't support these instructions, " + "so they would match any instruction:\n\n%s\nSearch anyway?", unknown.c_str()) != ASKBTN_YES) + return; + } try { last = pscan::parse(text, static_cast(kind)); } diff --git a/plugins/ida/search.hpp b/plugins/ida/search.hpp index 2a09568..14da5a6 100644 --- a/plugins/ida/search.hpp +++ b/plugins/ida/search.hpp @@ -38,6 +38,19 @@ namespace pscan { return patterns::RuntimePattern(text.c_str(), text.length(), db_arch()); } + // arm64 lines the encoder doesn't know, one per line: they become wildcards matching any instruction. A line that + // is just "?" is a deliberate wildcard and isn't listed + inline qstring unsupported_lines(const qstring& text) { + qstring out; + patterns::detail::parse_arm(text.c_str(), text.length(), + [&](size_t, const char* instruction) { + if (!(instruction[0] == '?' && !instruction[1]) && !patterns::detail::a64_encode_instruction(instruction).mask) + out.cat_sprnt("%s\n", instruction); + }, + [](size_t) {}, [](const char*) {}); + return out; + } + struct hit_t { ea_t match = BADADDR; // where the pattern matched ea_t result = BADADDR; // what find() gives: the X offset and /d /r applied diff --git a/plugins/ida/test/arm64.S b/plugins/ida/test/arm64.S index 1f4618a..2eb3768 100644 --- a/plugins/ida/test/arm64.S +++ b/plugins/ida/test/arm64.S @@ -11,6 +11,21 @@ code_bl: bl target_func target_func: ret +// Two prologues: the first is a near miss for the STP/MOV pattern in test_search.cpp, the second matches it +near_miss: + sub sp, sp, #0x20 + stp x29, x30, [sp, #0x10] + add x29, sp, #0x10 + mov x19, x1 + mov x0, x30 + ret +wanted: + stp x20, x19, [sp, #-0x20]! + stp x29, x30, [sp, #0x10] + add x29, sp, #0x10 + mov x19, x1 + mov x0, x30 + ret .data .balign 16 diff --git a/plugins/ida/test/test_search.cpp b/plugins/ida/test/test_search.cpp index 3053870..1d76fee 100644 --- a/plugins/ida/test/test_search.cpp +++ b/plugins/ida/test/test_search.cpp @@ -94,6 +94,19 @@ static void arm64(const char* path) { CHECK(bytes.match == sym("code_adrp") && bytes.result == data); CHECK(all("NOP\nRET", pscan::arm64_pattern).empty()); + // A prologue as IDA prints it (STP, MOV): the first hit must be the function that has it, not a near miss + const char* prologue = "STP X20, X19, [SP,#?]!\n" + "STP X29, X30, [SP,#?]\n" + "ADD X29, SP, #0x10\n" + "MOV X19, X1\n" + "MOV X0, X30"; + CHECK(pscan::unsupported_lines(prologue).empty()); + const auto fn = all(prologue, pscan::arm64_pattern); + CHECK(fn.size() == 1 && fn[0].match == sym("wanted")); + + // Unknown instructions are reported (they'd match anything); a deliberate "?" line isn't + CHECK(pscan::unsupported_lines("NOP\nFOO X1, X2\n ? \nstp x29, x30, [sp,#?]\nBAR\n/d") == "FOO X1, X2\nBAR\n"); + bool threw = false; try { pscan::parse("48 8G", pscan::bytes_pattern); } catch (const std::logic_error&) { threw = true; } CHECK(threw); diff --git a/tests/test_arm.cpp b/tests/test_arm.cpp index 16a16c0..492c72e 100644 --- a/tests/test_arm.cpp +++ b/tests/test_arm.cpp @@ -11,6 +11,22 @@ // Fully specified instruction -> its encoding. The pattern must be exactly the encoding, every bit masked #define CONCRETE(X) \ X("NOP", 0xD503201F) \ + X("STP X20, X19, [SP,#-0x20]!", 0xA9BE4FF4) \ + X("STP X29, X30, [SP,#0x10]", 0xA9017BFD) \ + X("STP X29, X30, [SP],#0x10", 0xA8817BFD) \ + X("STP W1, W2, [X0,#8]", 0x29010801) \ + X("LDP X29, X30, [SP,#0x10]", 0xA9417BFD) \ + X("MOV X19, X1", 0xAA0103F3) \ + X("MOV X0, X30", 0xAA1E03E0) \ + X("MOV W0, W1", 0x2A0103E0) \ + X("MOV X29, SP", 0x910003FD) \ + X("MOV SP, X29", 0x910003BF) \ + X("MOV W0, #0x10", 0x52800200) \ + X("MOV X0, #0x10000", 0xD2A00020) \ + X("MOV W0, #-1", 0x12800000) \ + X("MOV X0, #-2", 0x92800020) \ + X("MOV W0, #0x10001", 0x320083E0) \ + X("MOV X0, #0xFFFF0000FFFF0000", 0xB2103FE0) \ X("RET", 0xD65F03C0) \ X("BR X16", 0xD61F0200) \ X("BLR X8", 0xD63F0100) \ @@ -83,6 +99,16 @@ // Wildcard pattern, an instruction, its encoding, and whether the pattern must match it #define WILDCARD(X) \ + X("STP X29, X30, [SP,#?]", "STP X29, X30, [SP,#0x30]", 0xA9037BFD, 1) \ + X("STP X29, X30, [SP,#?]", "LDP X29, X30, [SP,#0x10]", 0xA9417BFD, 0) \ + X("STP X29, X30, [SP,#?]", "STP X29, X30, [SP],#0x10", 0xA8817BFD, 0) \ + X("STP X20, X19, [SP,#?]!", "STP X20, X19, [SP,#-0x20]!", 0xA9BE4FF4, 1) \ + X("STP X20, X19, [SP,#?]!", "SUB SP, SP, #0x20", 0xD10083FF, 0) \ + X("MOV X?, X1", "MOV X5, X1", 0xAA0103E5, 1) \ + X("MOV X19, X?", "MOV X19, X1", 0xAA0103F3, 1) \ + X("MOV W0, #?", "MOV W0, #5", 0x528000A0, 1) \ + X("MOV W0, #?", "MOV W0, #-1", 0x12800000, 1) \ + X("MOV W0, #?", "MOV W0, W1", 0x2A0103E0, 0) \ X("LDR W?, [X?,#0x18]", "LDR W3, [X9,#0x18]", 0xB9401923, 1) \ X("LDR W?, [X?,#0x18]", "LDR W3, [X9,#0x1C]", 0xB9401D23, 0) \ X("LDR W?, [X?,#0x18]", "LDR X3, [X9,#0x18]", 0xF9400D23, 0) \ @@ -202,6 +228,24 @@ static void line_formats() { puts(" line formats ok"); } +// A function prologue as IDA prints it (STP, MOV): must skip a similar prologue, not treat STP/MOV as wildcards +static void prologue() { + alignas(4) static const uint32_t code[] = { + 0xD10083FF, 0xA9017BFD, 0x910043FD, 0xAA0103F3, 0xAA1E03E0, // SUB SP, SP, #0x20; STP; ADD X29; MOV; MOV + 0xA9BE4FF4, 0xA9017BFD, 0x910043FD, 0xAA0103F3, 0xAA1E03E0, // STP X20, X19, [SP,#-0x20]!; ... the one wanted + }; + const auto bytes = reinterpret_cast(code); +#define PROLOGUE "STP X20, X19, [SP,#?]!\n" \ + "STP X29, X30, [SP,#?]\n" \ + "ADD X29, SP, #0x10\n" \ + "MOV X19, X1\n" \ + "MOV X0, X30" + CHECK(ARM_PATTERN(PROLOGUE).find(bytes, sizeof code) == code + 5); + CHECK(patterns::RuntimeArmPattern(PROLOGUE).find(bytes, sizeof code) == code + 5); +#undef PROLOGUE + puts(" prologue ok"); +} + // RuntimeArmPattern: the same parse as ArmPattern, from text only known at runtime static void runtime_patterns() { int n = 0; @@ -335,6 +379,7 @@ int main() { concrete(); wildcards(); line_formats(); + prologue(); runtime_patterns(); xor_patterns(); deref_synthetic(); From b6136478695e0ee34a4c2bfe686205da9a2a8f80 Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:15:54 -0700 Subject: [PATCH 17/19] Encode arm64 B.cond, CMP/CMN/TST, AND/ANDS/EOR, byte/half/unscaled loads and stores, PAC One parser for single-register loads/stores (LDR/STR/LDRB/STRB/LDRH/STRH/LDRSW and the LDUR/STUR forms). LDR/STR offsets the scaled form can't hold are encoded as LDUR/STUR, as assemblers do, instead of a wrong immediate. FP/SIMD loads/stores and out-of-range offsets become wildcards rather than never-matching patterns. Fixes ADD/SUB register-form wildcards never matching an odd Rm. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- include/PatternScanner/ArmPattern.hpp | 197 ++++++++++++++++---------- tests/test_arm.cpp | 113 ++++++++++++++- 3 files changed, 232 insertions(+), 80 deletions(-) diff --git a/README.md b/README.md index d38a0fa..a02646c 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ CompileTimePattern creates the pattern & mask during compile time and stored in XORPattern creates the pattern & mask along with two keys arrays during compile time. Pattern and mask are both stored XOR'd (with separate keys), otherwise it acts just the same as the CompileTimePattern. (the pattern() and mask() methods however return the XOR'd bytes; access the original pattern bytes with the [] operator and the original mask bytes with unmask()) -ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Supported: `B`, `BL`, `BR`, `BLR`, `RET`, `CBZ`, `CBNZ`, `ADR`, `ADRP`, `ADD(S)`, `SUB(S)`, `LDR`, `LDRH`, `STR`, `LDP`, `STP`, `MOV`, `MOVZ`, `MOVN`, `MOVK`, `ORR`, `NOP`. Any other instruction becomes a wildcard for those 4 bytes, matching any instruction (the IDA plugin asks before searching with one); write `?` for a deliberate one. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). One instruction per line; case, indentation and spacing don't matter, so lines pasted from IDA (`LDR W8, [X8,#0x18]`) or lldb (`ldr w8, [x8, #0x18]`) both work. lldb prints ADRP operands as page counts and branch targets as absolute addresses, so wildcard those (`#?`). `MOV` is encoded the way assemblers pick it (register, SP, or immediate via `MOVZ`/`MOVN`/`ORR`); `MOV Rd, #?` matches `MOVZ` or `MOVN`. Other aliases aren't implemented. +ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Supported: `B`, `BL`, `B.cond` (`B.?` for any condition), `BR`, `BLR`, `RET`, `RETAA`, `RETAB`, `CBZ`, `CBNZ`, `ADR`, `ADRP`, `ADD(S)`, `SUB(S)`, `CMP`, `CMN`, `AND(S)`, `ORR`, `EOR`, `TST`, `LDR`, `LDRB`, `LDRH`, `LDRSW`, `STR`, `STRB`, `STRH`, `LDUR`/`STUR` (and their `B`/`H`/`SW` forms), `LDP`, `STP`, `MOV`, `MOVZ`, `MOVN`, `MOVK`, `NOP`, `PACIASP`, `PACIBSP`, `AUTIASP`, `AUTIBSP`. Loads and stores take W/X registers only; floating-point/SIMD ones (`LDR Q0, ...`, `STP D9, D8, ...`) are wildcards. Any other instruction becomes a wildcard for those 4 bytes, matching any instruction (the IDA plugin asks before searching with one); write `?` for a deliberate one. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). One instruction per line; case, indentation and spacing don't matter, so lines pasted from IDA (`LDR W8, [X8,#0x18]`) or lldb (`ldr w8, [x8, #0x18]`) both work. lldb prints ADRP operands as page counts and branch targets as absolute addresses, so wildcard those (`#?`). `MOV` is encoded the way assemblers pick it (register, SP, or immediate via `MOVZ`/`MOVN`/`ORR`); `MOV Rd, #?` matches `MOVZ` or `MOVN`. `CMP`/`CMN`/`TST` are encoded as `SUBS`/`ADDS`/`ANDS` to the zero register, and an `LDR`/`STR` offset the scaled form can't hold (negative, unaligned) as `LDUR`/`STUR`, as assemblers do. Other aliases aren't implemented. ArmXORPattern (`PatternScanner/ArmXORPattern.hpp`) is ArmPattern with the instruction words and masks stored XOR'd, the way XORPattern stores them. Same syntax, flags and results, plus `[]` and unmask() for the originals. Decoded on the stack only while scanning. diff --git a/include/PatternScanner/ArmPattern.hpp b/include/PatternScanner/ArmPattern.hpp index c44b526..e6c43b0 100644 --- a/include/PatternScanner/ArmPattern.hpp +++ b/include/PatternScanner/ArmPattern.hpp @@ -54,6 +54,8 @@ namespace patterns { return i; } + constexpr size_t max_line = 128; + constexpr bool starts_with(const char* a, const char* prefix) { while (*prefix) { if (*a != *prefix) return false; @@ -246,17 +248,37 @@ namespace patterns { return base; } - constexpr std::tuple parse_load_store_defaults(const char* instr, size_t i, ArmOpcode& base, uint32_t size_bit = 0) { + // imm19 (bits 5-23) from a byte offset, for CBZ/CBNZ, B.cond and LDR (literal) + constexpr void parse_imm19(const char* instr, size_t i, ArmOpcode& base) { + char imm[24] = {}; + parse_imm_token(instr, i, imm); + if (imm[0] != '?') base.set_bits(static_cast(stoi(imm) >> 2), 0x7FFFF, 5); + } + + // Single-register loads/stores. op is the unsigned offset form; the unscaled (LDUR), pre/post-index and + // register offset forms differ from it in fixed bits. scale is log2 of the access size, or -1 to take it + // from Rt (W or X, bit 30). An offset the unsigned form can't hold (negative, unaligned) is encoded unscaled, + // the way assemblers pick LDUR/STUR. Floating-point/SIMD registers aren't supported: the line is a wildcard + template + constexpr ArmOpcode parse_load_store(const char* instr, ArmOpcode) { + ArmOpcode base{ 0, 0 }; + size_t i = skip_whitespace(instr, skip_mnemonic(instr)); + char rt[5] = {}, imm[24] = {}; i = parse_register(instr, parse_token(instr, i, rt), rt, 0x1F, 0, base, ','); - if (size_bit && rt[0] != '?') base.set_bits((rt[0] == 'W' ? 0 : 1), 1, size_bit); - if (instr[i++] != '[') return { false, false, false, false }; + if (rt[0] != 'W' && rt[0] != 'X' && rt[0] != '?') return { 0, 0 }; + int size = scale; + if (scale < 0 && rt[0] != '?') { + size = rt[0] == 'W' ? 2 : 3; + base.set_bits(size - 2, 1, 30); + } + if (instr[i++] != '[') return { 0, 0 }; // Rn i = parse_register(instr, i, 0x1F, 5, base); bool is_register_offset = false, is_pre_index = false, is_post_index = false; - uint32_t option_val = 0b011, s_val = 0; + uint32_t option_val = 0b011, s_val = 0, s_mask = 1; if (instr[i] == ',') { i = skip_whitespace(instr, ++i); @@ -274,7 +296,9 @@ namespace patterns { i = skip_whitespace(instr, skip_char(instr, i, ',')); if (instr[i] == '#') { - s_val = static_cast(stoi(&instr[++i])) > 0 ? 1 : 0; + // For bytes an explicit #0 is the shifted form; for the rest the amount is 0 or the scale + if (instr[++i] == '?') s_mask = 0; + else s_val = (size == 0 || stoi(&instr[i]) != 0) ? 1 : 0; } } } @@ -290,59 +314,42 @@ namespace patterns { } } - if (is_register_offset) base.set_bits((option_val << 1) | s_val, 0xF, 12); - - if (imm[0] != '?') { - int32_t val = stoi(imm); - if (!is_register_offset && (is_pre_index || is_post_index)) { - if (val >= -256 && val <= 255) - base.set_bits(val, 0x1FF, 12); - } else if (!is_register_offset && (size_bit == 0 || rt[0] != '?')) { - auto shift = size_bit > 0 ? (rt[0] == 'W' ? 2 : 3) : 1; - if ((val & 1) == 0 && (val >> shift) <= 0xFFF) { - base.set_bits((val >> shift), 0xFFF, 10); - } + const bool known_imm = imm[0] != '?'; + const int32_t val = known_imm ? stoi(imm) : 0; + bool is_unsigned_offset = !unscaled && !is_register_offset && !is_pre_index && !is_post_index; + if (is_unsigned_offset && known_imm && (val < 0 || (size > 0 && (val & ((1 << size) - 1))))) + is_unsigned_offset = false; + + const uint32_t width_mask = scale < 0 ? ~(1u << 30) : ~0u; // Rt's width bit, set above when known + if (is_unsigned_offset) { + base.set_bits(op, 0xFFC00000 & width_mask); + if (known_imm && size >= 0) { + if ((val >> size) > 0xFFF) return { 0, 0 }; + base.set_bits(val >> size, 0xFFF, 10); } + return base; } - return { true, is_register_offset, is_pre_index, is_post_index }; - } - - constexpr ArmOpcode parse_ldrh(const char* instr, ArmOpcode base) { - size_t i = skip_whitespace(instr, skip_mnemonic(instr)); - - auto [ ok, is_register_offset, is_pre_index, is_post_index ] = parse_load_store_defaults(instr, i, base); - if (!ok) return base; - - if (is_register_offset) base.set_bits(0x78600800, 0xFFE00C00); - else if (is_pre_index) base.set_bits(0x78400C00, 0xFFE00C00); - else if (is_post_index) base.set_bits(0x78400400, 0xFFE00C00); - else /* Unsigned offset*/ base.set_bits(0x79400000, 0xFFC00000); - + const uint32_t form = is_register_offset ? 0x00200800 : is_pre_index ? 0xC00 : is_post_index ? 0x400 : 0; + base.set_bits((op & ~0x01000000u) | form, 0xFFE00C00 & width_mask); + if (is_register_offset) { + base.set_bits(option_val << 1 | s_val, 0xE | s_mask, 12); + } else if (known_imm) { + if (val < -256 || val > 255) return { 0, 0 }; + base.set_bits(val, 0x1FF, 12); + } return base; } + // LDR is a load/store, or a PC-relative literal load (LDR Rt, #offset) constexpr ArmOpcode parse_ldr(const char* instr, ArmOpcode base) { + char rt[5] = {}; size_t i = skip_whitespace(instr, skip_mnemonic(instr)); - - auto [ ok, is_register_offset, is_pre_index, is_post_index ] = parse_load_store_defaults(instr, i, base, 30); - if (!ok) { - char imm[24] = {}; - i = parse_imm_token(instr, skip_whitespace(instr, skip_past(instr, i, ',')), imm); - - if (!(imm[0] == '?' && imm[1] == '\0')) { - int32_t val = stoi(imm); - if ((val & 3) != 0) val &= ~3; - base.set_bits((val >> 2), 0x7FFFF, 5); - } - return base; - } - - if (is_register_offset) base.set_bits(0xB8600800, 0xBFE00C00); - else if (is_pre_index) base.set_bits(0xB8400C00, 0xBFE00C00); - else if (is_post_index) base.set_bits(0xB8400400, 0xBFE00C00); - else base.set_bits(0xB9400000, 0xBFC00000); - + i = parse_register(instr, parse_token(instr, i, rt), rt, 0x1F, 0, base, ','); + if (instr[i] == '[') return parse_load_store<0xB9400000, -1>(instr, base); + if (rt[0] != 'W' && rt[0] != 'X' && rt[0] != '?') return { 0, 0 }; + if (rt[0] != '?') base.set_bits((rt[0] == 'W' ? 0 : 1), 1, 30); + parse_imm19(instr, i, base); return base; } @@ -465,7 +472,7 @@ namespace patterns { } } } else { - base.set_bits(0, (7 << 11 | 0x7F), 10); + base.set_bits(0, (7 << 11 | 0x3F), 10); } return base; @@ -475,6 +482,7 @@ namespace patterns { size_t i = skip_whitespace(instr, skip_mnemonic(instr)); char rt1[5] = {}; i = parse_register(instr, parse_token(instr, i, rt1), rt1, 0x1F, 0, base, ','); + if (rt1[0] != 'W' && rt1[0] != 'X' && rt1[0] != '?') return { 0, 0 }; // FP/SIMD pairs aren't supported bool know_size = rt1[0] != '?'; uint32_t size_bits = know_size ? (rt1[0] == 'W' ? 0b0 : 0b1) : 0; if (know_size) base.set_bits(size_bits, 1, 31); @@ -581,35 +589,46 @@ namespace patterns { return op; } - constexpr ArmOpcode parse_str(const char* instr, ArmOpcode base) { + constexpr ArmOpcode parse_cbz(const char* instr, ArmOpcode base) { size_t i = skip_whitespace(instr, skip_mnemonic(instr)); - auto [ ok, is_register_offset, is_pre_index, is_post_index ] = parse_load_store_defaults(instr, i, base, 30); - if (!ok) return base; - - if (is_register_offset) base.set_bits(0xB8200800, 0xBFE00C00); - else if (is_pre_index) base.set_bits(0xB8000C00, 0xBFE00C00); - else if (is_post_index) base.set_bits(0xB8000400, 0xBFE00C00); - else base.set_bits(0xB9000000, 0xBFC00000); + char rt[5] = {}; + i = parse_register(instr, parse_token(instr, i, rt), rt, 0x1F, 0, base, ','); + if (rt[0] != '?') base.set_bits((rt[0] == 'W' ? 0 : 1), 1, 31); + parse_imm19(instr, i, base); return base; } - constexpr ArmOpcode parse_cbz(const char* instr, ArmOpcode base) { + // CMP/CMN/TST Rn, ... are SUBS/ADDS/ANDS ZR, Rn, ...: parsed as that, with the zero register of Rn's width + template + constexpr ArmOpcode parse_zr_dest(const char* instr, ArmOpcode base) { size_t i = skip_whitespace(instr, skip_mnemonic(instr)); + const char* zr = instr[i] == 'W' ? "WZR, " : instr[i] == 'X' ? "XZR, " : "?, "; + char line[max_line + 8] = { '_', ' ' }; + size_t j = 2; + while (*zr) line[j++] = *zr++; + while (instr[i]) line[j++] = instr[i++]; + ArmOpcode op = parse(line, base); + op.set_bits(31, 0x1F); + return op; + } - char rt[5] = {}; - i = parse_register(instr, parse_token(instr, i, rt), rt, 0x1F, 0, base, ','); - if (rt[0] != '?') base.set_bits((rt[0] == 'W' ? 0 : 1), 1, 31); - - char imm[24] = {}; - i = parse_imm_token(instr, i, imm); + constexpr const char* conditions[] = { "EQ", "NE", "CS", "CC", "MI", "PL", "VS", "VC", "HI", "LS", "GE", "LT", "GT", "LE", "AL", "NV" }; - if (!(imm[0] == '?' && imm[1] == '\0')) { - int32_t val = stoi(imm); - if ((val & 3) != 0) val &= ~3; - base.set_bits((val >> 2), 0x7FFFF, 5); + // B.cond: the condition is part of the mnemonic (B.EQ, B.HS); B.? matches any + constexpr ArmOpcode parse_bcond(const char* instr) { + ArmOpcode base{ 0x54000000, 0xFF000010 }; + char cond[4] = {}; + for (size_t j = 0; j < 3 && instr[2 + j] && !isspace(instr[2 + j]); ++j) cond[j] = instr[2 + j]; + if (cond[0] != '?') { + uint32_t c = str_eq(cond, "HS") ? 2 : str_eq(cond, "LO") ? 3 : 16; + for (uint32_t k = 0; k < 16 && c == 16; ++k) + if (str_eq(cond, conditions[k])) c = k; + if (c == 16) return { 0, 0 }; + base.set_bits(c, 0xF); } + parse_imm19(instr, skip_whitespace(instr, skip_mnemonic(instr)), base); return base; } @@ -623,12 +642,30 @@ namespace patterns { { "BL", parse_branch_imm26 }, { "ADR", parse_adr }, { "ADRP", parse_adr }, - { "LDRH", parse_ldrh }, { "LDR", parse_ldr }, + { "LDRB", parse_load_store<0x39400000, 0> }, + { "LDRH", parse_load_store<0x79400000, 1> }, + { "LDRSW", parse_load_store<0xB9800000, 2> }, + { "STR", parse_load_store<0xB9000000, -1> }, + { "STRB", parse_load_store<0x39000000, 0> }, + { "STRH", parse_load_store<0x79000000, 1> }, + { "LDUR", parse_load_store<0xB9400000, -1, true> }, + { "LDURB", parse_load_store<0x39400000, 0, true> }, + { "LDURH", parse_load_store<0x79400000, 1, true> }, + { "LDURSW", parse_load_store<0xB9800000, 2, true> }, + { "STUR", parse_load_store<0xB9000000, -1, true> }, + { "STURB", parse_load_store<0x39000000, 0, true> }, + { "STURH", parse_load_store<0x79000000, 1, true> }, { "MOVZ", parse_movX }, { "MOVN", parse_movX }, { "MOVK", parse_movX }, { "ORR", parse_orr }, + { "AND", parse_orr }, + { "ANDS", parse_orr }, + { "EOR", parse_orr }, + { "TST", parse_zr_dest }, + { "CMP", parse_zr_dest }, + { "CMN", parse_zr_dest }, { "LDP", parse_ldp }, { "STP", parse_stp }, { "MOV", parse_mov }, @@ -636,7 +673,6 @@ namespace patterns { { "ADDS", parse_arithmetic }, { "SUB", parse_arithmetic }, { "SUBS", parse_arithmetic }, - { "STR", parse_str }, { "CBZ", parse_cbz }, { "CBNZ", parse_cbz }, { "BR", parse_branch_reg }, @@ -653,21 +689,31 @@ namespace patterns { { "ADDS", { 0x21000000, 0x65800000 } }, { "SUB", { 0x41000000, 0x65000000 } }, { "SUBS", { 0x61000000, 0x65000000 } }, - { "LDRH", { 0x38400000, 0x3B200000 } }, { "LDR", { 0x18000000, 0xBF000000 } }, { "MOVZ", { 0x52800000, 0x7F800000 } }, { "MOVN", { 0x12800000, 0x7F800000 } }, { "MOVK", { 0x72800000, 0x7F800000 } }, { "ORR", { 0x22000000, 0x67000000 } }, + { "AND", { 0x02000000, 0x67000000 } }, + { "ANDS", { 0x62000000, 0x67000000 } }, + { "EOR", { 0x42000000, 0x67000000 } }, + { "TST", { 0x62000000, 0x67000000 } }, // ANDS ZR + { "CMP", { 0x61000000, 0x65000000 } }, // SUBS ZR + { "CMN", { 0x21000000, 0x65800000 } }, // ADDS ZR { "LDP", { 0x28400000, 0x7E400000 } }, { "STP", { 0x28000000, 0x7E400000 } }, { "BR", { 0xD61F0000, 0xFFFFFC1F } }, { "BLR", { 0xD63F0000, 0xFFFFFC1F } }, - { "STR", { 0xB8000000, 0xBEC00000 } }, { "CBZ", { 0x34000000, 0x7F000000 } }, { "CBNZ", { 0x35000000, 0x7F000000 } }, { "NOP", { 0xD503201F, 0xFFFFFFFF } }, { "RET", { 0xD65F0000, 0xFFFFFC1F } }, + { "RETAA", { 0xD65F0BFF, 0xFFFFFFFF } }, + { "RETAB", { 0xD65F0FFF, 0xFFFFFFFF } }, + { "PACIASP", { 0xD503233F, 0xFFFFFFFF } }, + { "PACIBSP", { 0xD503237F, 0xFFFFFFFF } }, + { "AUTIASP", { 0xD50323BF, 0xFFFFFFFF } }, + { "AUTIBSP", { 0xD50323FF, 0xFFFFFFFF } }, }; constexpr ArmOpcode find_opcode(const char* mnemonic) { @@ -688,6 +734,7 @@ namespace patterns { } mnemonic[i] = '\0'; + if (starts_with(mnemonic, "B.")) return parse_bcond(instr); ArmOpcode base = find_opcode(mnemonic); for (const auto& entry : parseTable) { @@ -714,8 +761,6 @@ namespace patterns { return str[i] ? i + 1 : i; } - constexpr size_t max_line = 128; - // The ArmPattern syntax: one instruction per line, X before the one to resolve, flags after '/'. Calls // line(index, instruction) for each instruction (trimmed and uppercased), x(index) for the X line and // flags(text) for the text after '/'. Reads up to n chars or a terminator. Returns the number of instructions diff --git a/tests/test_arm.cpp b/tests/test_arm.cpp index 492c72e..66d5c26 100644 --- a/tests/test_arm.cpp +++ b/tests/test_arm.cpp @@ -95,7 +95,58 @@ X("STR W0, [X1,X2,LSL#2]", 0xB8227820) \ X("CBZ X0, #0x40", 0xB4000200) \ X("CBZ W0, #-0x8", 0x34FFFFC0) \ - X("CBNZ X1, #0x100", 0xB5000801) + X("CBNZ X1, #0x100", 0xB5000801) \ + X("LDR X0, [X1,#4]", 0xF8404020) \ + X("LDR X0, [X29,#-8]", 0xF85F83A0) \ + X("LDR W0, [X1,#-4]", 0xB85FC020) \ + X("STR X0, [X1,#-8]", 0xF81F8020) \ + X("STR WZR, [SP,#0xC]", 0xB9000FFF) \ + X("LDRH W0, [X1,#3]", 0x78403020) \ + X("LDUR X0, [X1,#0x10]", 0xF8410020) \ + X("STUR W0, [X29,#-4]", 0xB81FC3A0) \ + X("LDURB W0, [X1,#-1]", 0x385FF020) \ + X("STURH W0, [X1,#-2]", 0x781FE020) \ + X("LDURSW X0, [X1,#-4]", 0xB89FC020) \ + X("LDRB W0, [X1,#0x10]", 0x39404020) \ + X("LDRB W0, [X1,#3]", 0x39400C20) \ + X("LDRB W0, [X1,#1]!", 0x38401C20) \ + X("LDRB W0, [X1],#1", 0x38401420) \ + X("LDRB W0, [X1,X2]", 0x38626820) \ + X("LDRB W0, [X1,X2,LSL#0]", 0x38627820) \ + X("LDRB W0, [X1,W2,UXTW]", 0x38624820) \ + X("STRB W0, [X1,#0x10]", 0x39004020) \ + X("STRB WZR, [X1]", 0x3900003F) \ + X("STRH W0, [X1,#0x10]", 0x79002020) \ + X("STRH W0, [X1,#-2]!", 0x781FEC20) \ + X("LDRSW X0, [X1,#0x10]", 0xB9801020) \ + X("LDRSW X0, [X1,X2,LSL#2]", 0xB8A27820) \ + X("CMP X0, #0x10", 0xF100401F) \ + X("CMP W0, W1", 0x6B01001F) \ + X("CMP X0, X1, LSL#2", 0xEB01081F) \ + X("CMP X0, W1, UXTW", 0xEB21401F) \ + X("CMP W0, #1, LSL#12", 0x7140041F) \ + X("CMN W0, #1", 0x3100041F) \ + X("CMN X0, X1", 0xAB01001F) \ + X("TST W0, #0xFF", 0x72001C1F) \ + X("TST X0, X1", 0xEA01001F) \ + X("TST X0, #0x8000000000000000", 0xF241001F) \ + X("AND X0, X1, #0xFF", 0x92401C20) \ + X("AND W0, W1, W2", 0x0A020020) \ + X("ANDS X0, X1, X2, LSR#3", 0xEA420C20) \ + X("EOR W0, W1, #0x80000000", 0x52010020) \ + X("EOR X0, X1, X2", 0xCA020020) \ + X("B.EQ #0x10", 0x54000080) \ + X("B.NE #-0x8", 0x54FFFFC1) \ + X("B.HS #0x100", 0x54000802) \ + X("B.CS #0x100", 0x54000802) \ + X("B.LO #0x100", 0x54000803) \ + X("B.GT #0x20", 0x5400010C) \ + X("RETAA", 0xD65F0BFF) \ + X("RETAB", 0xD65F0FFF) \ + X("PACIASP", 0xD503233F) \ + X("PACIBSP", 0xD503237F) \ + X("AUTIASP", 0xD50323BF) \ + X("AUTIBSP", 0xD50323FF) // Wildcard pattern, an instruction, its encoding, and whether the pattern must match it #define WILDCARD(X) \ @@ -141,6 +192,7 @@ X("ADD W?, W?, #?", "ADD W0, W1, #1", 0x11000420, 1) \ X("ADD X?, X?, #?, LSL#12", "ADD X0, X1, #1, LSL#12", 0x91400420, 1) \ X("ADD X?, X?, X?", "ADD X0, X1, X2", 0x8B020020, 1) \ + X("ADD X?, X?, X?", "ADD X0, X1, X3", 0x8B030020, 1) \ X("ADD X?, X?, X?", "ADD X0, X1, #2", 0x91000820, 0) \ X("SUB SP, SP, #?", "SUB SP, SP, #0x80", 0xD10203FF, 1) \ X("SUB X?, X?, #?", "ADD X0, X1, #1", 0x91000420, 0) \ @@ -173,7 +225,49 @@ X("BR X?", "BLR X16", 0xD63F0200, 0) \ X("BLR X?", "BLR X8", 0xD63F0100, 1) \ X("RET", "RET X1", 0xD65F0020, 0) \ - X("NOP", "NOP", 0xD503201F, 1) + X("NOP", "NOP", 0xD503201F, 1) \ + X("RET", "RETAA", 0xD65F0BFF, 0) \ + X("LDR X?, [X29,#-8]", "LDUR X3, [X29,#-8]", 0xF85F83A3, 1) \ + X("LDR ?, [X29,#-8]", "LDUR W0, [X29,#-8]", 0xB85F83A0, 1) \ + X("LDR ?, [X29,#-8]", "LDR X0, [X29,#8]", 0xF94007A0, 0) \ + X("LDR X?, [X?,X?,LSL#?]", "LDR X2, [X3,X4]", 0xF8646862, 1) \ + X("LDUR X?, [X?,#?]", "LDUR X0, [X1,#0x10]", 0xF8410020, 1) \ + X("LDUR X?, [X?,#?]", "LDR X0, [X1,#0x10]", 0xF9400820, 0) \ + X("LDUR X?, [X?,#?]", "LDR X0, [X1,#8]!", 0xF8408C20, 0) \ + X("STUR ?, [X?,#?]", "STUR W0, [X29,#-4]", 0xB81FC3A0, 1) \ + X("STUR ?, [X?,#?]", "LDUR W0, [X29,#-4]", 0xB85FC3A0, 0) \ + X("LDRB W?, [X?,#?]", "LDRB W0, [X1,#0x10]", 0x39404020, 1) \ + X("LDRB W?, [X?,#?]", "LDRH W0, [X1,#0x10]", 0x79402020, 0) \ + X("LDRB W?, [X?,#?]", "STRB W0, [X1,#0x10]", 0x39004020, 0) \ + X("LDRB W?, [X?,X?,LSL#?]", "LDRB W0, [X1,X2,LSL#0]", 0x38627820, 1) \ + X("LDRB W?, [X?,X?,LSL#?]", "LDRB W0, [X1,X2]", 0x38626820, 1) \ + X("STRB ?, [X?,#?]", "STRB WZR, [X1]", 0x3900003F, 1) \ + X("STRH W?, [X?,#?]!", "STRH W0, [X1,#-2]!", 0x781FEC20, 1) \ + X("STRH W?, [X?,#?]!", "STRH W0, [X1,#0x10]", 0x79002020, 0) \ + X("LDRSW X?, [X?,#?]", "LDRSW X0, [X1,#0x10]", 0xB9801020, 1) \ + X("LDRSW X?, [X?,#?]", "LDR W0, [X1,#0x10]", 0xB9401020, 0) \ + X("CMP X?, #?", "CMP X0, #0x10", 0xF100401F, 1) \ + X("CMP X?, #?", "SUBS X1, X0, #0x10", 0xF1004001, 0) \ + X("CMP X?, #?", "CMP W0, #1", 0x7100041F, 0) \ + X("CMP ?, #?", "CMP W0, #1", 0x7100041F, 1) \ + X("CMP W?, W?", "CMP W0, W1", 0x6B01001F, 1) \ + X("CMP W?, W?", "CMN W0, W1", 0x2B01001F, 0) \ + X("CMP X?, X?, LSL#?", "CMP X0, X1, LSL#2", 0xEB01081F, 1) \ + X("CMN W?, #?", "CMN W0, #1", 0x3100041F, 1) \ + X("TST W?, #?", "TST W0, #0xFF", 0x72001C1F, 1) \ + X("TST W?, #?", "ANDS W1, W0, #0xFF", 0x72001C01, 0) \ + X("TST X?, X?", "TST X0, X1", 0xEA01001F, 1) \ + X("AND X?, X?, #?", "AND X0, X1, #0xFF", 0x92401C20, 1) \ + X("AND X?, X?, #?", "ORR X0, X1, #0xFF", 0xB2401C20, 0) \ + X("EOR X?, X?, X?", "EOR X0, X1, X2", 0xCA020020, 1) \ + X("EOR X?, X?, X?", "ORR X0, X1, X2", 0xAA020020, 0) \ + X("ANDS X?, X?, X?, LSR#?", "ANDS X0, X1, X2, LSR#3", 0xEA420C20, 1) \ + X("B.EQ #?", "B.EQ #0x10", 0x54000080, 1) \ + X("B.EQ #?", "B.NE #0x10", 0x54000081, 0) \ + X("B.? #?", "B.GT #0x20", 0x5400010C, 1) \ + X("B.? #?", "B #0x10", 0x14000004, 0) \ + X("B.? ?", "CBZ W0, #0x10", 0x34000080, 0) \ + X("B.HS ?", "B.CS #0x100", 0x54000802, 1) struct Word { uint32_t pat, mask; }; template static Word word(const P& p, size_t idx = 0) { @@ -207,6 +301,18 @@ static void wildcards() { printf(" wildcard matching ok (%d)\n", n); } +// Forms the encoder doesn't support become a whole-word wildcard (mask 0), which the IDA plugin warns about, +// rather than a pattern that can never match +static void unsupported() { + CHECK(word(ARM_PATTERN("LDR Q0, [X1,#0x10]")).mask == 0); + CHECK(word(ARM_PATTERN("STR D0, [SP,#8]")).mask == 0); + CHECK(word(ARM_PATTERN("STP D9, D8, [SP,#-0x20]!")).mask == 0); + CHECK(word(ARM_PATTERN("LDR W0, [X1,#0x4000]")).mask == 0); // past the unsigned offset's range + CHECK(word(ARM_PATTERN("LDUR X0, [X1,#0x100]")).mask == 0); // past imm9 + CHECK(word(ARM_PATTERN("B.XX #0x10")).mask == 0); + puts(" unsupported forms ok"); +} + template static bool same(const A& a, const B& b) { return a.length() == b.length() && a.offset() == b.offset() && a.deref() == b.deref() && a.aligned() == b.aligned() && !memcmp(a.pattern(), b.pattern(), a.length()) && !memcmp(a.mask(), b.mask(), a.length()); @@ -322,7 +428,7 @@ static void deref_synthetic() { #define SYM(x) "_" x #define ASM_LINE(text, ...) text "\n" #define ASM_WILD(ptext, itext, ...) itext "\n" -__asm__(".data\n.p2align 2\n.globl " SYM("asm_concrete") "\n" SYM("asm_concrete") ":\n" CONCRETE(ASM_LINE) +__asm__(".data\n.p2align 2\n.arch_extension pauth\n.globl " SYM("asm_concrete") "\n" SYM("asm_concrete") ":\n" CONCRETE(ASM_LINE) ".globl " SYM("asm_wildcard") "\n" SYM("asm_wildcard") ":\n" WILDCARD(ASM_WILD)); extern "C" const uint32_t asm_concrete[], asm_wildcard[]; @@ -378,6 +484,7 @@ static void deref_relocated() { int main() { concrete(); wildcards(); + unsupported(); line_formats(); prologue(); runtime_patterns(); From f4bd686c8cd90c78497c0a9ff18340659511b742 Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:53:43 -0700 Subject: [PATCH 18/19] Fix arm64 encodings and /d decoding, nibble wildcards, plugin crash Review fixes for PR #4, each checked against clang's assembler: arm64 encoder - ORR/AND/EOR/TST: shift amount after a space ("LSL #3") was read as 0, and #? never a wildcard. - ADD/SUB/CMP/CMN immediates: negative values are the opposite operation, multiples of 0x1000 the LSL #12 form; #? no longer pins the shift bit (missed SUB SP, SP, #1, LSL #12). - ADD/SUB register with SP: extended UXTX/UXTW form, as assemblers emit. - MOV Rd, #? no longer matches MOVK. LDP/STP [Rn] is the signed form with offset 0, not a partial wildcard. - ADR #-1 was a wildcard. With Rd (Rt1) "?", the width comes from Rn (Rt2). - A line starting with X followed by a letter (XPACLRI) isn't the X marker. arm64 /d - MOVZ shifted by hw instead of hw*16; MOVN decoded. - One load/store decoder for all general-purpose immediate forms (LDRB/LDRSW/STRB/STRH/LDUR/STUR): signed offsets, pre/post-index, and STR offsets >= 0x4000 that threw. - B.cond, CBZ/CBNZ and LDR (literal) resolve to their target. - ADRP chains skip instructions that leave the register alone (another ADRP between ADRP and ADD). Byte patterns - Nibble wildcards ("4?", "?8"); "?8" used to silently become a whole-byte wildcard. - /r /d reads use memcpy (misaligned loads were UB, caught by UBSan). - stoi accumulates unsigned: long immediates no longer overflow a signed int. IDA plugin: an over-long arm64 line threw out of the dialog handler (listing unsupported lines ran outside the try) and would take IDA down. CI: also run the library tests on Linux. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- README.md | 11 +- include/PatternScanner/ArmPattern.hpp | 96 +++++--- include/PatternScanner/CompileTimePattern.hpp | 2 +- include/PatternScanner/Pattern.hpp | 220 +++++++----------- plugins/ida/pattern_scanner.cpp | 18 +- tests/test_arm.cpp | 65 +++++- tests/test_patterns.cpp | 18 ++ 8 files changed, 249 insertions(+), 183 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9e4c9d..b61599e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,7 +12,7 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/README.md b/README.md index a02646c..43b6543 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ CompileTimePattern creates the pattern & mask during compile time and stored in XORPattern creates the pattern & mask along with two keys arrays during compile time. Pattern and mask are both stored XOR'd (with separate keys), otherwise it acts just the same as the CompileTimePattern. (the pattern() and mask() methods however return the XOR'd bytes; access the original pattern bytes with the [] operator and the original mask bytes with unmask()) -ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Supported: `B`, `BL`, `B.cond` (`B.?` for any condition), `BR`, `BLR`, `RET`, `RETAA`, `RETAB`, `CBZ`, `CBNZ`, `ADR`, `ADRP`, `ADD(S)`, `SUB(S)`, `CMP`, `CMN`, `AND(S)`, `ORR`, `EOR`, `TST`, `LDR`, `LDRB`, `LDRH`, `LDRSW`, `STR`, `STRB`, `STRH`, `LDUR`/`STUR` (and their `B`/`H`/`SW` forms), `LDP`, `STP`, `MOV`, `MOVZ`, `MOVN`, `MOVK`, `NOP`, `PACIASP`, `PACIBSP`, `AUTIASP`, `AUTIBSP`. Loads and stores take W/X registers only; floating-point/SIMD ones (`LDR Q0, ...`, `STP D9, D8, ...`) are wildcards. Any other instruction becomes a wildcard for those 4 bytes, matching any instruction (the IDA plugin asks before searching with one); write `?` for a deliberate one. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). One instruction per line; case, indentation and spacing don't matter, so lines pasted from IDA (`LDR W8, [X8,#0x18]`) or lldb (`ldr w8, [x8, #0x18]`) both work. lldb prints ADRP operands as page counts and branch targets as absolute addresses, so wildcard those (`#?`). `MOV` is encoded the way assemblers pick it (register, SP, or immediate via `MOVZ`/`MOVN`/`ORR`); `MOV Rd, #?` matches `MOVZ` or `MOVN`. `CMP`/`CMN`/`TST` are encoded as `SUBS`/`ADDS`/`ANDS` to the zero register, and an `LDR`/`STR` offset the scaled form can't hold (negative, unaligned) as `LDUR`/`STUR`, as assemblers do. Other aliases aren't implemented. +ArmPattern creates the pattern & mask from the arm64 assembly, with wildcard support. Supported: `B`, `BL`, `B.cond` (`B.?` for any condition), `BR`, `BLR`, `RET`, `RETAA`, `RETAB`, `CBZ`, `CBNZ`, `ADR`, `ADRP`, `ADD(S)`, `SUB(S)`, `CMP`, `CMN`, `AND(S)`, `ORR`, `EOR`, `TST`, `LDR`, `LDRB`, `LDRH`, `LDRSW`, `STR`, `STRB`, `STRH`, `LDUR`/`STUR` (and their `B`/`H`/`SW` forms), `LDP`, `STP`, `MOV`, `MOVZ`, `MOVN`, `MOVK`, `NOP`, `PACIASP`, `PACIBSP`, `AUTIASP`, `AUTIBSP`. Loads and stores take W/X registers only; floating-point/SIMD ones (`LDR Q0, ...`, `STP D9, D8, ...`) are wildcards. Any other instruction becomes a wildcard for those 4 bytes, matching any instruction (the IDA plugin asks before searching with one); write `?` for a deliberate one. The class is constexpr also, so the assembly is not in the final product (if declaring the pattern with constexpr). One instruction per line; case, indentation and spacing don't matter, so lines pasted from IDA (`LDR W8, [X8,#0x18]`) or lldb (`ldr w8, [x8, #0x18]`) both work. lldb prints ADRP operands as page counts and branch targets as absolute addresses, so wildcard those (`#?`). `MOV` is encoded the way assemblers pick it (register, SP, or immediate via `MOVZ`/`MOVN`/`ORR`); `MOV Rd, #?` matches `MOVZ` or `MOVN`. `CMP`/`CMN`/`TST` are encoded as `SUBS`/`ADDS`/`ANDS` to the zero register, and an `LDR`/`STR` offset the scaled form can't hold (negative, unaligned) as `LDUR`/`STUR`, `ADD`/`SUB`/`CMP` immediates that are negative or multiples of 0x1000 as the opposite operation or `LSL #12`, and register `ADD`/`SUB` with `SP` in the extended (`UXTX`) form, as assemblers do. Other aliases aren't implemented. ArmXORPattern (`PatternScanner/ArmXORPattern.hpp`) is ArmPattern with the instruction words and masks stored XOR'd, the way XORPattern stores them. Same syntax, flags and results, plus `[]` and unmask() for the originals. Decoded on the stack only while scanning. @@ -37,10 +37,17 @@ Compile-time patterns (`_ctpattern`, `_xorpattern`, `_armpattern`, `_armxorpatte RuntimeArmPattern (`PatternScanner/RuntimeArmPattern.hpp`) is ArmPattern built at runtime, for assembly only known then (user input, config files). #### Target architecture -/d, /r, `X` and /a depend on the architecture of the code being scanned (`patterns::arch::x86`, `x64` or `arm64`). By default that's the one you compile for (`patterns::host_arch`); ArmPattern, ArmXORPattern and RuntimeArmPattern are always arm64. To scan code of another architecture, e.g. from a disassembler plugin, pass it to RuntimePattern: `patterns::RuntimePattern("E8 X4 ? ? ? ? /d", patterns::arch::x86)`. `resolve(match, end, &is_value)` reports whether the result is an address or a value read from the code (/r; arm64 /d on MOVZ, ADD, or a lone LDR/STR offset). +/d, /r, `X` and /a depend on the architecture of the code being scanned (`patterns::arch::x86`, `x64` or `arm64`). By default that's the one you compile for (`patterns::host_arch`); ArmPattern, ArmXORPattern and RuntimeArmPattern are always arm64. To scan code of another architecture, e.g. from a disassembler plugin, pass it to RuntimePattern: `patterns::RuntimePattern("E8 X4 ? ? ? ? /d", patterns::arch::x86)`. `resolve(match, end, &is_value)` reports whether the result is an address or a value read from the code (/r; arm64 /d on MOVZ/MOVN, ADD, or a lone load/store offset). + +arm64 /d at the X instruction: `B`/`BL`, `B.cond`, `CBZ`/`CBNZ`, `LDR` (literal) and `ADR` give their target; `ADRP` gives the page plus the `ADD`/`SUB` and load/store offset that follow it through the same register (skipping NOPs and instructions that leave the register alone); `MOVZ`/`MOVN`, `ADD`/`SUB` immediates and load/store offsets give the value. Scanning uses SSE2 (x86/x64) or NEON (arm64) when available. Define `PATTERNS_NO_SIMD` before including to use a portable 64-bit integer scan instead (e.g. kernel code). +#### Wildcards: +- `?` or `??`: any byte +- `4?` / `?8`: a nibble wildcard (high nibble 4, any low nibble / any high nibble, low nibble 8) +- ArmPattern: `?` for a register (`X?`), immediate (`#?`), shift (`LSL #?`), condition (`B.?`) or a whole instruction line + #### Pattern Flags: - d: Dereference from the found address, or found address + offset ( addr + size + relative address ) - r: Read the relative value at the address, or at the address + offset (not used on arm64) diff --git a/include/PatternScanner/ArmPattern.hpp b/include/PatternScanner/ArmPattern.hpp index e6c43b0..dcb5bf6 100644 --- a/include/PatternScanner/ArmPattern.hpp +++ b/include/PatternScanner/ArmPattern.hpp @@ -101,6 +101,15 @@ namespace patterns { return 0xFFFFFFFF; } + // 1 for a 64-bit register (X, SP, XZR), 0 for 32-bit (W, WSP, WZR), -1 when unknown ("?") + constexpr int register_width(const char* token) { + return token[0] == 'X' || str_eq(token, "SP") ? 1 : token[0] == 'W' ? 0 : -1; + } + + constexpr bool is_sp(const char* token) { + return str_eq(token, "SP") || str_eq(token, "WSP"); + } + template constexpr void parse_register(char(&buffer)[N], uint32_t mask, uint32_t shift, ArmOpcode& base) { if (auto val = parse_register(buffer); val != 0xFFFFFFFF) @@ -240,10 +249,9 @@ namespace patterns { char imm[24] = {}; i = parse_imm_token(instr, skip_char(instr, i, '#'), imm); if (!(imm[0] == '?' && imm[1] == '\0')) { - if (int64_t offset = stoi(imm); offset != -1) { - if ((base.pattern >> 31) & 1) offset >>= 12; - base.set_bits(static_cast((offset << 24) | ((offset >> 2) & 0x7FFFF)), (3 << 24 | 0x7FFFF), 5); - } + int64_t offset = stoi(imm); + if ((base.pattern >> 31) & 1) offset >>= 12; + base.set_bits(static_cast((offset << 24) | ((offset >> 2) & 0x7FFFF)), (3 << 24 | 0x7FFFF), 5); } return base; } @@ -356,13 +364,14 @@ namespace patterns { constexpr ArmOpcode parse_orr(const char* instr, ArmOpcode base) { size_t i = skip_whitespace(instr, skip_mnemonic(instr)); - char rd[5] = {}/*, rn[5] = {}*/; + char rd[5] = {}, rn[5] = {}; i = parse_register(instr, parse_token(instr, i, rd), rd, 0x1F, 0, base, ','); - if (rd[0] != '?') base.set_bits((rd[0] == 'W' ? 0 : 1), 1, 31); - // Rn - i = parse_register(instr, i, 0x1F, 5, base, ','); - + i = parse_register(instr, parse_token(instr, i, rn), rn, 0x1F, 5, base, ','); + // Width from Rd, or from Rn when Rd is "?" + const int width = register_width(rd) >= 0 ? register_width(rd) : register_width(rn); + if (width >= 0) base.set_bits(static_cast(width), 1, 31); + if (instr[i] == '#') { base.set_bits((2 << 4), (3 << 4 | 1), 23); @@ -371,8 +380,8 @@ namespace patterns { if (imm[0] != '?') { uint64_t immVal = stoi(imm); // Unknown width (Rd is "?"): a value that fits in 32 bits can be either, and the two only differ in N - const bool is64 = rd[0] == 'X' || (rd[0] != 'W' && immVal > 0xFFFFFFFFULL); - const uint32_t n_mask = (rd[0] == 'X' || rd[0] == 'W' || is64) ? 1 : 0; + const bool is64 = width == 1 || (width < 0 && immVal > 0xFFFFFFFFULL); + const uint32_t n_mask = (width >= 0 || is64) ? 1 : 0; auto [ ok, N, imms, immr ] = encode_bitmask_immediate(immVal, is64); if (ok) base.set_bits((N << 12 | immr << 6 | imms), (n_mask << 12 | 0x3F << 6 | 0x3F), 10); } @@ -391,7 +400,7 @@ namespace patterns { else if (starts_with(&instr[i], "ROR")) { base.set_bits(3, 3, 22); i += 3; } else base.set_bits(0, 3, 22); - i = skip_whitespace(instr, skip_char(instr, i, '#')); + i = skip_char(instr, skip_whitespace(instr, i), '#'); if (instr[i] != '?') base.set_bits(stoi(&instr[i]), 0x3F, 10); } else { @@ -404,31 +413,44 @@ namespace patterns { constexpr ArmOpcode parse_arithmetic(const char* instr, ArmOpcode base) { size_t i = skip_whitespace(instr, skip_mnemonic(instr)); - char rd[5] = {}; + char rd[5] = {}, rn[5] = {}; i = parse_register(instr, parse_token(instr, i, rd), rd, 0x1F, 0, base, ','); - if (rd[0] != '?') base.set_bits((rd[0] == 'W' ? 0b0 : 0b1), 1, 31); - // Rn - i = parse_register(instr, i, 0x1F, 5, base, ','); + i = parse_register(instr, parse_token(instr, i, rn), rn, 0x1F, 5, base, ','); + // Width from Rd, or from Rn when Rd is "?" + const int width = register_width(rd) >= 0 ? register_width(rd) : register_width(rn); + if (width >= 0) base.set_bits(static_cast(width), 1, 31); if (instr[i] == '#') { base.set_bits((2 << 4 | 2), (3 << 4 | 7), 23); char imm[24] = {}; - i = parse_imm_token(instr, i, imm); - if (imm[0] != '?') base.set_bits(stoi(imm), 0xFFF, 10); - + i = skip_whitespace(instr, parse_imm_token(instr, i, imm)); + int sh = -1; // LSL #12 (1) or #0 (0) when written, -1 if not or #? if (instr[i] == ',') { - i = skip_whitespace(instr, i+1); + i = skip_whitespace(instr, i + 1); if (starts_with(&instr[i], "LSL")) { i = skip_char(instr, skip_whitespace(instr, i + 3), '#'); - if (instr[i] != '?') base.set_bits(stoi(&instr[i]) == 12 ? 1 : 0, 1, 22); + if (instr[i] != '?') sh = stoi(&instr[i]) == 12 ? 1 : 0; } - } else { - base.set_mask(1, 22); } + if (imm[0] != '?') { + int64_t val = stoi(imm); + // As assemblers do: a negative immediate is the opposite operation (ADD #-8 is SUB #8, CMP #-1 + // is CMN #1), and a multiple of 0x1000 the shifted form (#0x1000 is #1, LSL #12) + if (val < 0) { base.pattern ^= 1u << 30; val = -val; } + if (sh < 0 && val > 0xFFF && !(val & 0xFFF)) { sh = 1; val >>= 12; } + if (val > 0xFFF) return { 0, 0 }; + base.set_bits(static_cast(val), 0xFFF, 10); + if (sh < 0) sh = 0; + } + if (sh >= 0) base.set_bits(static_cast(sh), 1, 22); return base; } + // With SP as Rd or Rn only the extended register form exists: assemblers write LSL as UXTX (UXTW for W) + const bool sp = is_sp(rd) || is_sp(rn); + const uint32_t sp_extend = str_eq(rd, "WSP") || str_eq(rn, "WSP") ? 2 : 3; + base.set_bits(5, 0xF, 25); // Rm i = parse_register(instr, i, 0x1F, 16, base); @@ -436,7 +458,8 @@ namespace patterns { if (instr[i] == ',') { i = skip_whitespace(instr, i + 1); bool shifted = false, extended = false; - if (starts_with(&instr[i], "LSL")) { base.set_bits(0, 7, 21); shifted = true; i += 3; } + if (starts_with(&instr[i], "LSL") && sp) { base.set_bits(sp_extend, 7, 13); extended = true; i += 3; } + else if (starts_with(&instr[i], "LSL")) { base.set_bits(0, 7, 21); shifted = true; i += 3; } else if (starts_with(&instr[i], "LSR")) { base.set_bits(2, 7, 21); shifted = true; i += 3; } else if (starts_with(&instr[i], "ASR")) { base.set_bits(4, 7, 21); shifted = true; i += 3; } else if (starts_with(&instr[i], "UXTB")) { base.set_bits(0, 7, 13); extended = true; i += 4; } @@ -471,6 +494,8 @@ namespace patterns { base.set_bits((1 << 11), (7 << 11 | 7), 10); } } + } else if (sp) { + base.set_bits((1 << 11 | sp_extend << 3), (7 << 11 | 0x3F), 10); } else { base.set_bits(0, (7 << 11 | 0x3F), 10); } @@ -483,25 +508,25 @@ namespace patterns { char rt1[5] = {}; i = parse_register(instr, parse_token(instr, i, rt1), rt1, 0x1F, 0, base, ','); if (rt1[0] != 'W' && rt1[0] != 'X' && rt1[0] != '?') return { 0, 0 }; // FP/SIMD pairs aren't supported - bool know_size = rt1[0] != '?'; - uint32_t size_bits = know_size ? (rt1[0] == 'W' ? 0b0 : 0b1) : 0; + char rt2[5] = {}; + i = parse_register(instr, parse_token(instr, i, rt2), rt2, 0x1F, 10, base, ','); + // Width from Rt1, or from Rt2 when Rt1 is "?" + const int width = register_width(rt1) >= 0 ? register_width(rt1) : register_width(rt2); + const bool know_size = width >= 0; + const uint32_t size_bits = know_size ? static_cast(width) : 0; if (know_size) base.set_bits(size_bits, 1, 31); - // Rt2 - i = parse_register(instr, i, 0x1F, 10, base, ','); - if (instr[i] != '[') return base; i = skip_whitespace(instr, skip_char(instr, i, '[')); // Rn i = parse_register(instr, i, 0x1F, 5, base, ','); - bool post_index = false, signed_offset = false, pre_index = false; + // [Rn] alone is the signed offset form with offset 0 (imm stays empty, which reads as 0) + bool post_index = false, pre_index = false; char imm[24] = {}; - if (instr[i] == '#') { + if (instr[i] == '#') i = skip_whitespace(instr, parse_imm_token(instr, i, imm)); - signed_offset = true; - } if (instr[i] == ']') { i = skip_whitespace(instr, skip_char(instr, skip_char(instr, i, ']'), ',')); if (instr[i] == '!') { @@ -511,7 +536,6 @@ namespace patterns { i = parse_imm_token(instr, i, imm); } } - if (!(post_index || pre_index || signed_offset)) return base; base.set_bits((post_index ? 0b0010100011 : pre_index ? 0b0010100111 : 0b0010100101), 0x1FF, 22); @@ -552,7 +576,7 @@ namespace patterns { char imm[24] = {}; parse_imm_token(instr, i, imm); if (imm[0] == '?') { - op.set_bits(0x12800000, 0x1F800000); // MOVZ or MOVN: opc (bits 29-30) left open + op.set_bits(0x12800000, 0x3F800000); // MOVZ or MOVN: bit 30 of opc left open, bit 29 clear (not MOVK) return op; } const unsigned width = is64 ? 64 : 32; @@ -775,7 +799,7 @@ namespace patterns { if (text[0] == '/') break; if (!text[0]) continue; size_t start = 0; - if (text[0] == 'X') { x(count); start = skip_whitespace(text, 1); } + if (text[0] == 'X' && (!text[1] || isspace(text[1]))) { x(count); start = skip_whitespace(text, 1); } line(count++, &text[start]); } return count; diff --git a/include/PatternScanner/CompileTimePattern.hpp b/include/PatternScanner/CompileTimePattern.hpp index 7e08bf8..c92d336 100644 --- a/include/PatternScanner/CompileTimePattern.hpp +++ b/include/PatternScanner/CompileTimePattern.hpp @@ -6,7 +6,7 @@ namespace patterns { namespace detail { constexpr size_t pattern_length(const char* s, size_t nstr) { - return parse_bytes(s, nstr - 1, [](size_t, const char*) {}, [](size_t, const char*) {}, [](const char*) {}); + return parse_bytes(s, nstr - 1, [](size_t, char, char) {}, [](size_t, const char*) {}, [](const char*) {}); } template diff --git a/include/PatternScanner/Pattern.hpp b/include/PatternScanner/Pattern.hpp index 3161710..aa7ca9c 100644 --- a/include/PatternScanner/Pattern.hpp +++ b/include/PatternScanner/Pattern.hpp @@ -3,6 +3,7 @@ #include #include #include +#include // Define PATTERNS_NO_SIMD where vector registers are off-limits (e.g. kernel code); find() then uses 64-bit integer (SWAR) scanning #if defined(PATTERNS_NO_SIMD) @@ -85,29 +86,19 @@ namespace patterns { constexpr __forceinline char get_bits(char c) { return is_digit(c) ? (c - '0') : ((c & (~0x20)) - 'A' + 0xA); } - template - constexpr T stoi_impl(const char* str, T value = 0, bool negative = false, bool hex = false) { - if (*str == '\0') return negative ? -value : value; - if (hex) { - if (is_hex_digit(*str)) - return stoi_impl(str + 1, get_bits(*str) + value * 16, negative, hex); - } else { - if (is_digit(*str)) - return stoi_impl(str + 1, (*str - '0') + value * 10, negative, hex); - } - return negative ? -value : value; - } + // Decimal or 0x hex, optionally negative; stops at the first other character. Accumulates unsigned, so long + // numbers wrap instead of overflowing template constexpr T stoi(const char* str) { - if (*str == '-') { - ++str; - if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) - return stoi_impl(str + 2, 0, true, true); - return stoi_impl(str, 0, true); - } - if (str[0] == '0' && (str[1] == 'x' || str[1] == 'X')) - return stoi_impl(str + 2, 0, false, true); - return stoi_impl(str); + using U = std::make_unsigned_t; + const bool negative = *str == '-'; + if (negative) ++str; + const bool hex = str[0] == '0' && (str[1] == 'x' || str[1] == 'X'); + if (hex) str += 2; + U value = 0; + for (; hex ? is_hex_digit(*str) : is_digit(*str); ++str) + value = value * (hex ? 16 : 10) + static_cast(get_bits(*str)); + return static_cast(negative ? U(0) - value : value); } // Masked compare, 8 bytes at a time (memcpy keeps unaligned loads legal; compilers emit a single mov) @@ -123,9 +114,9 @@ namespace patterns { return true; } - // The byte pattern syntax ("48 8B ? X4 ? ? ? ? /d"). Calls byte(index, text) for each byte (text at its two hex - // digits or '?'), x(index, len_text) for the X marker (len_text at the digits after X, or nullptr) and - // flags(text) for the text after '/'. Returns the number of bytes + // The byte pattern syntax ("48 8B ? X4 ? ? ? ? /d"). Calls byte(index, hi, lo) for each byte (its two characters: + // hex digits or '?'; lo is '\0' past the end), x(index, len_text) for the X marker (len_text at the digits after + // X, or nullptr) and flags(text) for the text after '/'. Returns the number of bytes template constexpr size_t parse_bytes(const char* p, size_t len, Byte&& byte, X&& x, Flags&& flags) { size_t n = 0; @@ -142,7 +133,7 @@ namespace patterns { break; } else if (c != ' ') - byte(n++, &p[i]); + byte(n++, c, i + 1 < len ? p[i + 1] : '\0'); else --i; } return n; @@ -203,102 +194,47 @@ namespace patterns { return false; } - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/LDRH--immediate---Load-Register-Halfword--immediate-- - inline bool a64_decode_ldrh(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { - // rn - name of general-purpose base register or stack pointer - // rt - name of general-purpose base register to be transfered - // Post-index & Pre-index masks - // 0111 1000 010? ???? ???? x1?? ???? ???? - // 1 - pre-index - // 0 - post-index - if (decode_masked_match(insn, 0b1111'1111'1110'0000'0000'0100'0000'0000, 0b0111'1000'0100'0000'0000'0100'0000'0000)) { - *offset = extract_bitfield(insn, 9, 12); - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; + // https://developer.arm.com/documentation/ddi0602/2023-12/Index-by-Encoding/Loads-and-Stores + // Single general-purpose register loads/stores with an immediate offset (LDR/STR/LDRB/LDRH/LDRSW... and the + // LDUR/STUR forms). *offset is signed; *post_index is set for [Rn],#imm, which accesses Rn itself + inline bool a64_decode_load_store(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset, bool* post_index = nullptr) { + // size 111 0 01 opc imm12 Rn Rt: unsigned offset, scaled by the access size + if (decode_masked_match(insn, 0x3F000000, 0x39000000)) { + *offset = static_cast(extract_bitfield(insn, 12, 10)) << (insn >> 30); + if (post_index) *post_index = false; } - // Unsigned offset mask - // 0111 1001 01?? ???? ???? ???? ???? ???? - if (decode_masked_match(insn, 0b1111'1111'1100'0000'0000'0000'0000'0000, 0b0111'1001'0100'0000'0000'0000'0000'0000)) { - *offset = extract_bitfield(insn, 12, 10) << 1; - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - return false; - } - - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/LDR--immediate---Load-Register--immediate-- - inline bool a64_decode_ldr(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { - // rn - name of general-purpose base register or stack pointer - // rt - name of general-purpose register to be transferred - // Post-index & Pre-index masks - // 1x11 1000 010? ???? ???? x1?? ???? ???? - // 1 - 64 bit 1 - pre-index - // 0 - 32 bit 0 - post-index - if (decode_masked_match(insn, 0b1011'1111'1110'0000'0000'0100'0000'0000, 0b1011'1000'0100'0000'0000'0100'0000'0000)) { - // Have not tested this yet + // size 111 0 00 opc 0 imm9 mode Rn Rt: unscaled (00), post-index (01), unprivileged (10), pre-index (11) + else if (decode_masked_match(insn, 0x3F200000, 0x38000000)) { *offset = extract_bitfield(insn, 9, 12); - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; + if (post_index) *post_index = ((insn >> 10) & 3) == 1; } - // Unsigned offset mask - // 1x11 1001 01?? ???? ???? ???? ???? ???? - // 1 - 64 bit - // 0 - 32 bit - if (decode_masked_match(insn, 0b1011'1011'1100'0000'0000'0000'0000'0000, 0b1011'1001'0100'0000'0000'0000'0000'0000)) { - *offset = extract_bitfield(insn, 12, 10) << (insn >> 30); - // bit 26 annotated as "VR" - Variant Register (Not sure it's use yet) - //auto vr = (insn >> 26) & 1; - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - return false; + else return false; + if (rn) *rn = (insn >> 5) & 0x1f; + if (rt) *rt = insn & 0x1f; + return true; } - // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/STR--immediate---Store-Register--immediate-- - inline bool a64_decode_str(uint32_t insn, unsigned* rn, unsigned* rt, int64_t* offset) { - // rn - name of general-purpose base register or stack pointer - // rt - name of general-purpose register to be transferred - // Post-index & Pre-index masks - // 1x11 1000 000? ???? ???? x1?? ???? ???? - // 1 - 64 bit 1 - pre-index - // 0 - 32 bit 0 - post-index - if (decode_masked_match(insn, 0b1011'1011'1110'0000'0000'0100'0000'0000, 0b1011'1000'0000'0000'0000'0100'0000'0000)) { - *offset = extract_bitfield(insn, 9, 12); - // bit 26 annotated as "VR" - Variant Register (Not sure it's use yet) - //auto vr = (insn >> 26) & 1; - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; - return true; - } - // Unsigned offset mask - // 1x11 1000 000? ???? ???? ???? ???? ???? - // 1 - 64 bit - // 0 - 32 bit - if (decode_masked_match(insn, 0b1011'1011'1110'0000'0000'0000'0000'0000, 0b1011'1001'0000'0000'0000'0000'0000'0000)) { - *offset = extract_bitfield(insn, 12, 10) << (insn >> 30); - // bit 26 annotated as "VR" - Variant Register (Not sure it's use yet) - //auto vr = (insn >> 26) & 1; - if (rn) *rn = (insn >> 5) & 0x1f; - if (rt) *rt = insn & 0x1f; + // B.cond, CBZ/CBNZ and LDR (literal): a PC-relative imm19 + inline bool a64_decode_imm19(uint32_t insn, int64_t* offset) { + if (decode_masked_match(insn, 0xFF000010, 0x54000000) || decode_masked_match(insn, 0x7E000000, 0x34000000) + || decode_masked_match(insn, 0x3F000000, 0x18000000)) { + *offset = static_cast(extract_bitfield(insn, 19, 5)) * 4; return true; } return false; - } + } // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/MOVZ--Move-wide-with-zero- + // https://developer.arm.com/documentation/ddi0602/2023-12/Base-Instructions/MOVN--Move-wide-with-NOT- inline bool a64_decode_movz(uint32_t insn, unsigned* sf, unsigned* rd, int64_t* offset) { // sf - 1 for 64 bit, 0 for 32 bit // rd - destination register - // x101 0010 1xx? ???? ???? ???? ???? ???? - // 1 = 64 bit 0x - shift ammount - // 0 = 32 bit - if (decode_masked_match(insn, 0b0111'1111'1000'0000'0000'0000'0000'0000, 0b0101'0010'1000'0000'0000'0000'0000'0000)) { - auto hw = (insn >> 21) & 3; - *offset = hw ? extract_bitfield(insn, 16, 5) << hw : extract_bitfield(insn, 16, 5); + // sx01 0010 1hh? ???? ???? ???? ???? ???? + // s: 1 = 64 bit, 0 = 32 bit. x: 1 = MOVZ, 0 = MOVN (bit 29 clear excludes MOVK). hh: shift / 16 + if (decode_masked_match(insn, 0b0011'1111'1000'0000'0000'0000'0000'0000, 0b0001'0010'1000'0000'0000'0000'0000'0000)) { + const auto hw = (insn >> 21) & 3; + const uint64_t imm = static_cast(extract_bitfield(insn, 16, 5)) << (hw * 16); + *offset = static_cast((insn >> 30) & 1 ? imm : ~imm); if (sf) *sf = (insn >> 31) & 0x1; if (rd) *rd = insn & 0x1f; return true; @@ -409,9 +345,10 @@ namespace patterns { template constexpr size_t parse(const char* p, size_t len, Emit&& emit) { return detail::parse_bytes(p, len, - [&](size_t n, const char* c) { - if (*c == '?') emit(n, uint8_t(0), uint8_t(0)); - else emit(n, value(c), uint8_t(0xFF)); + [&](size_t n, char hi, char lo) { + // A lone ? (or ??) is a whole byte; a ? next to a hex digit is a nibble ("4?", "?8") + if (hi == '?' && lo != '?' && !detail::is_hex_digit(lo)) lo = '?'; + emit(n, uint8_t(nibble(hi) << 4 | nibble(lo)), uint8_t((hi == '?' ? 0 : 0xF0) | (lo == '?' ? 0 : 0x0F))); }, [&](size_t n, const char* len_text) { offset_ = static_cast(n); @@ -507,10 +444,11 @@ namespace patterns { return nullptr; } // Credits to EJT for the helper functions here! - constexpr __forceinline uint8_t value(const char* c) const { - if (!detail::is_hex_digit(c[0]) || !detail::is_hex_digit(c[1])) + static constexpr uint8_t nibble(char c) { + if (c == '?') return 0; + if (!detail::is_hex_digit(c)) throw std::logic_error("Invalid hex byte in pattern!"); - return (detail::get_bits(c[0]) << 4 | detail::get_bits(c[1])); + return static_cast(detail::get_bits(c)); } constexpr void handle_options(const char* ptr) { const bool arm64 = arch_ == arch::arm64; @@ -548,9 +486,12 @@ namespace patterns { auto saved_rd = rd; auto curr_insn = insn + 1; if (is_adrp) { - // Follow the instructions after ADRP, never reading past the end of the scanned buffer - while (reinterpret_cast(curr_insn + 1) <= end && *curr_insn) { + // Follow the instructions after ADRP, never reading past the end of the scanned buffer. + // Instructions that leave the register alone (NOP, another ADRP, unrelated ADD/MOVZ/loads/ + // stores; compilers interleave them) are skipped; anything else ends the chain + for (; reinterpret_cast(curr_insn + 1) <= end && *curr_insn; ++curr_insn) { unsigned rt = 0; + bool post_index = false; if (detail::a64_decode_arithmetic(*curr_insn, &is_sub, &sf, &rd, &rn, &offset)) { // ADD may write a different register (ADRP X8 / ADD X0, X8, #lo); keep following that one if (rn == saved_rd) { @@ -558,29 +499,33 @@ namespace patterns { if (is_sub) saved_offset -= val; else saved_offset += val; saved_rd = rd; - ++curr_insn; continue; } - // Placing this here for sneaky programs that want to put nops in between ADRL - } else if (detail::a64_decode_nop(*curr_insn)) { - ++curr_insn; - continue; - } else if (detail::a64_decode_ldr(*curr_insn, &rn, &rt, &offset) - || detail::a64_decode_ldrh(*curr_insn, &rn, &rt, &offset) - || detail::a64_decode_str(*curr_insn, &rn, &rt, &offset)) { - if (saved_rd == rn) { - saved_offset += (offset < 0 ? -offset : offset); + if (rd == saved_rd) break; + } + else if (detail::a64_decode_load_store(*curr_insn, &rn, &rt, &offset, &post_index)) { + // The address it accesses; [Rn],#imm accesses Rn and only then adds + if (rn == saved_rd) { + if (!post_index) saved_offset += offset; + break; } + if (rt == saved_rd) break; + } + else if (detail::a64_decode_adr(*curr_insn, nullptr, &rd, &offset) + || detail::a64_decode_movz(*curr_insn, nullptr, &rd, &offset)) { + if (rd == saved_rd) break; } - break; + else if (!detail::a64_decode_nop(*curr_insn)) + break; } } const auto from_addr = is_adrp ? reinterpret_cast(insn) & ~0xfff : reinterpret_cast(insn); return reinterpret_cast(from_addr + saved_offset); } - else if (detail::a64_decode_ldr(*insn, nullptr, nullptr, &offset) - || detail::a64_decode_ldrh(*insn, nullptr, nullptr, &offset) - || detail::a64_decode_str(*insn, nullptr, nullptr, &offset)) { + else if (detail::a64_decode_imm19(*insn, &offset)) { + return reinterpret_cast(reinterpret_cast(insn) + offset); + } + else if (detail::a64_decode_load_store(*insn, nullptr, nullptr, &offset)) { if (is_value) *is_value = true; return reinterpret_cast(offset); } @@ -608,17 +553,24 @@ namespace patterns { } return nullptr; } + // memcpy: operands in x86 code are rarely aligned for their type + template + static intptr_t read_signed(const uint8_t* ptr) { + T v; + memcpy(&v, ptr, sizeof v); + return static_cast(v); + } intptr_t relative_value(uint8_t* ptr) const { switch (size_) { case 1: - return static_cast(*reinterpret_cast(ptr)); + return read_signed(ptr); case 2: - return static_cast(*reinterpret_cast(ptr)); + return read_signed(ptr); case 4: - return static_cast(*reinterpret_cast(ptr)); + return read_signed(ptr); #if UINTPTR_MAX > 0xFFFFFFFF case 8: - return static_cast(*reinterpret_cast(ptr)); + return read_signed(ptr); #endif } // Should never hit here diff --git a/plugins/ida/pattern_scanner.cpp b/plugins/ida/pattern_scanner.cpp index 061d546..bce340f 100644 --- a/plugins/ida/pattern_scanner.cpp +++ b/plugins/ida/pattern_scanner.cpp @@ -101,20 +101,24 @@ namespace { ti.text = text; if (ask_form(form, &ti, &kind, &opts) != ASKBTN_YES) return; text = ti.text; - if (kind == pscan::arm64_pattern) { - const qstring unknown = pscan::unsupported_lines(text); - if (!unknown.empty() && ask_yn(ASKBTN_NO, "HIDECANCEL\nPatternScanner doesn't support these instructions, " - "so they would match any instruction:\n\n%s\nSearch anyway?", unknown.c_str()) != ASKBTN_YES) - return; - } + // Parsed before listing unsupported lines: both throw on a line too long, and an exception escaping + // into IDA would take it down + std::optional parsed; try { - last = pscan::parse(text, static_cast(kind)); + parsed = pscan::parse(text, static_cast(kind)); } catch (const std::logic_error& e) { last.reset(); warning("PatternScanner: %s", e.what()); return; } + if (kind == pscan::arm64_pattern) { + const qstring unknown = pscan::unsupported_lines(text); + if (!unknown.empty() && ask_yn(ASKBTN_NO, "HIDECANCEL\nPatternScanner doesn't support these instructions, " + "so they would match any instruction:\n\n%s\nSearch anyway?", unknown.c_str()) != ASKBTN_YES) + return; + } + last = std::move(parsed); const auto& p = pscan::base(*last); if (!p.length()) { last.reset(); diff --git a/tests/test_arm.cpp b/tests/test_arm.cpp index 66d5c26..7e12f6e 100644 --- a/tests/test_arm.cpp +++ b/tests/test_arm.cpp @@ -146,7 +146,21 @@ X("PACIASP", 0xD503233F) \ X("PACIBSP", 0xD503237F) \ X("AUTIASP", 0xD50323BF) \ - X("AUTIBSP", 0xD50323FF) + X("AUTIBSP", 0xD50323FF) \ + X("ORR X0, X1, X2, LSL #3", 0xAA020C20) \ + X("TST X0, X1, LSL #2", 0xEA01081F) \ + X("ORR X0, X1, X2, ROR #5", 0xAAC21420) \ + X("ADD X0, X1, #0x1000", 0x91400420) \ + X("SUB SP, SP, #0x1000", 0xD14007FF) \ + X("ADD X0, X1, #-8", 0xD1002020) \ + X("CMP X0, #-1", 0xB100041F) \ + X("ADD X0, SP, X1", 0x8B2163E0) \ + X("ADD SP, SP, X1", 0x8B2163FF) \ + X("ADD X0, SP, X1, LSL #2", 0x8B216BE0) \ + X("ADD W0, WSP, W1", 0x0B2143E0) \ + X("LDP X29, X30, [SP]", 0xA9407BFD) \ + X("STP X29, X30, [SP]", 0xA9007BFD) \ + X("ADR X0, #-1", 0x70FFFFE0) // Wildcard pattern, an instruction, its encoding, and whether the pattern must match it #define WILDCARD(X) \ @@ -267,7 +281,16 @@ X("B.? #?", "B.GT #0x20", 0x5400010C, 1) \ X("B.? #?", "B #0x10", 0x14000004, 0) \ X("B.? ?", "CBZ W0, #0x10", 0x34000080, 0) \ - X("B.HS ?", "B.CS #0x100", 0x54000802, 1) + X("B.HS ?", "B.CS #0x100", 0x54000802, 1) \ + X("EOR X0, X1, X2, LSR #?", "EOR X0, X1, X2, LSR #3", 0xCA420C20, 1) \ + X("SUB SP, SP, #?", "SUB SP, SP, #1, LSL #12", 0xD14007FF, 1) \ + X("SUB SP, SP, #?", "SUB SP, SP, #0x20", 0xD10083FF, 1) \ + X("CMP X0, #?", "CMP X0, #1, LSL #12", 0xF140041F, 1) \ + X("MOV X0, #?", "MOVN X0, #5", 0x928000A0, 1) \ + X("MOV X0, #?", "MOVK X0, #5", 0xF28000A0, 0) \ + X("LDP X29, X30, [SP]", "LDP X29, X30, [SP],#0x20", 0xA8C27BFD, 0) \ + X("ADD ?, X1, #8", "ADD W0, W1, #8", 0x11002020, 0) \ + X("STP ?, X30, [SP,#-0x10]!", "STP W29, W30, [SP,#-8]!", 0x29BF7BFD, 0) struct Word { uint32_t pat, mask; }; template static Word word(const P& p, size_t idx = 0) { @@ -421,6 +444,44 @@ static void deref_synthetic() { CHECK(movz.resolve(movz.match(bytes, sizeof chain), bytes + sizeof chain, &is_value) == reinterpret_cast(0x7777) && is_value); const auto ldr = ARM_PATTERN("ADD X1, X0, #0x10\nX LDR X2, [X1,#?]\n/d"); CHECK(ldr.resolve(ldr.match(bytes, sizeof chain), bytes + sizeof chain, &is_value) == reinterpret_cast(8) && is_value); + + // MOVZ/MOVN shifts, and /d values of loads/stores the old decoders missed (LDUR, LDRB, big STR offsets) + static const uint32_t vals[] = { + 0xD2A00020, // MOVZ X0, #1, LSL #16 + 0xD2C00020, // MOVZ X0, #1, LSL #32 + 0x92800000, // MOVN X0, #0 (MOV X0, #-1) + 0xF85F83A0, // LDUR X0, [X29,#-8] + 0x39448C20, // LDRB W0, [X1,#0x123] + 0xF9200020, // STR X0, [X1,#0x4000] + }; + const int64_t want[] = { 0x10000, 0x100000000, -1, -8, 0x123, 0x4000 }; + for (size_t k = 0; k < 6; ++k) { + const patterns::RuntimeArmPattern p("X ?\n/d"); + const auto at = reinterpret_cast(&vals[k]); + CHECK(p.resolve(at, at + 4, &is_value) == reinterpret_cast(want[k]) && is_value); + } + // Branches and literal loads with imm19 resolve to their target + static const uint32_t imm19[] = { 0x54000100, 0xB4000100, 0x58000100 }; // B.EQ, CBZ X0, LDR X0 (literal): #0x20 + for (auto& w : imm19) { + const auto at = reinterpret_cast(&w); + CHECK(patterns::RuntimeArmPattern("X ?\n/d").resolve(at, at + 4) == at + 0x20); + } + + // ADRP X8, #0x5000 followed by: LDRB, LDRSW, pre-index with a negative offset, post-index, and an unrelated + // ADRP between ADRP and its ADD + struct { uint32_t next[2]; int64_t add; } follow[] = { + { { 0x39448D00, 0 }, 0x123 }, // LDRB W0, [X8,#0x123] + { { 0xB9801100, 0 }, 0x10 }, // LDRSW X0, [X8,#0x10] + { { 0xF85F8D00, 0 }, -8 }, // LDR X0, [X8,#-8]! + { { 0xF8408500, 0 }, 0 }, // LDR X0, [X8],#8 + { { 0x90000009, 0x91004108 }, 0x10 }, // ADRP X9, #0; ADD X8, X8, #0x10 + }; + for (auto& f : follow) { + alignas(4) uint32_t code[4] = { 0x528EEEE9, 0xB0000028, f.next[0], f.next[1] }; // MOVZ W9; ADRP X8, #0x5000 + const auto b = reinterpret_cast(code); + const auto pg = (reinterpret_cast(&code[1]) & ~uintptr_t(0xFFF)) + 0x5000; + CHECK(adrl.find(b, sizeof code) == pg + f.add); + } puts(" /d on synthetic code ok"); } diff --git a/tests/test_patterns.cpp b/tests/test_patterns.cpp index 9cbd2bc..4837b67 100644 --- a/tests/test_patterns.cpp +++ b/tests/test_patterns.cpp @@ -46,6 +46,24 @@ static void edge_cases() { try { "48 8G"_rtpattern; } catch (const std::logic_error&) { threw = true; } CHECK(threw); // invalid hex rejected { const char s[] = "48 8B X4"; patterns::RuntimePattern rp(s, 8); CHECK(rp.length() == 2); } // X at the end, no overrun + // Wildcards: ? and ?? are a whole byte, ? beside a hex digit a nibble + { + const patterns::RuntimePattern w("? ?? 4? ?8 4F"); + const uint8_t pat[] = { 0, 0, 0x40, 0x08, 0x4F }, msk[] = { 0, 0, 0xF0, 0x0F, 0xFF }; + CHECK(w.length() == 5 && !memcmp(w.pattern(), pat, 5) && !memcmp(w.mask(), msk, 5)); + constexpr auto ct = COMPILETIME_PATTERN("? ?? 4? ?8 4F"); + CHECK(ct.length() == 5 && !memcmp(ct.pattern(), pat, 5) && !memcmp(ct.mask(), msk, 5)); + const auto x = XOR_PATTERN("? ?? 4? ?8 4F"); + for (size_t i = 0; i < 5; ++i) CHECK(x[i] == pat[i] && x.unmask(i) == msk[i]); + const uint8_t hit[] = { 0x11, 0x22, 0x4A, 0xB8, 0x4F }, miss[] = { 0x11, 0x22, 0x5A, 0xB8, 0x4F }; + CHECK(w.find(hit, 5) == hit && x.find(hit, 5) == hit); + CHECK(!w.find(miss, 5) && !x.find(miss, 5)); + const char s[] = "48 ?"; patterns::RuntimePattern rp(s, 4); // trailing ?, not terminated + CHECK(rp.length() == 2 && rp.mask()[1] == 0); + threw = false; + try { "4G ?"_rtpattern; } catch (const std::logic_error&) { threw = true; } + CHECK(threw); + } // X offset, /r (relative value) and /d (rip-relative deref) on a call: E8 alignas(8) uint8_t code[16] = { 0x90, 0xE8, 0x10, 0x00, 0x00, 0x00, 0xC3 }; From d4dc2a5b25dd5ab21bd9ca5d56bd20da6a0112d2 Mon Sep 17 00:00:00 2001 From: scizzydo <34661795+scizzydo@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:24:06 -0700 Subject: [PATCH 19/19] test_arm: NOPs inside an ADRP chain Co-Authored-By: Claude Opus 5.5 --- tests/test_arm.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_arm.cpp b/tests/test_arm.cpp index 7e12f6e..82a9916 100644 --- a/tests/test_arm.cpp +++ b/tests/test_arm.cpp @@ -467,14 +467,16 @@ static void deref_synthetic() { CHECK(patterns::RuntimeArmPattern("X ?\n/d").resolve(at, at + 4) == at + 0x20); } - // ADRP X8, #0x5000 followed by: LDRB, LDRSW, pre-index with a negative offset, post-index, and an unrelated - // ADRP between ADRP and its ADD + // ADRP X8, #0x5000 followed by: LDRB, LDRSW, pre-index with a negative offset, post-index, an unrelated + // ADRP between ADRP and its ADD, and NOPs in the chain struct { uint32_t next[2]; int64_t add; } follow[] = { { { 0x39448D00, 0 }, 0x123 }, // LDRB W0, [X8,#0x123] { { 0xB9801100, 0 }, 0x10 }, // LDRSW X0, [X8,#0x10] { { 0xF85F8D00, 0 }, -8 }, // LDR X0, [X8,#-8]! { { 0xF8408500, 0 }, 0 }, // LDR X0, [X8],#8 { { 0x90000009, 0x91004108 }, 0x10 }, // ADRP X9, #0; ADD X8, X8, #0x10 + { { 0xD503201F, 0x91004108 }, 0x10 }, // NOP; ADD X8, X8, #0x10 + { { 0xD503201F, 0x39448D00 }, 0x123 }, // NOP; LDRB W0, [X8,#0x123] }; for (auto& f : follow) { alignas(4) uint32_t code[4] = { 0x528EEEE9, 0xB0000028, f.next[0], f.next[1] }; // MOVZ W9; ADRP X8, #0x5000