From 4bfa5814811e3bbcef6caf2311892922e32ccb6d Mon Sep 17 00:00:00 2001
From: mykola2312 <49044616+mykola2312@users.noreply.github.com>
Date: Fri, 16 Aug 2024 09:10:28 +0300
Subject: [PATCH] begin working on rex prefix testing

---
 include/rtdisasm.h       |  7 +++++
 src/rtdisasm.c           | 57 +++++++++++++++++++++++++++++++++++++++-
 src/rtdisasm_test_data.s |  2 ++
 3 files changed, 65 insertions(+), 1 deletion(-)

diff --git a/include/rtdisasm.h b/include/rtdisasm.h
index 56dfe0b..3644268 100644
--- a/include/rtdisasm.h
+++ b/include/rtdisasm.h
@@ -3,4 +3,11 @@
 
 #include <stdint.h>
 
+// code should point to place with machine instructions, and size
+// limits the area of analyze, so no segfaults would be triggered on
+// page boundaries.
+// returns -1 when no instruction was found, 0 when size limit reached
+// and non-negative-non-zero number of actual instruction size
+int rtdisasm_analyze_single(const uint8_t* code, uint8_t size);
+
 #endif
\ No newline at end of file
diff --git a/src/rtdisasm.c b/src/rtdisasm.c
index e9f5e91..1bf1b0d 100644
--- a/src/rtdisasm.c
+++ b/src/rtdisasm.c
@@ -1,4 +1,5 @@
 #include "rtdisasm.h"
+#include "rtdisasm_table.h"
 
 // prefix definitions. must be declared with macro in order
 // to be readable later in prefix table
@@ -41,4 +42,58 @@ static int is_std_prefix(const uint8_t prefix)
         if (prefix == std_prefixes[i]) return 1;
     
     return 0;
-}
\ No newline at end of file
+}
+
+#define REX_SIG         0b01000000
+#define REX_MASK        0b11110000
+#define REX_VALUE_MASK  0b00001111
+#define REX_B_VALUE     (1<<0)
+#define REX_X_VALUE     (1<<1)
+#define REX_R_VALUE     (1<<2)
+#define REX_W_VALUE     (1<<3)
+
+// returns -1 if not rex, and non-negative is REX_* define
+static int test_rex_prefix(const uint8_t rex)
+{
+    if (rex & REX_MASK != REX_SIG) return -1;
+    
+    const uint8_t rex_value = rex & REX_VALUE_MASK;
+    switch (rex_value)
+    {
+        case REX_B_VALUE: return REX_B;
+        case REX_X_VALUE: return REX_X;
+        case REX_R_VALUE: return REX_R;
+        case REX_W_VALUE: return REX_W;
+    }
+
+    return -1;
+}
+
+// returns -1 if reached end, 0 if no instruction found,
+// and non-zero is actual instruction length 
+static int match_instruction(const uint8_t* cur, const uint8_t* const end, instruction_t* ins)
+{
+    // test if its rex prefix, if so we will look specifically for
+    // instructions with rex prefix
+    int rex = test_rex_prefix(*cur);
+    if (rex != -1)
+    {
+        // it's rex, so advance 1 byte
+        if (++cur == end) return -1;
+    }
+}
+
+int rtdisasm_analyze_single(const uint8_t* code, uint8_t size)
+{
+    const uint8_t* cur = code;
+    const uint8_t* const end = code + size;
+    if (cur == end) return 0;
+
+    // skip standard prefixes
+    while (is_std_prefix(*cur))
+    {
+        if (++cur == end) return 0;
+    }
+
+    // now we need to taste instructions
+}
diff --git a/src/rtdisasm_test_data.s b/src/rtdisasm_test_data.s
index 566d437..d07ea70 100644
--- a/src/rtdisasm_test_data.s
+++ b/src/rtdisasm_test_data.s
@@ -1,8 +1,10 @@
 .section .rodata
 .globl test_1
+.globl test_1_end
 
 test_1:
     lock
     mov %gs:(%rbx), %rax
 
     nop # target that rtdisasm must reach
+test_1_end:
\ No newline at end of file