Reversing Windows DRM: VMProtect Obfuscation and License Validation Bypass

Introduction

Commercial Windows applications employ sophisticated DRM and obfuscation to protect licensing systems. Understanding these protection mechanisms is essential for security research and vulnerability analysis.

VMProtect Analysis

Code Virtualization

VMProtect converts x86/x64 code into custom bytecode executed by a virtual machine:

; Original function (before VMProtect)
CheckLicense:
    push    ebp
    mov     ebp, esp
    sub     esp, 10h
    
    ; Read license file
    call    ReadLicenseFile
    
    ; Validate checksum
    call    ValidateChecksum
    test    eax, eax
    jz      license_invalid
    
    ; Check expiration
    call    CheckExpiration
    test    eax, eax
    jz      license_expired
    
    mov     eax, 1
    jmp     exit
    
license_invalid:
license_expired:
    xor     eax, eax
    
exit:
    mov     esp, ebp
    pop     ebp
    ret

; After VMProtect (virtualized)
CheckLicense:
    push    esi
    push    edi
    push    ebx
    
    ; Jump to VM dispatcher
    call    vm_entry_point
    
    ; VM bytecode (obfuscated, non-standard)
    db      0x4D, 0x3F, 0x7A, 0x21, ...  ; Custom opcodes
    db      0x89, 0xC3, 0x1F, 0x44, ...
    ; ... hundreds of bytes of bytecode
    
    ; VM exit
    pop     ebx
    pop     edi
    pop     esi
    ret

vm_entry_point:
    ; Initialize VM context
    push    all_registers
    lea     esi, [vm_bytecode]
    
vm_dispatch_loop:
    lodsb                    ; Read VM opcode
    movzx   eax, al
    
    ; Dispatch to handler
    call    [vm_handlers + eax*4]
    
    ; Check for VM exit
    test    byte [vm_flags], VM_EXIT_FLAG
    jz      vm_dispatch_loop
    
    pop     all_registers
    ret

Devirtualization Techniques

import idaapi
import ida_bytes

def detect_vm_dispatcher():
    """
    Identify VMProtect dispatcher pattern
    """
    # Pattern: lodsb + movzx + call [table + eax*4]
    pattern = "AC 0F B6 C0 FF 14 85"  # Approximate pattern
    
    ea = ida_search.find_binary(0, BADADDR, pattern, 16, SEARCH_DOWN)
    
    if ea != BADADDR:
        print(f"VM dispatcher found at 0x{ea:X}")
        
        # Extract handler table address
        table_addr = idc.get_operand_value(ea + 7, 1)
        print(f"Handler table at 0x{table_addr:X}")
        
        # Enumerate handlers
        for i in range(256):
            handler = idc.get_wide_dword(table_addr + i*4)
            if handler:
                print(f"Handler {i:02X}: 0x{handler:X}")
                idc.set_name(handler, f"vm_handler_{i:02X}")

def trace_vm_execution():
    """
    Dynamic analysis to trace VM bytecode
    """
    import sys
    sys.path.append(r"C:\Program Files\x64dbg\x64\plugins")
    import x64dbgpy
    
    # Breakpoint on VM dispatcher
    x64dbgpy.SetBreakpoint(vm_dispatcher_addr)
    
    vm_trace = []
    
    def bp_callback():
        # Read current VM opcode
        opcode = x64dbgpy.ReadByte(x64dbgpy.GetReg("ESI"))
        
        # Read VM registers/stack
        vm_state = {
            'opcode': opcode,
            'vm_ip': x64dbgpy.GetReg("ESI"),
            'vm_regs': read_vm_registers()
        }
        
        vm_trace.append(vm_state)
        
        # Continue execution
        return True
    
    x64dbgpy.Run()
    
    # Analyze trace to reconstruct original code
    devirtualized = reconstruct_from_trace(vm_trace)
    return devirtualized

License Validation Systems

Common Patterns

// Typical license validation
class LicenseValidator {
private:
    std::string license_key;
    time_t expiration_date;
    uint32_t hardware_id;
    
    // Obfuscated with VMProtect
    bool ValidateLicenseFormat() {
        // Check format: XXXX-XXXX-XXXX-XXXX
        if (license_key.length() != 19) return false;
        
        // Validate checksum (Reed-Solomon or custom)
        uint8_t checksum = 0;
        for (char c : license_key) {
            if (c != '-') {
                checksum ^= c;
                checksum = (checksum << 1) | (checksum >> 7);
            }
        }
        
        return checksum == expected_checksum;
    }
    
    // Obfuscated with code flow flattening
    bool CheckHardwareLock() {
        uint32_t current_hwid = GetHardwareID();
        
        // XOR obfuscation
        uint32_t stored_hwid = license_data.hw_id ^ 0xDEADBEEF;
        
        return current_hwid == stored_hwid;
    }
    
    uint32_t GetHardwareID() {
        // Combine multiple hardware identifiers
        uint32_t hash = 0x12345678;
        
        // CPU ID
        int cpu_info[4];
        __cpuid(cpu_info, 0);
        hash = crc32_update(hash, cpu_info, sizeof(cpu_info));
        
        // MAC address
        uint8_t mac[6];
        GetAdapterMAC(mac);
        hash = crc32_update(hash, mac, sizeof(mac));
        
        // Volume serial number
        DWORD serial;
        GetVolumeInformation("C:\\", NULL, 0, &serial, NULL, NULL, NULL, 0);
        hash = crc32_update(hash, &serial, sizeof(serial));
        
        return hash;
    }
    
public:
    bool Validate() {
        // Anti-debugging checks
        if (IsDebuggerPresent()) return false;
        if (CheckRemoteDebugger()) return false;
        
        // Timing checks (debugger detection)
        uint64_t start = __rdtsc();
        Sleep(10);
        uint64_t end = __rdtsc();
        if (end - start > 1000000) return false;  // Debugger slows execution
        
        // Validate license
        if (!ValidateLicenseFormat()) return false;
        if (!CheckHardwareLock()) return false;
        if (!CheckExpiration()) return false;
        
        return true;
    }
};

Bypass Techniques

# IDA Python script to patch license checks
import idaapi
import idc

def patch_license_check():
    """
    Patch license validation to always return true
    """
    # Find CheckLicense function
    func_ea = idc.get_name_ea_simple("CheckLicense")
    
    if func_ea == BADADDR:
        # Search for pattern
        func_ea = find_license_validation_function()
    
    # Patch to always return 1
    # mov eax, 1
    # ret
    idc.patch_byte(func_ea, 0xB8)  # mov eax, imm32
    idc.patch_dword(func_ea + 1, 1)
    idc.patch_byte(func_ea + 5, 0xC3)  # ret
    
    print(f"Patched license check at 0x{func_ea:X}")

def find_license_validation_function():
    """
    Heuristic search for license validation
    """
    # Look for strings
    license_strings = [
        "Invalid license",
        "License expired",
        "Hardware mismatch",
        "Trial period ended"
    ]
    
    for string in license_strings:
        str_ea = idc.find_text(0, SEARCH_DOWN, 0, 0, string)
        if str_ea != BADADDR:
            # Find xrefs
            for xref in idautils.XrefsTo(str_ea):
                func = idaapi.get_func(xref.frm)
                if func:
                    print(f"Potential license check: {idc.get_func_name(func.start_ea)}")
                    return func.start_ea
    
    return BADADDR

Anti-Debugging Techniques

Detection Methods

// Anti-debugging techniques
class AntiDebug {
public:
    // Method 1: IsDebuggerPresent API
    static bool Method1() {
        return IsDebuggerPresent();
    }
    
    // Method 2: PEB BeingDebugged flag
    static bool Method2() {
        #ifdef _WIN64
        PPEB peb = (PPEB)__readgsqword(0x60);
        #else
        PPEB peb = (PPEB)__readfsdword(0x30);
        #endif
        
        return peb->BeingDebugged;
    }
    
    // Method 3: NtQueryInformationProcess
    static bool Method3() {
        HANDLE process = GetCurrentProcess();
        DWORD debugPort = 0;
        
        NtQueryInformationProcess(
            process,
            ProcessDebugPort,  // 7
            &debugPort,
            sizeof(debugPort),
            NULL
        );
        
        return debugPort != 0;
    }
    
    // Method 4: Hardware breakpoints (DR registers)
    static bool Method4() {
        CONTEXT ctx = {};
        ctx.ContextFlags = CONTEXT_DEBUG_REGISTERS;
        
        GetThreadContext(GetCurrentThread(), &ctx);
        
        // Check if any DR registers are set
        return (ctx.Dr0 || ctx.Dr1 || ctx.Dr2 || ctx.Dr3);
    }
    
    // Method 5: Timing checks
    static bool Method5() {
        LARGE_INTEGER freq, start, end;
        QueryPerformanceFrequency(&freq);
        
        QueryPerformanceCounter(&start);
        
        // Execute some code
        volatile int x = 0;
        for (int i = 0; i < 100; i++) x++;
        
        QueryPerformanceCounter(&end);
        
        // If took too long → debugger
        double elapsed = (end.QuadPart - start.QuadPart) * 1000.0 / freq.QuadPart;
        return elapsed > 10.0;  // ms
    }
    
    // Method 6: Exception-based detection
    static bool Method6() {
        __try {
            // Trigger single-step exception
            __asm {
                pushfd
                or dword ptr [esp], 0x100  // Set trap flag
                popfd
                nop  // Debugger catches this
            }
        }
        __except(EXCEPTION_EXECUTE_HANDLER) {
            // Normal execution - exception caught
            return false;
        }
        
        // Debugger swallowed exception
        return true;
    }
};

Bypass in x64dbg

// x64dbg script to bypass anti-debug
function bypassAntiDebug() {
    // 1. Patch IsDebuggerPresent
    var isDebuggerPresentAddr = GetProcAddress("kernel32.dll", "IsDebuggerPresent");
    WriteMemory(isDebuggerPresentAddr, "33 C0 C3");  // xor eax,eax; ret
    
    // 2. Clear PEB BeingDebugged flag
    var peb = ReadQWord(GetContextData("gs") + 0x60);
    WriteByte(peb + 0x02, 0);  // BeingDebugged = 0
    
    // 3. Hook NtQueryInformationProcess
    var ntQueryInfoAddr = GetProcAddress("ntdll.dll", "NtQueryInformationProcess");
    setHook(ntQueryInfoAddr, function() {
        // If querying ProcessDebugPort, return 0
        if (GetArg(2) == 7) {  // ProcessDebugPort
            WriteMemory(GetArg(3), "00 00 00 00");
        }
    });
    
    // 4. Clear hardware breakpoints
    SetReg("dr0", 0);
    SetReg("dr1", 0);
    SetReg("dr2", 0);
    SetReg("dr3", 0);
    SetReg("dr6", 0);
    SetReg("dr7", 0);
    
    log("Anti-debug bypassed");
}

bypassAntiDebug();

Code Flow Obfuscation

Control Flow Flattening

// Original code
void ProcessData(uint8_t *data, size_t len) {
    InitializeBuffer();
    
    for (size_t i = 0; i < len; i++) {
        data[i] = Transform(data[i]);
    }
    
    Finalize();
}

// After control flow flattening
void ProcessData_Obfuscated(uint8_t *data, size_t len) {
    int state = 0x1A3F;  // Random initial state
    size_t i = 0;
    
    while (true) {
        switch (state) {
            case 0x1A3F:
                InitializeBuffer();
                state = 0x7B21 ^ 0x1234;
                break;
                
            case 0x6915:
                if (i >= len) {
                    state = 0x4D8A;
                } else {
                    state = 0x92C7;
                }
                break;
                
            case 0x92C7:
                data[i] = Transform(data[i]);
                i++;
                state = 0x6915;
                break;
                
            case 0x4D8A:
                Finalize();
                state = 0xFFFF;  // Exit state
                break;
                
            case 0xFFFF:
                return;
                
            default:
                __ud2();  // Invalid state
        }
    }
}

Deobfuscation

import angr
import claripy

def deobfuscate_control_flow(binary_path, function_addr):
    """
    Use symbolic execution to reconstruct control flow
    """
    project = angr.Project(binary_path, auto_load_libs=False)
    
    # Create initial state
    state = project.factory.blank_state(addr=function_addr)
    
    # Symbolic input
    input_data = claripy.BVS("input", 8 * 100)
    state.memory.store(state.regs.rdi, input_data)
    
    # Simulation manager
    simgr = project.factory.simulation_manager(state)
    
    # Explore all paths
    simgr.explore()
    
    # Extract path constraints
    for path in simgr.deadended:
        if is_success_path(path):
            print(f"Success path found:")
            print(f"  Constraints: {path.solver.constraints}")
            
            # Solve for input
            solution = path.solver.eval(input_data, cast_to=bytes)
            print(f"  Input: {solution.hex()}")

Dynamic Instrumentation

Frida for Windows

// Frida script to hook license validation
function hookLicenseValidation() {
    // Find module
    var module = Process.getModuleByName("protected_app.exe");
    
    // Pattern scan for license check
    var pattern = "55 8B EC 83 EC 10 56 57";
    var results = Memory.scanSync(module.base, module.size, pattern);
    
    if (results.length > 0) {
        var licenseCheck = results[0].address;
        
        Interceptor.attach(licenseCheck, {
            onEnter: function(args) {
                console.log("License check called");
                console.log("  Arg0:", args[0]);
            },
            
            onLeave: function(retval) {
                console.log("License check returned:", retval);
                
                // Force return true
                retval.replace(ptr(1));
                
                console.log("  Patched to:", retval);
            }
        });
        
        console.log("Hooked license validation at", licenseCheck);
    }
}

// Hook GetHardwareID
function hookHWID() {
    var getHWID = Module.findExportByName(null, "GetHardwareID");
    
    if (getHWID) {
        Interceptor.attach(getHWID, {
            onLeave: function(retval) {
                // Return fake HWID
                retval.replace(ptr(0x12345678));
                console.log("Spoofed HWID");
            }
        });
    }
}

hookLicenseValidation();
hookHWID();

Keygen Development

// License key generation
#include <string>
#include <sstream>
#include <iomanip>

class KeyGenerator {
private:
    static const uint32_t MAGIC = 0xDEADBEEF;
    
    static uint8_t calculateChecksum(const std::string &key) {
        uint8_t checksum = 0;
        for (char c : key) {
            if (c != '-') {
                checksum ^= c;
                checksum = (checksum << 1) | (checksum >> 7);
            }
        }
        return checksum;
    }
    
public:
    static std::string Generate(uint32_t hwid, time_t expiration) {
        // Encode data
        uint32_t encoded_hwid = hwid ^ MAGIC;
        uint32_t encoded_exp = static_cast<uint32_t>(expiration) ^ MAGIC;
        
        // Generate 16-character key
        std::stringstream ss;
        ss << std::hex << std::setfill('0');
        ss << std::setw(4) << (encoded_hwid >> 16);
        ss << "-";
        ss << std::setw(4) << (encoded_hwid & 0xFFFF);
        ss << "-";
        ss << std::setw(4) << (encoded_exp >> 16);
        ss << "-";
        ss << std::setw(4) << (encoded_exp & 0xFFFF);
        
        std::string key = ss.str();
        
        // Add checksum
        uint8_t checksum = calculateChecksum(key);
        
        ss << "-";
        ss << std::setw(2) << static_cast<unsigned>(checksum);
        
        return ss.str();
    }
    
    static bool Validate(const std::string &key) {
        // Extract components
        // ... (reverse of Generate)
        
        return true;  // If valid
    }
};

// Usage
int main() {
    uint32_t hwid = 0x12345678;
    time_t expiration = time(NULL) + 365*24*3600;  // 1 year
    
    std::string license = KeyGenerator::Generate(hwid, expiration);
    std::cout << "License key: " << license << std::endl;
    
    return 0;
}

Conclusion

Modern DRM systems employ multiple protection layers:

1. Code virtualization (VMProtect, Themida)
2. Anti-debugging and anti-tampering
3. Hardware-locked licenses
4. Encrypted/obfuscated validation logic

Reverse engineering requires:

• Static analysis (IDA Pro, Ghidra)
• Dynamic analysis (x64dbg, WinDbg)
• Symbolic execution (angr)
• Dynamic instrumentation (Frida)

Understanding these techniques is essential for security research and vulnerability analysis.

References

1. VMProtect Software Protection Manual
2. Eilam, E. (2011). “Reversing: Secrets of Reverse Engineering”
3. Sikorski, M., Honig, A. (2012). “Practical Malware Analysis”
4. Yosifovich, P., et al. (2017). “Windows Internals”