Windows fixes to get mcsema-disass working (lifting-bits#226)

* Windows fixes to get mcsema-disass working

* Install protobuf, protoc, and python-protobuf from source
* Quote IDA path to handle paths with spaces and shell=True
* Set SystemRoot in env so IDAPython can still run
* Add entries to windows.txt to handle a HelloWorld release build

* Simplify protobuf finding for win32 and not win32

* Fix linux

Author: artemdinaburg

CMakeLists.txt

@@ -13,44 +13,43 @@ if(WIN32)
   SET(CMAKE_EXE_LINKER_FLAGS "/LARGEADDRESSAWARE ${CMAKE_EXE_LINKER_FLAGS}")
 endif(WIN32)
 
-set(CMAKE_MODULE_PATH "${LLVM_DIR}/cmake/modules ${CMAKE_MODULE_PATH}")
+set(CMAKE_MODULE_PATH "${LLVM_DIR}/cmake/modules" "${CMAKE_SOURCE_DIR}/cmake" "${CMAKE_ROOT}/Modules")
 
 # Configure with the build LLVM
 find_package(LLVM 3.8 REQUIRED CONFIG)
 add_definitions(${LLVM_DEFINITIONS})
 
-set (CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake" ${CMAKE_MODULE_PATH})
 include (LLVMUtils)
 include (GetGitRevisionDescription)
 
 # we need C++11
 if(NOT WIN32)
   # WIN32 defaults to CXX11 by default, and setting it explicitly here
   # confuses clang-cl, which will bail out with "unknown flag -std=g++11"
+
+  #FindPackage for protobuf is terrible for win32
+  # it assumes MSVC built protobuf from a visual studio project
+  #... we build with custom cmake, so it will never be found
+  #... but it should work for other OSes where we install from a package
+
   set(CMAKE_CXX_STANDARD 11)
-endif()
 
-# Get protobufs.
-if(WIN32)
-  # Change the slashes to Unix style or risk the wrath of invalid escape
-  # sequences!
-  string(REGEX REPLACE "\\\\" "/" CMAKE_SOURCE_DIR ${CMAKE_SOURCE_DIR})
+  if(APPLE)
+      set(PROTOBUF_ROOT "${CMAKE_SOURCE_DIR}/third_party/protobuf")
+      set(Protobuf_LIBRARIES "${PROTOBUF_ROOT}/build/lib")
+      set(Protobuf_INCLUDE_DIR "${PROTOBUF_ROOT}/src")
+  endif()
 
-  set(PROTOBUF_ROOT "${CMAKE_SOURCE_DIR}/third_party/protobuf")
-  set(Protobuf_LIBRARIES "${PROTOBUF_ROOT}/build/libprotobuf/Release")
-  set(Protobuf_INCLUDE_DIR "${PROTOBUF_ROOT}/src")
-  
-  include_directories(${Protobuf_INCLUDE_DIR})
-  link_directories(${Protobuf_LIBRARIES})
-elseif(APPLE)
-  set(PROTOBUF_ROOT "${CMAKE_SOURCE_DIR}/third_party/protobuf")
-  set(Protobuf_LIBRARIES "${PROTOBUF_ROOT}/build/lib")
-  set(Protobuf_INCLUDE_DIR "${PROTOBUF_ROOT}/src")
-  
-  include_directories(${Protobuf_INCLUDE_DIR})
-  link_directories(${Protobuf_LIBRARIES})
-endif(WIN32)
-find_package(Protobuf REQUIRED)
+  find_package(Protobuf REQUIRED)
+else()
+   string(REGEX REPLACE "\\\\" "/" CMAKE_SOURCE_DIR ${CMAKE_SOURCE_DIR})
+   # on win32, manuallys set protobuf goodness
+   set(PROTOBUF_INCLUDE_DIRS "${CMAKE_SOURCE_DIR}/include")
+   set(PROTOBUF_LIBRARIES "${CMAKE_SOURCE_DIR}/lib/protobuf.lib")
+   set(PROTOBUF_PROTOC_EXECUTABLE "${CMAKE_SOURCE_DIR}/bin/protoc.exe")
+endif()
+
+include_directories(${PROTOBUF_INCLUDE_DIRS})
 
 include_directories(${CMAKE_SOURCE_DIR})
 include_directories(${CMAKE_SOURCE_DIR}/third_party)
@@ -168,9 +167,6 @@ include_directories("${CMAKE_BINARY_DIR}")
 # this will make sure that the protobuf files are generated each time
 # the CFG.h header file is modified
 #
-# todo: use find protobuf CMake module to generate the files; this is
-# not easy on windows right now, since protoc is built on the fly.
-#
 
 set(PROTOBUF_CONFIGURATION "${CMAKE_SOURCE_DIR}/mcsema/CFG/CFG.proto")
 
@@ -180,16 +176,10 @@ set(PROTOBUF_OUTPUT_FILE_LIST
     "${MCSEMA_GEN_DIR}/CFG_pb2.py"
 )
 
-if (WIN32)
-    set(PROTOC_EXECUTABLE_PATH "${CMAKE_SOURCE_DIR}/third_party/protobuf/build/protoc/Release/protoc.exe")
-else ()
-    set(PROTOC_EXECUTABLE_PATH "protoc")
-endif()
-
 add_custom_command(
     OUTPUT ${PROTOBUF_OUTPUT_FILE_LIST}
 
-    COMMAND "${PROTOC_EXECUTABLE_PATH}" --cpp_out "${MCSEMA_GEN_DIR}" --python_out "${MCSEMA_GEN_DIR}" --proto_path "${CMAKE_SOURCE_DIR}/mcsema/CFG" "${PROTOBUF_CONFIGURATION}"
+    COMMAND "${PROTOBUF_PROTOC_EXECUTABLE}" --cpp_out "${MCSEMA_GEN_DIR}" --python_out "${MCSEMA_GEN_DIR}" --proto_path "${CMAKE_SOURCE_DIR}/mcsema/CFG" "${PROTOBUF_CONFIGURATION}"
 
     DEPENDS "${CMAKE_SOURCE_DIR}/mcsema/CFG/CFG.h"
     MAIN_DEPENDENCY "${PROTOBUF_CONFIGURATION}"
@@ -236,7 +226,7 @@ add_executable(mcsema-lift
   ${CMAKE_SOURCE_DIR}/mcsema/Arch/X86/Semantics/SUB.cpp)
 
 target_link_libraries(mcsema-lift
-  protobuf
+  ${PROTOBUF_LIBRARIES}
   LLVMBitReader
   LLVMBitWriter
   LLVMMCDisassembler

bootstrap.bat

@@ -21,7 +21,15 @@ if defined ProgramFiles(x86) (
 )
 set "PATH=%PATH%;%ProgramFiles%\7-zip"
 
+REM add third-party installs to our path
+set "PATH=%PATH%;%MCSEMA_DIR%\bin"
+
 REM sanity checks for installed software
+where cl >NUL 2>NUL
+if not %ERRORLEVEL% == 0 (
+    echo "[!] Visual Studio (cl.exe) is not found. Please run from a Visual Studio build prompt"
+    exit /B 1
+)
 where 7z >NUL 2>NUL
 if not %ERRORLEVEL% == 0 (
     echo [+] The 7z command is not found. Attempting to install
@@ -39,6 +47,19 @@ if not %ERRORLEVEL% == 0 (
     exit /B 1
 )
 
+where python >NUL 2>NUL
+if not %ERRORLEVEL% == 0 (
+    echo "[!] The python command is not found. Please install python (or add it to your PATH)"
+    exit /B 1
+)
+
+python -V 2>&1 | findstr /R /C:"Python 2.7" > NUL
+if not %ERRORLEVEL% == 0 (
+    echo [!] Detected python != 2.7
+    echo [!] Please install Python 2.7 and make sure it appears first in your system path
+    exit /B 1
+)
+
 REM This check is *SEPARATE* from the 3.8 check below for good reason
 REM Clang needs to be "installed" via the installer to it is available
 REM as a visual studio toolset. Mcsema uses it for the assembler and
@@ -134,13 +155,22 @@ REM the normal VS2013 toolset
 cmake.exe ^
   -G "%VSBUILD%" ^
   -DPROTOBUF_ROOT="%PROTO_DIR%" ^
+  -DCMAKE_INSTALL_PREFIX="%MCSEMA_DIR%" ^
   %MCSEMA_DIR%\cmake\protobuf
-cmake --build . --config Release
+cmake --build . --config Release --target install
 popd
+
+echo "[+] Installing protobuf for python"
+pushd python
+python setup.py build
+python setup.py install
 popd
 
 popd
 
+popd
+
+
 echo [+] Download and extract LLVM
 pushd third_party
 if exist llvm goto compile_llvm
@@ -190,5 +220,8 @@ cmake --build . --config Release --target install -- /maxcpucount:%NUMBER_OF_PRO
 
 popd
 
+echo "[+] Installing mcsema-disass"
+python %MCSEMA_DIR%\tools\setup.py install --user --install-scripts %MCSEMA_DIR%\bin
+
 popd
 

tools/mcsema_disass/__main__.py

@@ -102,28 +102,21 @@ def main(args=None):
       ret = ida.disass.execute(args, fixed_command_args)
 
       # in case IDA somehow says success, but no output was generated
-      if not os.path.isfile(args.output):
+      if not os.path.exists(args.output) or not os.path.isfile(args.output) :
           sys.stderr.write("Could not generate a CFG. Try using the --log_file option to see an error log.\n")
           ret = 1
 
       # The disassembler script probably threw an exception
-      if 0 == os.path.getsize(args.output):
+      elif 0 == os.path.getsize(args.output):
           sys.stderr.write("Generated an invalid (zero-sized) CFG. Please use the --log_file option to see an error log.\n")
           # remove the zero-sized file
           os.unlink(args.output)
           ret = 1
 
-      # in case IDA somehow says success, but no output was generated
-      if not os.path.isfile(args.output):
-          sys.stderr.write("Could not generate a CFG. Try using the --log_file option to see an error log.\n")
-          ret = 1
+      else:
+          # assume it all went well
+          pass
 
-      # The disassembler script probably threw an exception
-      if 0 == os.path.getsize(args.output):
-          sys.stderr.write("Generated an invalid (zero-sized) CFG. Please use the --log_file option to see an error log.\n")
-          # remove the zero-sized file
-          os.unlink(args.output)
-          ret = 1
 
     else:
       arg_parser.error("{} passed to --disassembler is not known.".format(

tools/mcsema_disass/defs/windows.txt

@@ -16,6 +16,11 @@ commode 0 C N
 fmode 0 C N
 
 #processing C runtime
+configthreadlocale 1 C N
+crtSetUnhandledExceptionFilter 1 C N
+XcptFilter 1 C N
+invoke_watson 5 C N
+
 getmainargs 5 C N
 __getmainargs 5 C N
 __wgetmainargs 5 C N

tools/mcsema_disass/ida/disass.py

@@ -25,6 +25,8 @@ def execute(args, command_args):
   env["HOME"] = os.path.expanduser('~')
   env["IDA_PATH"] = os.path.dirname(args.disassembler)
   env["PYTHONPATH"] = os.path.dirname(ida_dir)
+  if "SystemRoot" in os.environ:
+      env["SystemRoot"] = os.environ["SystemRoot"]
 
   script_cmd = []
   script_cmd.append(ida_get_cfg_path)
@@ -41,7 +43,7 @@ def execute(args, command_args):
   script_cmd.extend(command_args)  # Extra, script-specific arguments.
 
   cmd = []
-  cmd.append(args.disassembler)  # Path to IDA.
+  cmd.append(r'"{}"'.format(args.disassembler))  # Path to IDA.
   cmd.append("-B")  # Batch mode.
   cmd.append("-S\"{}\"".format(" ".join(script_cmd)))
   cmd.append(args.binary)