treewide: CI, cmake fixes (#132)

Massages the pe-parse build into a format that's more accomodating
for vcpkg, in preparation for imminent packaging.

.github/workflows/ci.yml

@@ -24,7 +24,7 @@ jobs:
         run: |
           mkdir build && cd build
           cmake ..
-          cmake --build . --target format
+          cmake --build . --target peparse_format
           cd .. && git diff --exit-code
 
   pe-parse:
@@ -32,6 +32,7 @@ jobs:
       matrix:
         platform: ["ubuntu-latest", "macos-latest"]
         build-type: ["Debug", "Release"]
+        build-shared: ["0", "1"]
         compiler:
         - { CC: "clang", CXX: "clang++" }
         - { CC: "gcc", CXX: "g++" }
@@ -48,7 +49,10 @@ jobs:
       run: |
         mkdir build
         cd build
-        cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} ..
+        cmake \
+          -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \
+          -DBUILD_SHARED_LIBS=${{ matrix.build-shared }} \
+          ..
         cmake --build .
     - name: test
       run: |
@@ -82,7 +86,9 @@ jobs:
   pe-parse-windows:
     strategy:
       matrix:
+        build-arch: ["x64", "Win32"]
         build-type: ["Debug", "Release"]
+        build-shared: ["0", "1"]
     runs-on: windows-latest
     steps:
     - uses: actions/checkout@v2
@@ -90,11 +96,19 @@ jobs:
       run: |
         mkdir build
         cd build
-        cmake -G "Visual Studio 16 2019" -A x64 ..
+        cmake `
+          -G "Visual Studio 16 2019" `
+          -A ${{ matrix.build-arch }} `
+          -DBUILD_SHARED_LIBS=${{ matrix.build-shared }} `
+          ..
         cmake --build . --config ${{ matrix.build-type }}
+    - name: install
+      run: |
+        cd build
+        cmake --build . --target install
     - name: test
       run: |
-        .\build\dump-pe\${{ matrix.build-type }}\dump-pe.exe .\test\assets\example.exe
+        .\build\bin\dump-pe.exe .\test\assets\example.exe
 
   pepy-windows:
     strategy:

CMakeLists.txt

@@ -1,11 +1,13 @@
 cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
 project(pe-parse)
 
+# NOTE(ww): CMake has bad defaults for install prefixes.
+# Instead of fussing over them, install everything to the build directory by default
+# and let the user set CMAKE_INSTALL_PREFIX explicitly for their own needs.
 if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT)
-  set(CMAKE_INSTALL_PREFIX "/usr" CACHE PATH "Default install directory" FORCE)
+  set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}" CACHE PATH "Default install directory" FORCE)
 endif ()
 
-set(CMAKE_VERBOSE_MAKEFILE True)
 if (NOT CMAKE_BUILD_TYPE)
   set(CMAKE_BUILD_TYPE "RelWithDebInfo")
 endif ()
@@ -13,7 +15,7 @@ endif ()
 include(cmake/compilation_flags.cmake)
 list(APPEND GLOBAL_CXXFLAGS ${DEFAULT_CXX_FLAGS})
 
-option(BUILD_SHARED_LIBS "Build Shared Libraries" OFF)
+option(BUILD_SHARED_LIBS "Build Shared Libraries" ON)
 option(BUILD_COMMAND_LINE_TOOLS "Build Command Line Tools" ON)
 
 if (MSVC)
@@ -26,10 +28,19 @@ if (BUILD_COMMAND_LINE_TOOLS)
   add_subdirectory(dump-pe)
 endif ()
 
-# `format` target.
+# `peparse_format` target.
-file(GLOB_RECURSE PEPARSE_ALL_SOURCES *.cpp *.h)
+file(
+    GLOB_RECURSE
+    PEPARSE_ALL_SOURCES
+        pe-parser-library/*.cpp
+        pe-parser-library/*.h
+        pepy/*.cpp
+        pepy/*.h
+        examples/*.cpp
+        examples/*.h
+)
 add_custom_target(
-  format
+  peparse_format
   COMMAND clang-format -i -style=file ${PEPARSE_ALL_SOURCES}
   WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
   COMMENT "Auto-format the codebase with clang-format"

README.md

@@ -4,7 +4,7 @@ pe-parse
 [![Build Status](https://img.shields.io/github/workflow/status/trailofbits/pe-parse/CI/master)](https://github.com/trailofbits/pe-parse/actions?query=workflow%3ACI)
 [![LGTM Total alerts](https://img.shields.io/lgtm/alerts/g/trailofbits/pe-parse.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/trailofbits/pe-parse/alerts/)
 
-pe-parse is a principled, lightweight parser for windows portable executable files.
+pe-parse is a principled, lightweight parser for Windows portable executable files.
 It was created to assist in compiled program analysis, potentially of programs of unknown origins.
 This means that it should be resistant to malformed or maliciously crafted PE files, and it should
 support questions that analysis software would ask of an executable program container.
@@ -50,6 +50,7 @@ More information about `pepy` can be found in its [README](./pepy/README.md).
 ## Building
 
 ### Generic instructions
+
 ```
 git clone https://github.com/trailofbits/pe-parse.git
 cd pe-parse
@@ -64,52 +65,32 @@ cmake --build .
 cmake --build . --target install
 ```
 
-PE files that have a Resource section with strings for the Type are encoded in UTF-16, but that
+### Windows-specific
-`std::string` expects UTF-8. Some cross-platform solution is desired.
 
-You can let `cmake` choose one it finds in your build environment or you can choose one from the
+VS 2017 and VS 2019 are supported.
-following options yourself and specify it with the `-DUNICODE_LIBRARY` argument when generating the
-project files with `cmake`:
-
-* `icu` (preferred) - "[ICU](http://site.icu-project.org/) is a mature, widely used set of C/C++
-and Java libraries providing Unicode and Globalization support for software applications"
-* `codecvt` - A C++ library header file
-([now deprecated](http://open-std.org/JTC1/SC22/WG21/docs/papers/2017/p0618r0.html)) supported
-by some C++ runtimes
-
-### Notes about Windows
-
-If you are building on Windows with Visual Studio, the generator option can be used to select the
-compiler version and the output architecture:
 
 ```
 # Compile 64-bit binaries with Visual Studio 2017
 cmake -G "Visual Studio 15 2017 Win64" ..
 
-# Compile 32-bit binaries with Visual Studio 2017
+# Or, with VS 2019, use the -A flag for architecture
-cmake -G "Visual Studio 15 2017" ..
+cmake -G "Visual Studio 16 2019" -A Win64 ..
+
+# Pass the build type at build time
+cmake --build . --config Release
 ```
 
-Visual Studio 2015 or higher is required to use codecvt, but you also have the option of using
-[ICU](http://site.icu-project.org/). The easiest way to get started with ICU in Windows is with
-[vcpkg](https://vcpkg.readthedocs.io/): `vcpkg install icu`.
-
-Then, add the `-DCMAKE_TOOLCHAIN_FILE=C:\src\vcpkg\scripts\buildsystems\vcpkg.cmake` argument when
-generating the project files with cmake to add the appropriate library and include directories to
-the project.
-
 ## Using the library
 
 Once the library is installed, linking to it is easy! Add the following lines in your CMake project:
 
 ```
-find_package(peparse REQUIRED)
+find_package(pe-parse REQUIRED)
 
-target_link_libraries(your_target_name ${PEPARSE_LIBRARIES})
+target_link_libraries(your_target_name PRIVATE pe-parse::pe-parser-library)
-target_include_directories(your_target_name PRIVATE ${PEPARSE_INCLUDE_DIRS})
 ```
 
-You can see a full example in the examples/peaddrconv folder.
+You can see a full example in the [examples/peaddrconv](examples/peaddrconv) folder.
 
 ## Authors
 

dump-pe/CMakeLists.txt

@@ -5,4 +5,4 @@ add_executable(${PROJECT_NAME} main.cpp)
 target_link_libraries(${PROJECT_NAME} PRIVATE pe-parser-library)
 target_compile_options(${PROJECT_NAME} PRIVATE ${GLOBAL_CXXFLAGS})
 
-install(TARGETS ${PROJECT_NAME} DESTINATION "bin")
+install(TARGETS ${PROJECT_NAME} RUNTIME DESTINATION "bin")

examples/peaddrconv/CMakeLists.txt

@@ -35,11 +35,10 @@ else ()
   endif ()
 endif ()
 
-find_package(peparse REQUIRED)
+find_package(pe-parse REQUIRED)
 
 add_executable(${PROJECT_NAME} main.cpp)
-target_link_libraries(${PROJECT_NAME} ${PEPARSE_LIBRARIES})
+target_link_libraries(${PROJECT_NAME} pe-parse::pe-parser-library)
-target_include_directories(${PROJECT_NAME} PRIVATE ${PEPARSE_INCLUDE_DIR})
 target_compile_options(${PROJECT_NAME} PRIVATE ${PEADDRCONV_CXXFLAGS})
 
 install(TARGETS ${PROJECT_NAME} DESTINATION "bin")

pe-parser-library/CMakeLists.txt

@@ -7,15 +7,6 @@ file(READ "${PROJECT_SOURCE_DIR}/../VERSION" PEPARSE_VERSION)
 string(STRIP "${PEPARSE_VERSION}" PEPARSE_VERSION)
 add_compile_definitions(PEPARSE_VERSION="${PEPARSE_VERSION}")
 
-set(UNICODE_LIBRARY "any" CACHE STRING "Select a unicode library")
-set_property(CACHE UNICODE_LIBRARY PROPERTY STRINGS "any" "icu" "codecvt")
-
-# This variable is used twice so setting once at the top here to prevent
-# the chance they get out of sync.
-# This is the minimum "required" version but there's a good chance early
-# versions of ICU support the simple functionality needed by this project.
-set(ICU_MINIMUM_REQUIRED 55.0)
-
 # List all files explicitly; this will make IDEs happy (i.e. QtCreator, CLion, ...)
 list(APPEND PEPARSERLIB_SOURCEFILES
   include/parser-library/parse.h
@@ -26,69 +17,47 @@ list(APPEND PEPARSERLIB_SOURCEFILES
   src/parse.cpp
 )
 
-# Check for codecvt support. Likely the proper way to do this would be to
+# NOTE(ww): On Windows we use the Win32 API's built-in UTF16 conversion
-# use CMake system inspection via methods like "try_compile" to determine
+# routines; on other platforms we use codecvt. codecvt is nominally deprecated
-# if the "#include <codecvt>" directive compiles successfully.
+# in C++17 and onwards, but will probably be available for quite some time.
-if(CXX_STANDARD GREATER_EQUAL 17)
+# Previous versions of pe-parse used ICU when available, but this caused
-  set(CODECVT_SUPPORTED OFF)
+# DLL hell on Windows and wasn't worth the additional dependency.
-else()
 if(MSVC)
-    if (MSVC_VERSION LESS 1900)
-      set(CODECVT_SUPPORTED OFF)
-    else ()
-      set(CODECVT_SUPPORTED ON)
-    endif ()
-  elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
-    if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.0")
-      set(CODECVT_SUPPORTED OFF)
-    else ()
-      set(CODECVT_SUPPORTED ON)
-    endif ()
-  else ()
-    find_path(CODECVT_INCLUDE_DIR NAMES "codecvt")
-    if (CODECVT_INCLUDE_DIR)
-      set(CODECVT_SUPPORTED OFF)
-    else ()
-      set(CODECVT_SUPPORTED ON)
-    endif ()
-  endif ()
-endif()
-
-if(${UNICODE_LIBRARY} MATCHES "icu")
-  find_package(ICU ${ICU_MINIMUM_REQUIRED} COMPONENTS uc REQUIRED)
-  add_definitions(-DUSE_ICU4C)
-  list(APPEND PEPARSERLIB_SOURCEFILES src/unicode_icu.cpp)
-elseif(${UNICODE_LIBRARY} MATCHES "codecvt")
-  if(NOT CODECVT_SUPPORTED)
-    message(SEND_ERROR "codecvt header not found")
-  endif()
-  list(APPEND PEPARSERLIB_SOURCEFILES src/unicode_codecvt.cpp)
-else()
-  find_package(ICU ${ICU_MINIMUM_REQUIRED} COMPONENTS uc)
-  if(ICU_FOUND)
-    add_definitions(-DUSE_ICU4C)
-    list(APPEND PEPARSERLIB_SOURCEFILES src/unicode_icu.cpp)
-  elseif(MSVC)
-    add_definitions(-DUSE_STRINGAPISET)
   list(APPEND PEPARSERLIB_SOURCEFILES src/unicode_winapi.cpp)
-  elseif(CODECVT_SUPPORTED)
-    list(APPEND PEPARSERLIB_SOURCEFILES src/unicode_codecvt.cpp)
 else()
-    message(SEND_ERROR "unable to find codecvt header or ICU library (hint: try installing libicu-dev)")
+  list(APPEND PEPARSERLIB_SOURCEFILES src/unicode_codecvt.cpp)
-  endif(ICU_FOUND)
 endif()
 
 add_library(${PROJECT_NAME} ${PEPARSERLIB_SOURCEFILES})
-target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include)
-target_compile_options(${PROJECT_NAME} PRIVATE ${GLOBAL_CXXFLAGS})
-if(ICU_FOUND)
-  target_link_libraries(${PROJECT_NAME} ICU::uc)
-endif()
 
-install(TARGETS ${PROJECT_NAME}
+target_include_directories(
-  RUNTIME DESTINATION "bin"
+  ${PROJECT_NAME}
-  LIBRARY DESTINATION "lib"
+  PUBLIC
-  ARCHIVE DESTINATION "lib"
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  $<INSTALL_INTERFACE:include>
+)
+target_compile_options(${PROJECT_NAME} PRIVATE ${GLOBAL_CXXFLAGS})
+
+install(
+  TARGETS ${PROJECT_NAME}
+  EXPORT pe-parse-config
+  RUNTIME
+    DESTINATION "bin"
+  LIBRARY
+    DESTINATION "lib"
+  ARCHIVE
+    DESTINATION "lib"
+)
+export(
+  TARGETS ${PROJECT_NAME}
+  NAMESPACE pe-parse::
+  FILE "${CMAKE_CURRENT_BINARY_DIR}/pe-parse-config.cmake"
+)
+install(
+  EXPORT
+  pe-parse-config
+  DESTINATION "lib/cmake/pe-parse"
+  NAMESPACE pe-parse::
+  EXPORT_LINK_INTERFACE_LIBRARIES
 )
-install(FILES "cmake/peparse-config.cmake" DESTINATION "lib/cmake/peparse")
 install(DIRECTORY "include/parser-library" DESTINATION "include")

pe-parser-library/cmake/pe-parse-config.cmake

@@ -0,0 +1,5 @@
+find_path(PEPARSE_INCLUDE_DIR "parser-library/parse.h")
+find_library(PEPARSE_LIBRARIES NAMES "libpe-parser-library" "pe-parser-library")
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(pe-parse DEFAULT_MSG PEPARSE_INCLUDE_DIR PEPARSE_LIBRARIES)

pe-parser-library/cmake/peparse-config.cmake

@@ -1,9 +0,0 @@
-if(CMAKE_CROSSCOMPILING)
-  find_path(PEPARSE_INCLUDE_DIR "parser-library/parse.h")
-else()
-  find_path(PEPARSE_INCLUDE_DIR $<SHELL_PATH:"parser-library/parse.h">)
-endif()
-find_library(PEPARSE_LIBRARIES NAMES "libpe-parser-library" "pe-parser-library")
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(peparse DEFAULT_MSG PEPARSE_INCLUDE_DIR PEPARSE_LIBRARIES)

pe-parser-library/include/parser-library/to_string.h

@@ -3,10 +3,7 @@
 #include <sstream>
 #include <string>
 
-#if defined(USE_ICU4C)
+#if defined(_MSC_VER)
-#include <unicode/unistr.h>
-typedef std::basic_string<UChar> UCharString;
-#elif defined(USE_STRINGAPISET)
 typedef std::basic_string<wchar_t> UCharString;
 #else
 typedef std::u16string UCharString;

pe-parser-library/src/unicode_codecvt.cpp

@@ -30,19 +30,7 @@ namespace peparse {
 // See
 // https://stackoverflow.com/questions/38688417/utf-conversion-functions-in-c11
 std::string from_utf16(const UCharString &u) {
-#if defined(_MSC_VER)
-  // std::wstring_convert<std::codecvt_utf8_utf16<char16_t>, char16_t>convert;
-  // // Doesn't compile with Visual Studio. See
-  // https://stackoverflow.com/questions/32055357/visual-studio-c-2015-stdcodecvt-with-char16-t-or-char32-t
-  std::wstring_convert<std::codecvt_utf8<std::int16_t>, std::int16_t> convert;
-  auto p = reinterpret_cast<const std::int16_t *>(u.data());
-  return convert.to_bytes(p, p + u.size());
-#else
-  // -std=c++11 or -std=c++14
-  // Requires GCC 5 or higher
-  // Requires Clang ??? or higher (tested on Clang 3.8, 5.0, 6.0)
   std::wstring_convert<std::codecvt_utf8<char16_t>, char16_t> convert;
   return convert.to_bytes(u);
-#endif
 }
 } // namespace peparse

pe-parser-library/src/unicode_icu.cpp

@@ -1,35 +0,0 @@
-/*
-The MIT License (MIT)
-
-Copyright (c) 2019 Trail of Bits, Inc.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.
-*/
-
-#include <parser-library/to_string.h>
-#include <unicode/unistr.h>
-
-namespace peparse {
-std::string from_utf16(const UCharString &u) {
-  icu::UnicodeString utf16_string = icu::UnicodeString(u.data(), u.length());
-  std::string result;
-  utf16_string.toUTF8String(result);
-  return result;
-}
-} // namespace peparse

setup.py

@@ -31,6 +31,7 @@ import platform
 here = os.path.dirname(__file__)
 pepy = os.path.join(here, "pepy")
 
+
 with open(os.path.join(pepy, "README.md")) as f:
     README = f.read()
 
@@ -41,28 +42,39 @@ SOURCE_FILES = [
     os.path.join(pepy, "pepy.cpp"),
     os.path.join(here, "pe-parser-library", "src", "parse.cpp"),
     os.path.join(here, "pe-parser-library", "src", "buffer.cpp"),
-    os.path.join(here, "pe-parser-library", "src", "unicode_codecvt.cpp"),
 ]
 
+INCLUDE_DIRS = []
+LIBRARY_DIRS = []
+
 if platform.system() == "Windows":
-    INCLUDE_DIRS = [
+    SOURCE_FILES.append(
+        os.path.join(here, "pe-parser-library", "src", "unicode_winapi.cpp")
+    )
+    INCLUDE_DIRS += [
         os.path.abspath(os.path.join(os.path.dirname(sys.executable), "include")),
         os.path.join(here, "pe-parser-library", "include"),
         "C:\\usr\\include",
     ]
-    LIBRARY_DIRS = [
+    LIBRARY_DIRS += [
         os.path.abspath(os.path.join(os.path.dirname(sys.executable), "libs")),
         "C:\\usr\\lib",
     ]
-    COMPILE_ARGS = ["/EHsc", f'/D"PEPARSE_VERSION=\\"{VERSION}\\""']
+    COMPILE_ARGS = [
+        "/EHsc",
+        f'/D"PEPARSE_VERSION=\\"{VERSION}\\""',
+    ]
 else:
-    INCLUDE_DIRS = [
+    SOURCE_FILES.append(
+        os.path.join(here, "pe-parser-library", "src", "unicode_codecvt.cpp")
+    )
+    INCLUDE_DIRS += [
         "/usr/local/include",
         "/opt/local/include",
         "/usr/include",
         os.path.join(here, "pe-parser-library", "include"),
     ]
-    LIBRARY_DIRS = ["/usr/lib", "/usr/local/lib"]
+    LIBRARY_DIRS += ["/usr/lib", "/usr/local/lib"]
     COMPILE_ARGS = ["-std=c++11", f'-DPEPARSE_VERSION="{VERSION}"']
 
 extension_mod = Extension(

util/release

@@ -17,9 +17,7 @@ function die {
 }
 
 # Fail early if we don't have the expected tools.
-for tool in git python3 twine; do
+installed git || die "Missing dependency: git"
-  installed "${tool}" || die "Missing dependency: ${tool}"
-done
 
 # Fail early if `git status` reports any untracked changes.
 [[ -n $(git status -s) ]] && die "Untracked changes in repo"