diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0146434..0d25599 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: run: | mkdir build && cd build cmake .. - cmake --build . --target format + cmake --build . --target peparse_format cd .. && git diff --exit-code pe-parse: @@ -32,6 +32,7 @@ jobs: matrix: platform: ["ubuntu-latest", "macos-latest"] build-type: ["Debug", "Release"] + build-shared: ["0", "1"] compiler: - { CC: "clang", CXX: "clang++" } - { CC: "gcc", CXX: "g++" } @@ -48,7 +49,10 @@ jobs: run: | mkdir build cd build - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} .. + cmake \ + -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} \ + -DBUILD_SHARED_LIBS=${{ matrix.build-shared }} \ + .. cmake --build . - name: test run: | @@ -82,7 +86,9 @@ jobs: pe-parse-windows: strategy: matrix: + build-arch: ["x64", "Win32"] build-type: ["Debug", "Release"] + build-shared: ["0", "1"] runs-on: windows-latest steps: - uses: actions/checkout@v2 @@ -90,11 +96,19 @@ jobs: run: | mkdir build cd build - cmake -G "Visual Studio 16 2019" -A x64 .. + cmake ` + -G "Visual Studio 16 2019" ` + -A ${{ matrix.build-arch }} ` + -DBUILD_SHARED_LIBS=${{ matrix.build-shared }} ` + .. cmake --build . --config ${{ matrix.build-type }} + - name: install + run: | + cd build + cmake --build . --target install - name: test run: | - .\build\dump-pe\${{ matrix.build-type }}\dump-pe.exe .\test\assets\example.exe + .\build\bin\dump-pe.exe .\test\assets\example.exe pepy-windows: strategy: diff --git a/CMakeLists.txt b/CMakeLists.txt index da840a0..da1ede4 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,11 +1,13 @@ cmake_minimum_required(VERSION 3.12 FATAL_ERROR) project(pe-parse) +# NOTE(ww): CMake has bad defaults for install prefixes. +# Instead of fussing over them, install everything to the build directory by default +# and let the user set CMAKE_INSTALL_PREFIX explicitly for their own needs. if (CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) - set(CMAKE_INSTALL_PREFIX "/usr" CACHE PATH "Default install directory" FORCE) + set(CMAKE_INSTALL_PREFIX "${CMAKE_BINARY_DIR}" CACHE PATH "Default install directory" FORCE) endif () -set(CMAKE_VERBOSE_MAKEFILE True) if (NOT CMAKE_BUILD_TYPE) set(CMAKE_BUILD_TYPE "RelWithDebInfo") endif () @@ -13,7 +15,7 @@ endif () include(cmake/compilation_flags.cmake) list(APPEND GLOBAL_CXXFLAGS ${DEFAULT_CXX_FLAGS}) -option(BUILD_SHARED_LIBS "Build Shared Libraries" OFF) +option(BUILD_SHARED_LIBS "Build Shared Libraries" ON) option(BUILD_COMMAND_LINE_TOOLS "Build Command Line Tools" ON) if (MSVC) @@ -26,10 +28,19 @@ if (BUILD_COMMAND_LINE_TOOLS) add_subdirectory(dump-pe) endif () -# `format` target. -file(GLOB_RECURSE PEPARSE_ALL_SOURCES *.cpp *.h) +# `peparse_format` target. +file( + GLOB_RECURSE + PEPARSE_ALL_SOURCES + pe-parser-library/*.cpp + pe-parser-library/*.h + pepy/*.cpp + pepy/*.h + examples/*.cpp + examples/*.h +) add_custom_target( - format + peparse_format COMMAND clang-format -i -style=file ${PEPARSE_ALL_SOURCES} WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" COMMENT "Auto-format the codebase with clang-format" diff --git a/README.md b/README.md index fbf2a80..94479e3 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ pe-parse [![Build Status](https://img.shields.io/github/workflow/status/trailofbits/pe-parse/CI/master)](https://github.com/trailofbits/pe-parse/actions?query=workflow%3ACI) [![LGTM Total alerts](https://img.shields.io/lgtm/alerts/g/trailofbits/pe-parse.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/trailofbits/pe-parse/alerts/) -pe-parse is a principled, lightweight parser for windows portable executable files. +pe-parse is a principled, lightweight parser for Windows portable executable files. It was created to assist in compiled program analysis, potentially of programs of unknown origins. This means that it should be resistant to malformed or maliciously crafted PE files, and it should support questions that analysis software would ask of an executable program container. @@ -50,6 +50,7 @@ More information about `pepy` can be found in its [README](./pepy/README.md). ## Building ### Generic instructions + ``` git clone https://github.com/trailofbits/pe-parse.git cd pe-parse @@ -64,52 +65,32 @@ cmake --build . cmake --build . --target install ``` -PE files that have a Resource section with strings for the Type are encoded in UTF-16, but that -`std::string` expects UTF-8. Some cross-platform solution is desired. +### Windows-specific -You can let `cmake` choose one it finds in your build environment or you can choose one from the -following options yourself and specify it with the `-DUNICODE_LIBRARY` argument when generating the -project files with `cmake`: - -* `icu` (preferred) - "[ICU](http://site.icu-project.org/) is a mature, widely used set of C/C++ -and Java libraries providing Unicode and Globalization support for software applications" -* `codecvt` - A C++ library header file -([now deprecated](http://open-std.org/JTC1/SC22/WG21/docs/papers/2017/p0618r0.html)) supported -by some C++ runtimes - -### Notes about Windows - -If you are building on Windows with Visual Studio, the generator option can be used to select the -compiler version and the output architecture: +VS 2017 and VS 2019 are supported. ``` # Compile 64-bit binaries with Visual Studio 2017 cmake -G "Visual Studio 15 2017 Win64" .. -# Compile 32-bit binaries with Visual Studio 2017 -cmake -G "Visual Studio 15 2017" .. +# Or, with VS 2019, use the -A flag for architecture +cmake -G "Visual Studio 16 2019" -A Win64 .. + +# Pass the build type at build time +cmake --build . --config Release ``` -Visual Studio 2015 or higher is required to use codecvt, but you also have the option of using -[ICU](http://site.icu-project.org/). The easiest way to get started with ICU in Windows is with -[vcpkg](https://vcpkg.readthedocs.io/): `vcpkg install icu`. - -Then, add the `-DCMAKE_TOOLCHAIN_FILE=C:\src\vcpkg\scripts\buildsystems\vcpkg.cmake` argument when -generating the project files with cmake to add the appropriate library and include directories to -the project. - ## Using the library Once the library is installed, linking to it is easy! Add the following lines in your CMake project: ``` -find_package(peparse REQUIRED) +find_package(pe-parse REQUIRED) -target_link_libraries(your_target_name ${PEPARSE_LIBRARIES}) -target_include_directories(your_target_name PRIVATE ${PEPARSE_INCLUDE_DIRS}) +target_link_libraries(your_target_name PRIVATE pe-parse::pe-parser-library) ``` -You can see a full example in the examples/peaddrconv folder. +You can see a full example in the [examples/peaddrconv](examples/peaddrconv) folder. ## Authors diff --git a/dump-pe/CMakeLists.txt b/dump-pe/CMakeLists.txt index 65bba70..22e413c 100644 --- a/dump-pe/CMakeLists.txt +++ b/dump-pe/CMakeLists.txt @@ -5,4 +5,4 @@ add_executable(${PROJECT_NAME} main.cpp) target_link_libraries(${PROJECT_NAME} PRIVATE pe-parser-library) target_compile_options(${PROJECT_NAME} PRIVATE ${GLOBAL_CXXFLAGS}) -install(TARGETS ${PROJECT_NAME} DESTINATION "bin") +install(TARGETS ${PROJECT_NAME} RUNTIME DESTINATION "bin") diff --git a/examples/peaddrconv/CMakeLists.txt b/examples/peaddrconv/CMakeLists.txt index 741e6d8..d8e6b64 100644 --- a/examples/peaddrconv/CMakeLists.txt +++ b/examples/peaddrconv/CMakeLists.txt @@ -35,11 +35,10 @@ else () endif () endif () -find_package(peparse REQUIRED) +find_package(pe-parse REQUIRED) add_executable(${PROJECT_NAME} main.cpp) -target_link_libraries(${PROJECT_NAME} ${PEPARSE_LIBRARIES}) -target_include_directories(${PROJECT_NAME} PRIVATE ${PEPARSE_INCLUDE_DIR}) +target_link_libraries(${PROJECT_NAME} pe-parse::pe-parser-library) target_compile_options(${PROJECT_NAME} PRIVATE ${PEADDRCONV_CXXFLAGS}) install(TARGETS ${PROJECT_NAME} DESTINATION "bin") diff --git a/pe-parser-library/CMakeLists.txt b/pe-parser-library/CMakeLists.txt index 2bcba55..5a0a4db 100644 --- a/pe-parser-library/CMakeLists.txt +++ b/pe-parser-library/CMakeLists.txt @@ -7,15 +7,6 @@ file(READ "${PROJECT_SOURCE_DIR}/../VERSION" PEPARSE_VERSION) string(STRIP "${PEPARSE_VERSION}" PEPARSE_VERSION) add_compile_definitions(PEPARSE_VERSION="${PEPARSE_VERSION}") -set(UNICODE_LIBRARY "any" CACHE STRING "Select a unicode library") -set_property(CACHE UNICODE_LIBRARY PROPERTY STRINGS "any" "icu" "codecvt") - -# This variable is used twice so setting once at the top here to prevent -# the chance they get out of sync. -# This is the minimum "required" version but there's a good chance early -# versions of ICU support the simple functionality needed by this project. -set(ICU_MINIMUM_REQUIRED 55.0) - # List all files explicitly; this will make IDEs happy (i.e. QtCreator, CLion, ...) list(APPEND PEPARSERLIB_SOURCEFILES include/parser-library/parse.h @@ -26,69 +17,47 @@ list(APPEND PEPARSERLIB_SOURCEFILES src/parse.cpp ) -# Check for codecvt support. Likely the proper way to do this would be to -# use CMake system inspection via methods like "try_compile" to determine -# if the "#include " directive compiles successfully. -if(CXX_STANDARD GREATER_EQUAL 17) - set(CODECVT_SUPPORTED OFF) +# NOTE(ww): On Windows we use the Win32 API's built-in UTF16 conversion +# routines; on other platforms we use codecvt. codecvt is nominally deprecated +# in C++17 and onwards, but will probably be available for quite some time. +# Previous versions of pe-parse used ICU when available, but this caused +# DLL hell on Windows and wasn't worth the additional dependency. +if(MSVC) + list(APPEND PEPARSERLIB_SOURCEFILES src/unicode_winapi.cpp) else() - if (MSVC) - if (MSVC_VERSION LESS 1900) - set(CODECVT_SUPPORTED OFF) - else () - set(CODECVT_SUPPORTED ON) - endif () - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS "5.0") - set(CODECVT_SUPPORTED OFF) - else () - set(CODECVT_SUPPORTED ON) - endif () - else () - find_path(CODECVT_INCLUDE_DIR NAMES "codecvt") - if (CODECVT_INCLUDE_DIR) - set(CODECVT_SUPPORTED OFF) - else () - set(CODECVT_SUPPORTED ON) - endif () - endif () -endif() - -if(${UNICODE_LIBRARY} MATCHES "icu") - find_package(ICU ${ICU_MINIMUM_REQUIRED} COMPONENTS uc REQUIRED) - add_definitions(-DUSE_ICU4C) - list(APPEND PEPARSERLIB_SOURCEFILES src/unicode_icu.cpp) -elseif(${UNICODE_LIBRARY} MATCHES "codecvt") - if(NOT CODECVT_SUPPORTED) - message(SEND_ERROR "codecvt header not found") - endif() list(APPEND PEPARSERLIB_SOURCEFILES src/unicode_codecvt.cpp) -else() - find_package(ICU ${ICU_MINIMUM_REQUIRED} COMPONENTS uc) - if(ICU_FOUND) - add_definitions(-DUSE_ICU4C) - list(APPEND PEPARSERLIB_SOURCEFILES src/unicode_icu.cpp) - elseif(MSVC) - add_definitions(-DUSE_STRINGAPISET) - list(APPEND PEPARSERLIB_SOURCEFILES src/unicode_winapi.cpp) - elseif(CODECVT_SUPPORTED) - list(APPEND PEPARSERLIB_SOURCEFILES src/unicode_codecvt.cpp) - else() - message(SEND_ERROR "unable to find codecvt header or ICU library (hint: try installing libicu-dev)") - endif(ICU_FOUND) endif() add_library(${PROJECT_NAME} ${PEPARSERLIB_SOURCEFILES}) -target_include_directories(${PROJECT_NAME} PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include) -target_compile_options(${PROJECT_NAME} PRIVATE ${GLOBAL_CXXFLAGS}) -if(ICU_FOUND) - target_link_libraries(${PROJECT_NAME} ICU::uc) -endif() -install(TARGETS ${PROJECT_NAME} - RUNTIME DESTINATION "bin" - LIBRARY DESTINATION "lib" - ARCHIVE DESTINATION "lib" +target_include_directories( + ${PROJECT_NAME} + PUBLIC + $ + $ +) +target_compile_options(${PROJECT_NAME} PRIVATE ${GLOBAL_CXXFLAGS}) + +install( + TARGETS ${PROJECT_NAME} + EXPORT pe-parse-config + RUNTIME + DESTINATION "bin" + LIBRARY + DESTINATION "lib" + ARCHIVE + DESTINATION "lib" +) +export( + TARGETS ${PROJECT_NAME} + NAMESPACE pe-parse:: + FILE "${CMAKE_CURRENT_BINARY_DIR}/pe-parse-config.cmake" +) +install( + EXPORT + pe-parse-config + DESTINATION "lib/cmake/pe-parse" + NAMESPACE pe-parse:: + EXPORT_LINK_INTERFACE_LIBRARIES ) -install(FILES "cmake/peparse-config.cmake" DESTINATION "lib/cmake/peparse") install(DIRECTORY "include/parser-library" DESTINATION "include") diff --git a/pe-parser-library/cmake/pe-parse-config.cmake b/pe-parser-library/cmake/pe-parse-config.cmake new file mode 100644 index 0000000..242d6c8 --- /dev/null +++ b/pe-parser-library/cmake/pe-parse-config.cmake @@ -0,0 +1,5 @@ +find_path(PEPARSE_INCLUDE_DIR "parser-library/parse.h") +find_library(PEPARSE_LIBRARIES NAMES "libpe-parser-library" "pe-parser-library") + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(pe-parse DEFAULT_MSG PEPARSE_INCLUDE_DIR PEPARSE_LIBRARIES) diff --git a/pe-parser-library/cmake/peparse-config.cmake b/pe-parser-library/cmake/peparse-config.cmake deleted file mode 100644 index 38a2280..0000000 --- a/pe-parser-library/cmake/peparse-config.cmake +++ /dev/null @@ -1,9 +0,0 @@ -if(CMAKE_CROSSCOMPILING) - find_path(PEPARSE_INCLUDE_DIR "parser-library/parse.h") -else() - find_path(PEPARSE_INCLUDE_DIR $) -endif() -find_library(PEPARSE_LIBRARIES NAMES "libpe-parser-library" "pe-parser-library") - -include(FindPackageHandleStandardArgs) -find_package_handle_standard_args(peparse DEFAULT_MSG PEPARSE_INCLUDE_DIR PEPARSE_LIBRARIES) diff --git a/pe-parser-library/include/parser-library/to_string.h b/pe-parser-library/include/parser-library/to_string.h index 23e16fb..92933b0 100644 --- a/pe-parser-library/include/parser-library/to_string.h +++ b/pe-parser-library/include/parser-library/to_string.h @@ -3,10 +3,7 @@ #include #include -#if defined(USE_ICU4C) -#include -typedef std::basic_string UCharString; -#elif defined(USE_STRINGAPISET) +#if defined(_MSC_VER) typedef std::basic_string UCharString; #else typedef std::u16string UCharString; diff --git a/pe-parser-library/src/unicode_codecvt.cpp b/pe-parser-library/src/unicode_codecvt.cpp index f297408..b702ddd 100644 --- a/pe-parser-library/src/unicode_codecvt.cpp +++ b/pe-parser-library/src/unicode_codecvt.cpp @@ -30,19 +30,7 @@ namespace peparse { // See // https://stackoverflow.com/questions/38688417/utf-conversion-functions-in-c11 std::string from_utf16(const UCharString &u) { -#if defined(_MSC_VER) - // std::wstring_convert, char16_t>convert; - // // Doesn't compile with Visual Studio. See - // https://stackoverflow.com/questions/32055357/visual-studio-c-2015-stdcodecvt-with-char16-t-or-char32-t - std::wstring_convert, std::int16_t> convert; - auto p = reinterpret_cast(u.data()); - return convert.to_bytes(p, p + u.size()); -#else - // -std=c++11 or -std=c++14 - // Requires GCC 5 or higher - // Requires Clang ??? or higher (tested on Clang 3.8, 5.0, 6.0) std::wstring_convert, char16_t> convert; return convert.to_bytes(u); -#endif } } // namespace peparse diff --git a/pe-parser-library/src/unicode_icu.cpp b/pe-parser-library/src/unicode_icu.cpp deleted file mode 100644 index ef1bd9e..0000000 --- a/pe-parser-library/src/unicode_icu.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/* -The MIT License (MIT) - -Copyright (c) 2019 Trail of Bits, Inc. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. -*/ - -#include -#include - -namespace peparse { -std::string from_utf16(const UCharString &u) { - icu::UnicodeString utf16_string = icu::UnicodeString(u.data(), u.length()); - std::string result; - utf16_string.toUTF8String(result); - return result; -} -} // namespace peparse diff --git a/setup.py b/setup.py index 11a4607..039488e 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,7 @@ import platform here = os.path.dirname(__file__) pepy = os.path.join(here, "pepy") + with open(os.path.join(pepy, "README.md")) as f: README = f.read() @@ -41,28 +42,39 @@ SOURCE_FILES = [ os.path.join(pepy, "pepy.cpp"), os.path.join(here, "pe-parser-library", "src", "parse.cpp"), os.path.join(here, "pe-parser-library", "src", "buffer.cpp"), - os.path.join(here, "pe-parser-library", "src", "unicode_codecvt.cpp"), ] +INCLUDE_DIRS = [] +LIBRARY_DIRS = [] + if platform.system() == "Windows": - INCLUDE_DIRS = [ + SOURCE_FILES.append( + os.path.join(here, "pe-parser-library", "src", "unicode_winapi.cpp") + ) + INCLUDE_DIRS += [ os.path.abspath(os.path.join(os.path.dirname(sys.executable), "include")), os.path.join(here, "pe-parser-library", "include"), "C:\\usr\\include", ] - LIBRARY_DIRS = [ + LIBRARY_DIRS += [ os.path.abspath(os.path.join(os.path.dirname(sys.executable), "libs")), "C:\\usr\\lib", ] - COMPILE_ARGS = ["/EHsc", f'/D"PEPARSE_VERSION=\\"{VERSION}\\""'] + COMPILE_ARGS = [ + "/EHsc", + f'/D"PEPARSE_VERSION=\\"{VERSION}\\""', + ] else: - INCLUDE_DIRS = [ + SOURCE_FILES.append( + os.path.join(here, "pe-parser-library", "src", "unicode_codecvt.cpp") + ) + INCLUDE_DIRS += [ "/usr/local/include", "/opt/local/include", "/usr/include", os.path.join(here, "pe-parser-library", "include"), ] - LIBRARY_DIRS = ["/usr/lib", "/usr/local/lib"] + LIBRARY_DIRS += ["/usr/lib", "/usr/local/lib"] COMPILE_ARGS = ["-std=c++11", f'-DPEPARSE_VERSION="{VERSION}"'] extension_mod = Extension( diff --git a/util/release b/util/release index 68a6cce..7dd7952 100755 --- a/util/release +++ b/util/release @@ -17,9 +17,7 @@ function die { } # Fail early if we don't have the expected tools. -for tool in git python3 twine; do - installed "${tool}" || die "Missing dependency: ${tool}" -done +installed git || die "Missing dependency: git" # Fail early if `git status` reports any untracked changes. [[ -n $(git status -s) ]] && die "Untracked changes in repo"