Draft [0.0.24] Protobuf analyzer

.github/workflows/build.yml

@@ -25,7 +25,8 @@ jobs:
             llvm_tag: "llvmorg-19.1.4",
             python_version: "3.10",
             os_version: "2022",
-            artifact_id: "RG3_Windows"
+            artifact_id: "RG3_Windows",
+            protobuf_tag: "v29.1"
           }
           - {
             name: "Ubuntu Linux",
@@ -38,7 +39,8 @@ jobs:
             llvm_tag: "llvmorg-19.1.4",
             python_version: "3.10",
             os_version: "24.04",
-            artifact_id: "RG3_Linux"
+            artifact_id: "RG3_Linux",
+            protobuf_tag: "v29.1"
           }
           - {
             name: "macOS",
@@ -51,7 +53,8 @@ jobs:
             llvm_tag: "llvmorg-19.1.4",
             python_version: "3.10",
             os_version: "13",
-            artifact_id: "RG3_macOS"
+            artifact_id: "RG3_macOS",
+            protobuf_tag: "v29.1"
           }
 
     steps:
@@ -118,7 +121,7 @@ jobs:
           tar xzf boost_1_81_0.tar.gz
           cd boost_1_81_0
           .\bootstrap.bat --with-libraries=system,filesystem,python
-          .\b2 link=static variant=release
+          .\b2 link=static runtime-link=static variant=release
 
       - name: Download and build Boost (macOS)
         if: steps.cache-boost-binaries.outputs.cache-hit != 'true' && matrix.config.os_id == 'macos'
@@ -166,7 +169,7 @@ jobs:
           CXX: ${{ matrix.config.cxx }}
         run: |
           mkdir build
-          cmake -DLLVM_ENABLE_PROJECTS="clang" -DCMAKE_BUILD_TYPE="MinSizeRel" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DLLVM_ENABLE_ZLIB=OFF -DLLVM_ENABLE_ZSTD=OFF -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_BENCHMARKS=OFF -DLLVM_INCLUDE_EXAMPLES=OFF -DLLVM_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_TOOLS=ON -DLLVM_TARGETS_TO_BUILD="host" -DLLVM_BUILD_32_BITS=OFF -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_BUILD_DOCS=OFF -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -S llvm -B build -G "${{ matrix.config.cmake_generator }}"
+          cmake -DLLVM_ENABLE_PROJECTS="clang" -DCMAKE_BUILD_TYPE="MinSizeRel" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DLLVM_ENABLE_ZLIB=OFF -DLLVM_ENABLE_ZSTD=OFF -DLLVM_ENABLE_RTTI=ON -DLLVM_INCLUDE_BENCHMARKS=OFF -DLLVM_INCLUDE_EXAMPLES=OFF -DLLVM_INCLUDE_TESTS=OFF -DLLVM_INCLUDE_TOOLS=ON -DLLVM_TARGETS_TO_BUILD="host" -DLLVM_BUILD_32_BITS=OFF -DLLVM_ENABLE_BINDINGS=OFF -DLLVM_BUILD_DOCS=OFF -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -S llvm -B build -G "${{ matrix.config.cmake_generator }}"
           cd build
           cmake --build . --config MinSizeRel
 
@@ -186,18 +189,42 @@ jobs:
             llvm_repo/llvm
             llvm_repo/clang
 
+      - name: Checkout Protobuf
+        uses: actions/checkout@v3
+        with:
+          submodules: "recursive"
+          repository: "protocolbuffers/protobuf"
+          ref: ${{ matrix.config.protobuf_tag }}
+          path: "protobuf_repo"
+
+      # Protobuf (will use cache later)
+      - name: Build Protobuf
+        working-directory: protobuf_repo
+        env:
+          CC: ${{ matrix.config.cc }}
+          CXX: ${{ matrix.config.cxx }}
+        run: |
+          mkdir build
+          mkdir build/install
+          cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DBUILD_SHARED_LIBS=OFF -DABSL_PROPAGATE_CXX_STD=ON -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=build/install -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -B build -S . -G "${{ matrix.config.cmake_generator }}"
+          cmake --build build --config Release
+          cmake --install build
+
       # Build our project
       - name: Build RG3
         env:
           LLVM_DIR: ${{ github.workspace }}/llvm_repo/build/lib/cmake/llvm
           CLANG_DIR: ${{ github.workspace }}/llvm_repo/build/lib/cmake/clang
           Boost_ROOT: ${{ github.workspace }}/boost_1_81_0/stage/lib/cmake/Boost-1.81.0
+          absl_DIR: ${{ github.workspace }}/protobuf_repo/build/install/lib/cmake/absl
+          utf8_range_DIR: ${{ github.workspace }}/protobuf_repo/build/install/lib/cmake/utf8_range
+          Protobuf_DIR: ${{ github.workspace }}/protobuf_repo/build/install/lib/cmake/protobuf
           Python3_USE_STATIC_LIBS: "TRUE"
           CC: ${{ matrix.config.cc }}
           CXX: ${{ matrix.config.cxx }}
         run: |
           mkdir build
-          cmake -DCMAKE_BUILD_TYPE="MinSizeRel" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -B build -G "${{ matrix.config.cmake_generator }}"
+          cmake -DCMAKE_BUILD_TYPE="MinSizeRel" -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64" -B build -G "${{ matrix.config.cmake_generator }}"
           cd build
           cmake --build . --config MinSizeRel
 

CMakeLists.txt

@@ -10,9 +10,10 @@ endif()
 
 add_subdirectory(ThirdParty/fmt)
 
-add_subdirectory(Cpp)
-add_subdirectory(LLVM)
-add_subdirectory(PyBind)
+add_subdirectory(Cpp) # Base entities & logic
+add_subdirectory(LLVM) # LLVM driver & AST parsing
+add_subdirectory(Protobuf) # Protobuf driver & analyzer
+add_subdirectory(PyBind) # Python bindings
 
 # Unit tests
 add_subdirectory(ThirdParty/googletest)

LLVM/CMakeLists.txt

@@ -2,6 +2,11 @@ project(RG3_LLVM)
 
 # ------- Boost
 set(Boost_USE_STATIC_LIBS ON)
+if (MSVC)
+    # Should use this on Windows to enable /MT for boost too
+    set(Boost_USE_STATIC_RUNTIME ON)
+endif()
+
 find_package(Boost COMPONENTS filesystem REQUIRED HINTS $ENV{Boost_ROOT})
 
 # ------- LLVM BACKEND

LLVM/source/CompilerInstanceFactory.cpp

@@ -9,6 +9,8 @@
 #include <clang/Frontend/CompilerInvocation.h>
 #include <clang/Frontend/FrontendActions.h>
 #include <clang/Lex/PreprocessorOptions.h>
+#include <clang/Lex/PPCallbacks.h>
+#include <clang/Lex/Preprocessor.h>
 #include <clang/Tooling/Tooling.h>
 #include <clang/Basic/LangOptions.h>
 #include <clang/Basic/TargetInfo.h>
@@ -214,6 +216,7 @@ namespace rg3::llvm
 #endif
 
 		pOutInstance->setInvocation(invocation);
+		pOutInstance->createPreprocessor(clang::TranslationUnitKind::TU_Complete);
 
 		// Set up FrontendOptions
 		clang::FrontendOptions &opts = pOutInstance->getFrontendOpts();
@@ -285,5 +288,6 @@ namespace rg3::llvm
 		assert(pOutInstance->hasDiagnostics() && "Diagnostics not set up!");
 		assert(pOutInstance->hasTarget() && "Target not set up!");
 		assert(pOutInstance->hasFileManager() && "FileManager not set up!");
+		assert(pOutInstance->hasPreprocessor() && "Preprocessor not set up!");
 	}
 }

Protobuf/CMakeLists.txt

@@ -0,0 +1,48 @@
+project(RG3_LLVM)
+
+# ------- Boost
+set(Boost_USE_STATIC_LIBS ON)
+if (MSVC)
+    # Should use this on Windows to enable /MT for boost too
+    set(Boost_USE_STATIC_RUNTIME ON)
+endif()
+
+find_package(Boost COMPONENTS filesystem REQUIRED HINTS $ENV{Boost_ROOT})
+
+# ------- Protobuf
+# Required to have Protobuf_DIR, absl_DIR and utf8_range_DIR
+
+find_package(absl REQUIRED HINTS $ENV{absl_DIR})
+find_package(utf8_range REQUIRED HINTS $ENV{utf8_range_DIR})
+find_package(Protobuf REQUIRED HINTS $ENV{Protobuf_DIR})
+
+message(STATUS "Found Protobuf ${Protobuf_VERSION} (${Protobuf_DIR})")
+
+# ------- RG3 Protobuf
+file(GLOB_RECURSE RG3_PROTOBUF_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/source/*.cpp")
+add_library(RG3_Protobuf STATIC ${RG3_PROTOBUF_SOURCES})
+add_library(RG3::Protobuf ALIAS RG3_Protobuf)
+target_include_directories(RG3_Protobuf PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include")
+target_include_directories(RG3_Protobuf PUBLIC ${Protobuf_INCLUDE_DIRS})
+target_include_directories(RG3_Protobuf PUBLIC ${absl_INCLUDE_DIRS})
+target_include_directories(RG3_Protobuf PUBLIC ${utf8_range_INCLUDE_DIRS})
+
+if (MSVC)
+    # /bigobj support
+    target_compile_options(RG3_LLVM PUBLIC /bigobj)
+endif()
+
+target_link_libraries(RG3_Protobuf PUBLIC
+        # Protobuf
+        protobuf::libprotobuf
+        protobuf::libprotoc
+        protobuf::libupb
+        utf8_range::utf8_range
+        absl::base
+        absl::strings
+        # Boost
+        ${Boost_LIBRARIES}       # RG3
+        RG3::Cpp
+        # FMT
+        fmt::fmt
+)

Protobuf/include/RG3/Protobuf/Compiler.h

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <filesystem>
+#include <vector>
+#include <string>
+
+
+namespace rg3::pb
+{
+	/**
+	 * @brief In most cases used syntax from .proto file, but you able to specify it directly here (in C++/Python code)
+	 */
+	enum class ProtobufSyntax { PB_SYNTAX_2 = 2, PB_SYNTAX_3 = 3 };
+
+	/**
+	 * @brief There are 2 issues in protoc: errors and warnings
+	 */
+	enum class IssueKind { IK_WARNING, IK_ERROR };
+
+	/**
+	 * @brief Most useful parameters for protoc (in-memory version). Unlike LLVM CompilerConfig, that options useful only for protoc
+	 */
+	struct CompilerConfig {
+		ProtobufSyntax eSyntax { ProtobufSyntax::PB_SYNTAX_3 };
+		std::vector<std::filesystem::path> vIncludeDirs {};
+		bool bUseStrictMode { true };
+		bool bEnableGRPC { false };
+		bool bGenerateClientStubs { false };
+		bool bUseLiteGenerator { false };
+	};
+
+	/**
+	 * @brief Describe issue in protoc
+	 */
+	struct CompilerIssue
+	{
+		IssueKind eKind { IssueKind::IK_WARNING };
+		int iLine { 0 };
+		int iColumn { 0 };
+		std::string sMessage {};
+	};
+
+	using CompilerIssuesVector = std::vector<CompilerIssue>;
+}

Protobuf/include/RG3/Protobuf/Enum.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+#include <string>
+#include <vector>
+#include <cstdint>
+
+
+namespace rg3::pb
+{
+	struct EnumEntry
+	{
+		std::string sName {};
+		std::int64_t iValue { 0 };
+	};
+
+	using EnumEntriesVector = std::vector<EnumEntry>;
+
+	class Enum : public boost::noncopyable
+	{
+	 public:
+		using Ptr = boost::shared_ptr<Enum>;
+
+	 public:
+		Enum();
+		Enum(const std::string& sName, const std::string& sPackage, const EnumEntriesVector& aEntries);
+
+		[[nodiscard]] const std::string& getName() const;
+		[[nodiscard]] const std::string& getPackage() const;
+		[[nodiscard]] const EnumEntriesVector& getEntries() const;
+
+	 private:
+		std::string m_sName {};
+		std::string m_sPackage {};
+		EnumEntriesVector m_aEntries {};
+	};
+}

Protobuf/include/RG3/Protobuf/Location.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <string>
+
+
+namespace rg3::pb
+{
+	struct Location
+	{
+		std::string sFile {};
+		int iLine { 0 };
+		int iColumn { 0 };
+	};
+}

Protobuf/include/RG3/Protobuf/Message.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <RG3/Protobuf/Location.h>
+
+#include <boost/noncopyable.hpp>
+#include <boost/shared_ptr.hpp>
+#include <string>
+#include <vector>
+
+
+namespace rg3::pb
+{
+	struct MessageField
+	{
+		std::string sName {};
+		int iIndex { 0 };
+		/// TYPE
+	};
+
+	using MessageFieldVector = std::vector<MessageField>;
+
+	class Message : public boost::noncopyable
+	{
+	 public:
+		using Ptr = boost::shared_ptr<Message>;
+
+	 public:
+		Message();
+		Message(const std::string& sName, const Location& sLocation, const MessageFieldVector& vFields);
+
+	 private:
+		std::string m_sName {};
+		Location m_sLocation {};
+		MessageFieldVector m_aFields {};
+	};
+}

Protobuf/include/RG3/Protobuf/Protobuf.h

@@ -0,0 +1,4 @@
+#pragma once
+
+namespace rg3::pb
+{}

Protobuf/include/RG3/Protobuf/ProtobufAnalyzer.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <RG3/Protobuf/Compiler.h>
+
+#include <boost/noncopyable.hpp>
+
+#include <filesystem>
+#include <variant>
+#include <string>
+
+
+namespace rg3::pb
+{
+	/**
+	 * @brief This class implements a single threaded analyzer of protobuf code.
+	 */
+	class ProtobufAnalyzer final : public boost::noncopyable
+	{
+	 public:
+		using CodeSource = std::variant<std::string, std::filesystem::path>; // string is a code repr in memory (id0.proto), filesystem::path for FS path
+
+		ProtobufAnalyzer();
+
+		void setCode(const std::string& sCode);
+		void setFile(const std::filesystem::path& sPath);
+		void setSource(const CodeSource& src);
+
+		void setCompilerConfig(const CompilerConfig& sConfig);
+		const CompilerConfig& getCompilerConfig() const;
+		CompilerConfig& getCompilerConfig();
+
+		[[nodiscard]] const CompilerIssuesVector& getIssues() const;
+
+		bool analyze();
+
+	 private:
+		CodeSource m_sSource {};
+		CompilerConfig m_sConfig {};
+		CompilerIssuesVector m_aIssues {};
+	};
+}