From aa8ab736be4f077cd5bbdb4579943342740c34c4 Mon Sep 17 00:00:00 2001 From: Florian Magin Date: Fri, 21 Nov 2025 12:13:57 +0100 Subject: [PATCH 01/10] Separate the two concerns of fetching function infos/details --- .../services/api/GhidraRevengService.java | 268 +++++++++++++----- .../services/api/types/FunctionDetails.java | 5 + .../plugins/AnalysisManagementPlugin.java | 5 +- 3 files changed, 199 insertions(+), 79 deletions(-) diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java index f47ac64..c5ecf3c 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java @@ -16,11 +16,14 @@ import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import ghidra.app.cmd.function.ApplyFunctionSignatureCmd; +import ghidra.app.cmd.function.SetFunctionNameCmd; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.address.Address; import ghidra.program.model.data.*; import ghidra.program.model.data.Structure; import ghidra.program.model.listing.Function; +import ghidra.program.model.listing.FunctionSignature; import ghidra.program.model.listing.Program; import ghidra.program.model.symbol.*; import ghidra.program.model.util.LongPropertyMap; @@ -47,7 +50,6 @@ import java.util.stream.Collectors; - /** * Implements a Ghidra compatible interface on top of the RevEngAI REST API * The idea is that all other plugin and UI code can simply use this service to interact with the API @@ -93,14 +95,15 @@ public URI getServer() { return this.apiInfo.hostURI(); } - public void registerFinishedAnalysisForProgram(ProgramWithBinaryID programWithBinaryID) throws ApiException { + public void registerFinishedAnalysisForProgram(ProgramWithBinaryID programWithBinaryID, TaskMonitor monitor) throws ApiException { statusCache.put(programWithBinaryID.binaryID(), AnalysisStatus.Complete); // Add the binary to the program before loading the function info. Checking that a function is present requires // the binary ID to be present in the program options. addBinaryIDtoProgramOptions(programWithBinaryID.program(), programWithBinaryID.binaryID()); - loadFunctionInfo(programWithBinaryID.program(), programWithBinaryID.binaryID()); + associateFunctionInfo(programWithBinaryID.program(), programWithBinaryID.binaryID()); + pullFunctionInfoFromAnalysis(programWithBinaryID, monitor); } public void addBinaryIDtoProgramOptions(Program program, BinaryID binID){ @@ -158,13 +161,15 @@ public Optional getBinaryIDfromOptions( return Optional.of(binID); } - /** - * Loads the function info into a dedicated user property map. - * Future task: we can potentially merge the function ID map with the mangled name map and have a single map of objects. - */ - private void loadFunctionInfo(Program program, BinaryID binID) throws ApiException { + /// Loads the function info into a dedicated user property map. + /// This method should only concern itself with associating the FunctionID with the Ghidra Function + /// This property is immutable within an Analysis: The function ID will never change unless an entirely different + /// analysis is associated with the program + /// Other function information like the name and signature should be loaded in [#pullFunctionInfoFromAnalysis(ProgramWithBinaryID,TaskMonitor)] + /// because this information can change on the server, and thus needs a dedicated method to refresh it + private void associateFunctionInfo(Program program, BinaryID binID) throws ApiException { List functionInfo = api.getFunctionInfo(binID); - var transactionID = program.startTransaction("Load Function Info"); + var transactionID = program.startTransaction("Associate Function Info"); // Create the FunctionID map LongPropertyMap functionIDMap; @@ -176,75 +181,38 @@ private void loadFunctionInfo(Program program, BinaryID binID) throws ApiExcepti } // Create the function mangled name map - StringPropertyMap mangledNameMap; try { - mangledNameMap = program.getUsrPropertyManager().createStringPropertyMap(REAI_FUNCTION_MANGLED_MAP); + program.getUsrPropertyManager().createStringPropertyMap(REAI_FUNCTION_MANGLED_MAP); } catch (DuplicateNameException e) { program.endTransaction(transactionID, false); throw new RuntimeException("Previous mangled name property map still exists",e); } LongPropertyMap finalFunctionIDMap = functionIDMap; - StringPropertyMap finalMangledNameMap = mangledNameMap; - - AtomicInteger ghidraRenamedFunctions = new AtomicInteger(); - AtomicInteger ghidraBoundariesMatchedFunction = new AtomicInteger(); - functionInfo.forEach( - info -> { - var oFunc = getFunctionFor(info, program); - if (oFunc.isEmpty()){ - Msg.error(this, "Function not found in Ghidra for info: %s".formatted(info)); - return; - } - var func = oFunc.get(); - // Extract the mangled name from Ghidra - var ghidraMangledName = func.getSymbol().getName(false); - var revEngMangledName = info.functionMangledName(); - var revEngDemangledName = info.functionName(); - - // Skip external and thunk functions because we don't support them - if (func.isExternal() || func.isThunk()){ - Msg.debug(this, "Skipping external/thunk function %s".formatted(ghidraMangledName)); - return; - } - - // Skip invalid function mangled names - if (revEngMangledName.contains(" ") || revEngDemangledName.contains(" ")) { - Msg.warn(this, "Skipping renaming of function %s to invalid name %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); - return; - } - - var funcSize = func.getBody().getNumAddresses(); - - // For unclear reasons the func size is off by one - if (funcSize - 1 != info.functionSize() && funcSize != info.functionSize()){ - Msg.warn(this, "Function size mismatch for function %s: %d vs %d".formatted(ghidraMangledName, funcSize, info.functionSize())); - return; - } - - // Source types: - // DEFAULT: placeholder name automatically assigned by Ghidra when it doesn’t know the real name. - // ANALYSIS: A name/signature inferred by one of Ghidra’s analysis engines (or demangler) rather than simply “default.” - // IMPORTED: Information taken from an external source — symbols or signatures imported from a file or database. - // USER_DEFINED: A name or signature explicitly set by the analyst. - if (func.getSymbol().getSource() == SourceType.DEFAULT && !revEngMangledName.startsWith("FUN_") ){ - Msg.info(this, "Renaming function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); - try { - // Set demangled name from RevEng.AI - func.setName(revEngDemangledName, SourceType.ANALYSIS); - ghidraRenamedFunctions.getAndIncrement(); - } catch (Exception e) { - throw new RuntimeException(e); - } + int ghidraBoundariesMatchedFunction = 0; + for (FunctionInfo info : functionInfo) { + var oFunc = getFunctionFor(info, program); + if (oFunc.isEmpty()) { + Msg.error(this, "Function not found in Ghidra for info: %s".formatted(info)); + continue; + } + var func = oFunc.get(); + // There are two ways to think about the size of a function + // They diverge for non-contiguous functions + var funcSizeByAddressCount = func.getBody().getNumAddresses(); + var funcSizeByDistance = func.getBody().getMaxAddress().subtract(func.getEntryPoint()) + 1; + + // For unclear reasons the func size is off by one + if (funcSizeByAddressCount - 1 != info.functionSize() && funcSizeByAddressCount != info.functionSize()) { + Msg.warn(this, "Function size mismatch for function %s: %d vs %d".formatted(func.getName(), funcSizeByAddressCount, info.functionSize())); + continue; + } - } - finalFunctionIDMap.add(func.getEntryPoint(), info.functionID().value()); - finalMangledNameMap.add(func.getEntryPoint(), revEngMangledName); + finalFunctionIDMap.add(func.getEntryPoint(), info.functionID().value()); - ghidraBoundariesMatchedFunction.getAndIncrement(); - } - ); + ghidraBoundariesMatchedFunction++; + } AtomicInteger ghidraFunctionCount = new AtomicInteger(); program.getFunctionManager().getFunctions(true).forEach( @@ -262,15 +230,151 @@ private void loadFunctionInfo(Program program, BinaryID binID) throws ApiExcepti // Print summary Msg.showInfo(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Function loading summary", - ("Found %d functions from RevEng.AI. Renamed %d. Your local Ghidra instance has %d/%d matching function " + + ("Found %d functions from RevEng.AI. Your local Ghidra instance has %d/%d matching function " + "boundaries. For better results, please start a new analysis from this plugin.").formatted( functionInfo.size(), - ghidraRenamedFunctions.get(), - ghidraBoundariesMatchedFunction.get(), + ghidraBoundariesMatchedFunction, ghidraFunctionCount.get() )); } + + /// Pull the server side information about the functions from a remote Analysis and update the local {@link Program} + /// based on it + /// This currently includes: + /// * the name of the function + /// * the type signature of the function + /// + /// It assumes that the initial load already happened, i.e. the functions have an associated FunctionID already. + /// The initial association happens in {@link #associateFunctionInfo(Program, BinaryID)} + /// + public void pullFunctionInfoFromAnalysis(ProgramWithBinaryID programWithBinaryID, TaskMonitor monitor) { + var transactionId = programWithBinaryID.program().startTransaction("RevEng.AI: Pull Function Info from Analysis"); + + StringPropertyMap mangledNameMap = programWithBinaryID.program() + .getUsrPropertyManager() + .getStringPropertyMap(REAI_FUNCTION_MANGLED_MAP); + + + int ghidraRenamedFunctions; + ghidraRenamedFunctions = 0; + + int failedRenames = 0; + for (Function function : programWithBinaryID.program().getFunctionManager().getFunctions(true)) { + if (monitor.isCancelled()) { + continue; + } + var ghidraMangledName = function.getSymbol().getName(false); + // Skip external and thunk functions because we don't support them + if (function.isExternal() || function.isThunk()) { + Msg.debug(this, "Skipping external/thunk function %s".formatted(ghidraMangledName)); + continue; + } + + var fID = getFunctionIDFor(function); + if (fID.isEmpty()) { + Msg.info(this, "Function %s has no associated FunctionID, skipping".formatted(function.getName())); + continue; + } + + // Get the current name on the server side + FunctionDetails details = api.getFunctionDetails(fID.get()); + var serverMangledName = details.functionName(); + + // Extract the mangled name from Ghidra + var revEngMangledName = details.functionName(); + // TODO: This is currently just a placeholder until the server provides demangled names at this endpoint! + var revEngDemangledName = details.demangledName(); + + // Skip invalid function mangled names + if (revEngMangledName.contains(" ") || revEngDemangledName.contains(" ")) { + Msg.warn(this, "Skipping renaming of function %s to invalid name %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); + continue; + } + + + // Get the type information on the server side + Optional functionSignatureMessageOpt = api.getFunctionDataTypes( + programWithBinaryID.analysisID(), + fID.get() + ) + // Try getting the data types if they are available + .flatMap(FunctionDataTypeStatus::data_types) + // If they are available, try converting them to a Ghidra signature + // If the conversion fails, act like there is no signature available + .flatMap((functionDataTypeMessage -> { + try { + return Optional.of(getFunctionSignature(functionDataTypeMessage)); + } catch (DataTypeDependencyException e) { + // Something went wrong loading the data type dependencies + // just skip applying the signature and treat it like none being available + Msg.error(this, "Could not get parse signature for function %s".formatted(function.getName())); + return Optional.empty(); + } + })); + + + mangledNameMap.add(function.getEntryPoint(), serverMangledName); + + /// Source types: + /// DEFAULT: placeholder name automatically assigned by Ghidra when it doesn’t know the real name. + /// ANALYSIS: A name/signature inferred by one of Ghidra’s analysis engines (or demangler) rather than simply “default.” + /// IMPORTED: Information taken from an external source — symbols or signatures imported from a file or database. + /// USER_DEFINED: A name or signature explicitly set by the analyst. + /// See {@link ghidra.program.model.symbol.SourceType} for more details + if (function.getSymbol().getSource() == SourceType.DEFAULT) { + if (functionSignatureMessageOpt.isEmpty()) { + // We don't have signature information for this function, so we can only try renaming it + if (function.getSymbol().getSource() == SourceType.DEFAULT && !revEngMangledName.startsWith("FUN_")) { + // The local function has the default name, so we can rename it + // The following check should never fail because it is a default name, + // and we checked above that the server name is not a default name + // but just to be safe and make that assumption explicit we check it explicitly + if (!function.getSymbol().getName(false).equals(serverMangledName)) { + Msg.info(this, "Renaming function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); + var success = new SetFunctionNameCmd(function.getEntryPoint(), revEngDemangledName, SourceType.ANALYSIS) + .applyTo(programWithBinaryID.program()); + if (success) { + ghidraRenamedFunctions++; + } else { + failedRenames++; + Msg.error(this, "Failed to rename function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); + } + } + } + + } else { + /// We could use {@link ghidra.program.model.listing.FunctionSignature#isEquivalentSignature(FunctionSignature)} + /// if we expect the server to have changing signatures at any point in time. + /// For now, we only apply signatures to functions that have the default signature + if (function.getSignatureSource() == SourceType.DEFAULT) { + var success = new ApplyFunctionSignatureCmd( + function.getEntryPoint(), + functionSignatureMessageOpt.get(), + SourceType.ANALYSIS + ).applyTo(programWithBinaryID.program(), monitor); + if (success) { + ghidraRenamedFunctions++; + } else { + Msg.error(this, "Failed to apply signature to function %s".formatted(function.getName())); + failedRenames++; + } + } + } + } + + + } + // Done iterating over all functions. If nothing changed, discard the transaction, to keep undo history clean + programWithBinaryID.program().endTransaction(transactionId, ghidraRenamedFunctions > 0); + if (failedRenames > 0){ + Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Function Update Summary", + ("Failed to update %d functions from RevEng.AI. Please check the error log for details.").formatted( + failedRenames + )); + } + } + /** * Get the FunctionID for a Ghidra Function, if there is one * @@ -285,13 +389,13 @@ public Optional getFunctionIDFor(Function function){ } public Optional getFunctionIDFor(ProgramWithBinaryID knownProgram, Function function){ - Optional functionIDMap = getFunctionIDMap(knownProgram); + Optional functionIDMap = getFunctionIDPropertyMap(knownProgram); return functionIDMap .flatMap(map -> Optional.ofNullable(map.get(function.getEntryPoint()))) .map(FunctionID::new); } - private Optional getFunctionIDMap(ProgramWithBinaryID program){ + private Optional getFunctionIDPropertyMap(ProgramWithBinaryID program){ return Optional.ofNullable(program.program().getUsrPropertyManager().getLongPropertyMap(REAI_FUNCTION_PROP_MAP)); } @@ -317,6 +421,9 @@ public BiMap getFunctionMap(Program program){ return functionMap; } + /** + * Get the Ghidra Function for a given FunctionInfo if there is one + */ public Optional getFunctionFor(FunctionInfo functionInfo, Program program){ // These addresses used to be relative, but are now absolute again var defaultAddressSpace = program.getAddressFactory().getDefaultAddressSpace(); @@ -475,10 +582,15 @@ public String decompileFunctionViaAI(Function function, TaskMonitor monitor, AID } } + + /// This method analyses a program by uploading it (if necessary), triggering an analysis, and _blocking_ + /// until the analysis is complete. This is for scripts and tests, and must not be used on the UI thread public ProgramWithBinaryID analyse(Program program, AnalysisOptionsBuilder analysisOptionsBuilder, TaskMonitor monitor) throws CancelledException, ApiException { + // Check if we are on the swing thread var programWithBinaryID = startAnalysis(program, analysisOptionsBuilder); - waitForFinishedAnalysis(monitor, programWithBinaryID, null, null); - registerFinishedAnalysisForProgram(programWithBinaryID); + var finalStatus = waitForFinishedAnalysis(monitor, programWithBinaryID, null, null); + // TODO: Check final status for errors, and do something appropriate on failure + registerFinishedAnalysisForProgram(programWithBinaryID, monitor); return programWithBinaryID; } @@ -490,9 +602,8 @@ public Optional getKnownProgram(Program program) { ); } - public Optional getFunctionSignatureArtifact(BinaryID binID, FunctionID functionID) { - var analysisID = api.getAnalysisIDfromBinaryID(binID); - return api.getFunctionDataTypes(analysisID, functionID).flatMap(FunctionDataTypeStatus::data_types); + public Optional getFunctionSignatureArtifact(ProgramWithBinaryID program, FunctionID functionID) { + return api.getFunctionDataTypes(program.analysisID(), functionID).flatMap(FunctionDataTypeStatus::data_types); } /** @@ -756,6 +867,7 @@ public List getActiveAnalysisIDsFilter() { } /** + * @param tool The UI tool for firing an event on status changes. Can be null * @return The final AnalysisStatus, should be either Complete or Error */ public AnalysisStatus waitForFinishedAnalysis( diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java index 105ac77..9ea14f5 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java @@ -28,4 +28,9 @@ public static FunctionDetails fromJSON(JSONObject json) { new BinaryHash(json.getString("sha_256_hash")) ); } + + public String demangledName() { + // This is currently a fake placeholder for the future field that will contain the demangled name + return functionName; + } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java index fe7a7f4..321eaa0 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java @@ -44,6 +44,7 @@ import ghidra.program.util.GhidraProgramUtilities; import ghidra.util.Msg; import ghidra.util.task.TaskBuilder; +import ghidra.util.task.TaskMonitor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -308,7 +309,9 @@ public void processEvent(PluginEvent event) { // If the analysis is complete, we refresh the function signatures from the server var program = analysisEvent.getProgramWithBinaryID(); try { - revengService.registerFinishedAnalysisForProgram(program); + // TODO: Can we get a better taskmonitor here? + // Or should we never do something here that warrants a monitor in the first place? + revengService.registerFinishedAnalysisForProgram(program, TaskMonitor.DUMMY); } catch (Exception e) { Msg.error(this, "Error registering finished analysis for program " + program, e); return; From af4e16d5afd48a10f98463f85a6fbab85924954e Mon Sep 17 00:00:00 2001 From: Florian Magin Date: Fri, 21 Nov 2025 14:48:08 +0100 Subject: [PATCH 02/10] Encode the lifecycle of an Analysis for a program into the type system --- ghidra_scripts/RevEngExamplePostScript.java | 6 +- .../aidecompiler/AIDecompilationdWindow.java | 49 ++-- .../ui/autounstrip/AutoUnstripDialog.java | 3 +- .../AbstractFunctionMatchingDialog.java | 13 +- .../BinaryLevelFunctionMatchingDialog.java | 5 +- .../FunctionLevelFunctionMatchingDialog.java | 7 +- .../ui/misc/AnalysisLogComponent.java | 7 +- .../recentanalyses/RecentAnalysisDialog.java | 3 +- .../core/RevEngAIAnalysisResultsLoaded.java | 8 +- .../RevEngAIAnalysisStatusChangedEvent.java | 8 +- .../services/api/GhidraRevengService.java | 211 +++++++++++++----- .../ghidra/core/tasks/StartAnalysisTask.java | 8 +- .../core/types/ProgramWithBinaryID.java | 17 -- .../devplugin/RevEngMetadataProvider.java | 6 +- .../plugins/AnalysisManagementPlugin.java | 43 ++-- .../plugins/BinarySimilarityPlugin.java | 47 ++-- .../reveng/PortalAnalysisIntegrationTest.java | 45 +++- .../ai/reveng/TestAnalysisLogComponent.java | 10 +- 18 files changed, 298 insertions(+), 198 deletions(-) delete mode 100644 src/main/java/ai/reveng/toolkit/ghidra/core/types/ProgramWithBinaryID.java diff --git a/ghidra_scripts/RevEngExamplePostScript.java b/ghidra_scripts/RevEngExamplePostScript.java index 600db86..dbdfe7e 100644 --- a/ghidra_scripts/RevEngExamplePostScript.java +++ b/ghidra_scripts/RevEngExamplePostScript.java @@ -1,6 +1,7 @@ import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.types.ApiInfo; +import ghidra.app.plugin.core.analysis.AutoAnalysisManager; import ghidra.app.script.GhidraScript; public class RevEngExamplePostScript extends GhidraScript { @@ -12,9 +13,6 @@ protected void run() throws Exception { ghidraRevengService.upload(currentProgram); AnalysisOptionsBuilder options = AnalysisOptionsBuilder.forProgram(currentProgram); - var binID = ghidraRevengService.analyse(currentProgram, options, monitor); - - // Wait for analysis to finish - ghidraRevengService.waitForFinishedAnalysis(monitor, binID, null, null); + var analyzedProgram = ghidraRevengService.analyse(currentProgram, options, monitor); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java index 07f4519..3957a38 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java @@ -53,10 +53,15 @@ public AIDecompilationdWindow(PluginTool tool, String owner) { public void actionPerformed(ActionContext context) { if (function != null) { var service = tool.getService(GhidraRevengService.class); - var fID = service.getFunctionIDFor(function); + var analyzedProgram = service.getAnalysedProgram(function.getProgram()); + if (analyzedProgram.isEmpty()) { + Msg.error(this, "Failed to send positive feedback: Program is not known to RevEng.AI"); + return; + } + var fID = analyzedProgram.get().getIDForFunction(function); fID.ifPresent(id -> { try { - service.getApi().aiDecompRating(id, "POSITIVE", ""); + service.getApi().aiDecompRating(id.functionID(), "POSITIVE", ""); } catch (ApiException e) { // Fail silently because this is not a critical feature Msg.error(this, "Failed to send positive feedback for function %s: %s".formatted(function.getName(), e.getMessage())); @@ -89,10 +94,15 @@ public void actionPerformed(ActionContext context) { if (!dialog.isCanceled()) { if (function != null) { var service = tool.getService(GhidraRevengService.class); - var fID = service.getFunctionIDFor(function); + var programWithID = service.getAnalysedProgram(function.getProgram()); + if (programWithID.isEmpty()) { + Msg.error(this, "Failed to send negative feedback: Program is not known to RevEng.AI"); + return; + } + var fID = programWithID.get().getIDForFunction(function); fID.ifPresent(id -> { try { - service.getApi().aiDecompRating(id, "NEGATIVE", dialog.getValue()); + service.getApi().aiDecompRating(id.functionID(), "NEGATIVE", dialog.getValue()); } catch (ApiException e) { // Fail silently because this is not a critical feature Msg.error(this, "Failed to send negative feedback for function %s: %s".formatted(function.getName(), e.getMessage())); @@ -164,11 +174,11 @@ private void clear() { descriptionArea.setText(""); } - public void refresh(Function function) { + public void refresh(GhidraRevengService.FunctionWithID function) { // Check if we know this function already - var cachedStatus = cache.get(function); + var cachedStatus = cache.get(function.function()); if (cachedStatus != null) { - setDisplayedValuesBasedOnStatus(function, cachedStatus); + setDisplayedValuesBasedOnStatus(function.function(), cachedStatus); } else { // TODO: Allow toggling auto decomp mode via local toggle action, for now do it always @@ -184,6 +194,13 @@ public void refresh(Function function) { } public void locationChanged(ProgramLocation loc) { + var service = tool.getService(GhidraRevengService.class); + var analyzedProgram = service.getAnalysedProgram(loc.getProgram()); + if (analyzedProgram.isEmpty()) { + clear(); + return; + } + var functionMgr = loc.getProgram().getFunctionManager(); var newFuncLocation = functionMgr.getFunctionContaining(loc.getAddress()); @@ -193,9 +210,8 @@ public void locationChanged(ProgramLocation loc) { } function = newFuncLocation; - if (function != null && !function.isExternal() && !function.isThunk()) { - refresh(function); - } + var functionWithID = analyzedProgram.get().getIDForFunction(function); + functionWithID.ifPresent(this::refresh); } @@ -225,18 +241,17 @@ private boolean hasPendingDecompilations() { class AIDecompTask extends Task { private final GhidraRevengService service; - private final Function function; + private final GhidraRevengService.FunctionWithID functionWithID; - public AIDecompTask(PluginTool tool, Function function) { + public AIDecompTask(PluginTool tool, GhidraRevengService.FunctionWithID functionWithID) { super("AI Decomp task", true, false, false); service = tool.getService(GhidraRevengService.class); - this.function = function; + this.functionWithID = functionWithID; } @Override public void run(TaskMonitor monitor) throws CancelledException { - var fID = service.getFunctionIDFor(function) - .orElseThrow(() -> new RuntimeException("Function has no associated FunctionID")); + var fID = functionWithID.functionID(); // Check if there is an existing process already, because the trigger API will fail with 400 if there is if (service.getApi().pollAIDecompileStatus(fID).status().equals("uninitialised")) { // Trigger the decompilation @@ -256,9 +271,9 @@ private void waitForDecomp(FunctionID id, TaskMonitor monitor) throws CancelledE if (lastDecompStatus == null || !Objects.equals(newStatus.status(), lastDecompStatus.status())) { lastDecompStatus = newStatus; - newStatusForFunction(function, newStatus); + newStatusForFunction(functionWithID.function(), newStatus); } - monitor.setMessage("Waiting for AI Decompilation for %s ... Current status: %s".formatted(function.getName(), lastDecompStatus.status())); + monitor.setMessage("Waiting for AI Decompilation for %s ... Current status: %s".formatted(functionWithID.function().getName(), lastDecompStatus.status())); monitor.checkCancelled(); switch (newStatus.status()) { case "pending": diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java index df5b121..b480b69 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java @@ -5,7 +5,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; import ai.reveng.toolkit.ghidra.core.services.api.types.AutoUnstripResponse; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionID; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.address.Address; @@ -49,7 +48,7 @@ public class AutoUnstripDialog extends RevEngDialogComponentProvider { private record RenameResult(String virtualAddress, String originalName, String newName) { } - public AutoUnstripDialog(PluginTool tool, ProgramWithBinaryID analysisID) { + public AutoUnstripDialog(PluginTool tool, GhidraRevengService.ProgramWithBinaryID analysisID) { super(ReaiPluginPackage.WINDOW_PREFIX + "Auto Unstrip", true); this.analysisID = analysisID.analysisID(); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java index d8fa825..7014ad4 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java @@ -3,7 +3,6 @@ import ai.reveng.model.*; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionID; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.dialog.RevEngDialogComponentProvider; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.components.CollectionSelectionPanel; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.components.BinarySelectionPanel; @@ -28,7 +27,7 @@ public abstract class AbstractFunctionMatchingDialog extends RevEngDialogComponentProvider { protected final GhidraRevengService revengService; - protected final ProgramWithBinaryID programWithBinaryID; + protected final GhidraRevengService.AnalysedProgram analyzedProgram; // UI Components protected JPanel contentPanel; @@ -77,16 +76,16 @@ public FunctionMatchResult(String bestMatchName, String bestMatchMangledName, St } protected AbstractFunctionMatchingDialog(String title, Boolean isModal, GhidraRevengService revengService, - ProgramWithBinaryID programWithBinaryID) { + GhidraRevengService.AnalysedProgram analyzedProgram) { super(title, isModal); this.revengService = revengService; - this.programWithBinaryID = programWithBinaryID; + this.analyzedProgram = analyzedProgram; this.taskMonitorComponent = new TaskMonitorComponent(false, true); this.functionMatchResults = new ArrayList<>(); this.filteredFunctionMatchResults = new ArrayList<>(); try { - this.analysisBasicInfo = revengService.getBasicDetailsForAnalysis(programWithBinaryID.analysisID()); + this.analysisBasicInfo = revengService.getBasicDetailsForAnalysis(analyzedProgram.analysisID()); } catch (Exception e) { JOptionPane.showMessageDialog(null, "Failed to fetch analysis details: " + e.getMessage(), @@ -126,7 +125,7 @@ protected void startFunctionMatching() { protected void processFunctionMatchingResults(FunctionMatchingBatchResponse response) { functionMatchResults.clear(); - var functionMap = revengService.getFunctionMap(programWithBinaryID.program()); + var functionMap = revengService.getFunctionMap(analyzedProgram.program()); response.getMatches().forEach(matchResult -> { // Process each matched function in this result @@ -807,7 +806,7 @@ protected void batchRenameFunctions(List functionMatches) { } protected void importFunctionNames(List functionMatches) { - var program = programWithBinaryID.program(); + var program = analyzedProgram.program(); var mangledNameMapOpt = revengService.getFunctionMangledNamesMap(program); var functionMap = revengService.getFunctionMap(program); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/BinaryLevelFunctionMatchingDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/BinaryLevelFunctionMatchingDialog.java index 057f189..def4897 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/BinaryLevelFunctionMatchingDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/BinaryLevelFunctionMatchingDialog.java @@ -2,7 +2,6 @@ import ai.reveng.model.*; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ghidra.framework.plugintool.PluginTool; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.program.model.listing.Function; @@ -15,7 +14,7 @@ public class BinaryLevelFunctionMatchingDialog extends AbstractFunctionMatchingDialog { - public BinaryLevelFunctionMatchingDialog(PluginTool tool, ProgramWithBinaryID programWithBinaryID) { + public BinaryLevelFunctionMatchingDialog(PluginTool tool, GhidraRevengService.AnalysedProgram programWithBinaryID) { super(ReaiPluginPackage.WINDOW_PREFIX + "Function Matching", true, tool.getService(GhidraRevengService.class), programWithBinaryID); } @@ -44,7 +43,7 @@ protected void pollFunctionMatchingStatus() { request.setFilters(filters); - functionMatchingResponse = revengService.getFunctionMatchingForAnalysis(programWithBinaryID.analysisID(), request); + functionMatchingResponse = revengService.getFunctionMatchingForAnalysis(analyzedProgram.analysisID(), request); updateUI(); // Check if we hit an error status diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/FunctionLevelFunctionMatchingDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/FunctionLevelFunctionMatchingDialog.java index a962048..469f874 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/FunctionLevelFunctionMatchingDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/FunctionLevelFunctionMatchingDialog.java @@ -2,7 +2,6 @@ import ai.reveng.model.*; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.listing.Function; @@ -16,7 +15,7 @@ public class FunctionLevelFunctionMatchingDialog extends AbstractFunctionMatchingDialog { private final Function function; - public FunctionLevelFunctionMatchingDialog(PluginTool tool, ProgramWithBinaryID programWithBinaryID, Function function) { + public FunctionLevelFunctionMatchingDialog(PluginTool tool, GhidraRevengService.AnalysedProgram programWithBinaryID, Function function) { super(ReaiPluginPackage.WINDOW_PREFIX + "Function Matching", true, tool.getService(GhidraRevengService.class), programWithBinaryID); this.function = function; @@ -31,7 +30,7 @@ protected void pollFunctionMatchingStatus() { request.setResultsPerFunction(25); // TODO: Make configurable? request.setModelId(analysisBasicInfo.getModelId()); - var functionIDOpt = revengService.getFunctionIDFor(function); + var functionIDOpt = analyzedProgram.getIDForFunction(function); if (functionIDOpt.isEmpty()) { handleError("Could not find function ID for the selected function"); stopPolling(); @@ -39,7 +38,7 @@ protected void pollFunctionMatchingStatus() { } var functionIds = new ArrayList(); - functionIds.add(functionIDOpt.get().value()); + functionIds.add(functionIDOpt.get().functionID().value()); request.setFunctionIds(functionIds); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java index f34fb41..497134a 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java @@ -4,7 +4,6 @@ import ai.reveng.toolkit.ghidra.core.AnalysisLogConsumer; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ghidra.framework.plugintool.ComponentProviderAdapter; import ghidra.framework.plugintool.PluginTool; import ghidra.util.exception.CancelledException; @@ -24,7 +23,7 @@ public class AnalysisLogComponent extends ComponentProviderAdapter implements An private final JScrollPane scroll; private TaskMonitorComponent taskMonitorComponent; private final JPanel mainPanel; - private final Map trackedPrograms = new ConcurrentHashMap<>(); + private final Map trackedPrograms = new ConcurrentHashMap<>(); public static String NAME = ReaiPluginPackage.WINDOW_PREFIX + "Analysis Log"; @@ -98,10 +97,10 @@ public void consumeLogs(String logs) { class AnalysisMonitoringTask extends Task { - private final ProgramWithBinaryID program; + private final GhidraRevengService.ProgramWithBinaryID program; private final AnalysisLogConsumer logConsumer; - public AnalysisMonitoringTask(ProgramWithBinaryID programWithBinaryID, AnalysisLogConsumer logConsumer) { + public AnalysisMonitoringTask(GhidraRevengService.ProgramWithBinaryID programWithBinaryID, AnalysisLogConsumer logConsumer) { super(programWithBinaryID.toString(), true, false, false); program = programWithBinaryID; this.logConsumer = logConsumer; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java index a51b9e9..a01e1ab 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java @@ -5,7 +5,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.types.LegacyAnalysisResult; import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryHash; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.listing.Program; @@ -98,7 +97,7 @@ public void mouseClicked(MouseEvent e) { private void pickAnalysis(LegacyAnalysisResult result) { var service = tool.getService(GhidraRevengService.class); var analysisID = service.getApi().getAnalysisIDfromBinaryID(result.binary_id()); - var programWithID = new ProgramWithBinaryID(program, result.binary_id(), analysisID); + var programWithID = new GhidraRevengService.ProgramWithBinaryID(program, result.binary_id(), analysisID); tool.firePluginEvent( new RevEngAIAnalysisStatusChangedEvent( diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisResultsLoaded.java b/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisResultsLoaded.java index 15e56a5..444ffe0 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisResultsLoaded.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisResultsLoaded.java @@ -1,6 +1,6 @@ package ai.reveng.toolkit.ghidra.core; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ghidra.framework.plugintool.PluginEvent; /// Indicates that RevEng AI analysis results have been loaded and other components and plugins can now start using them. @@ -9,13 +9,13 @@ /// - Function IDs have been associated with {@link ghidra.program.model.listing.Function}s public class RevEngAIAnalysisResultsLoaded extends PluginEvent { public static final String NAME = "RevEngAIAnalysisResultsLoaded"; - private final ProgramWithBinaryID programWithBinaryID; + private final GhidraRevengService.AnalysedProgram programWithBinaryID; - public RevEngAIAnalysisResultsLoaded(String source, ProgramWithBinaryID programWithBinaryID) { + public RevEngAIAnalysisResultsLoaded(String source, GhidraRevengService.AnalysedProgram programWithBinaryID) { super(NAME, source); this.programWithBinaryID = programWithBinaryID; } - public ProgramWithBinaryID getProgramWithBinaryID() { + public GhidraRevengService.AnalysedProgram getProgramWithBinaryID() { return programWithBinaryID; } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java b/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java index 4dadcf9..49ac0ae 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java @@ -1,8 +1,8 @@ package ai.reveng.toolkit.ghidra.core; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ghidra.framework.plugintool.PluginEvent; import ghidra.program.model.listing.Program; @@ -21,9 +21,9 @@ */ public class RevEngAIAnalysisStatusChangedEvent extends PluginEvent { private final AnalysisStatus status; - private final ProgramWithBinaryID programWithBinaryID; + private final GhidraRevengService.ProgramWithBinaryID programWithBinaryID; - public RevEngAIAnalysisStatusChangedEvent(String sourceName, ProgramWithBinaryID programWithBinaryID, AnalysisStatus status) { + public RevEngAIAnalysisStatusChangedEvent(String sourceName, GhidraRevengService.ProgramWithBinaryID programWithBinaryID, AnalysisStatus status) { super(sourceName, "RevEngAI Analysis Finished"); if (status == null || programWithBinaryID == null) { throw new IllegalArgumentException("args cannot be null"); @@ -36,7 +36,7 @@ public AnalysisStatus getStatus() { return status; } - public ProgramWithBinaryID getProgramWithBinaryID() { + public GhidraRevengService.ProgramWithBinaryID getProgramWithBinaryID() { return programWithBinaryID; } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java index c5ecf3c..057df03 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java @@ -13,7 +13,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.types.Collection; import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.*; import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIAuthenticationException; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import ghidra.app.cmd.function.ApplyFunctionSignatureCmd; @@ -95,22 +94,25 @@ public URI getServer() { return this.apiInfo.hostURI(); } - public void registerFinishedAnalysisForProgram(ProgramWithBinaryID programWithBinaryID, TaskMonitor monitor) throws ApiException { + public AnalysedProgram registerFinishedAnalysisForProgram(ProgramWithBinaryID programWithBinaryID, TaskMonitor monitor) throws ApiException { + var status = api.status(programWithBinaryID.analysisID); + if (!status.equals(AnalysisStatus.Complete)){ + throw new IllegalStateException("Analysis %s is not complete yet, current status: %s" + .formatted(programWithBinaryID.analysisID(), status)); + } statusCache.put(programWithBinaryID.binaryID(), AnalysisStatus.Complete); - // Add the binary to the program before loading the function info. Checking that a function is present requires - // the binary ID to be present in the program options. - addBinaryIDtoProgramOptions(programWithBinaryID.program(), programWithBinaryID.binaryID()); - - associateFunctionInfo(programWithBinaryID.program(), programWithBinaryID.binaryID()); - pullFunctionInfoFromAnalysis(programWithBinaryID, monitor); + var analysedProgram = associateFunctionInfo(programWithBinaryID); + pullFunctionInfoFromAnalysis(analysedProgram, monitor); + return analysedProgram; } - public void addBinaryIDtoProgramOptions(Program program, BinaryID binID){ + public ProgramWithBinaryID addBinaryIDtoProgramOptions(Program program, BinaryID binID){ var transactionId = program.startTransaction("Associate Binary ID with Program"); program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY) .setLong(ReaiPluginPackage.OPTION_KEY_BINID, binID.value()); program.endTransaction(transactionId, true); + return new ProgramWithBinaryID(program, binID, api.getAnalysisIDfromBinaryID(binID)); } /** @@ -167,7 +169,9 @@ public Optional getBinaryIDfromOptions( /// analysis is associated with the program /// Other function information like the name and signature should be loaded in [#pullFunctionInfoFromAnalysis(ProgramWithBinaryID,TaskMonitor)] /// because this information can change on the server, and thus needs a dedicated method to refresh it - private void associateFunctionInfo(Program program, BinaryID binID) throws ApiException { + private AnalysedProgram associateFunctionInfo(ProgramWithBinaryID knownProgram) throws ApiException { + var binID = knownProgram.binaryID(); + var program = knownProgram.program(); List functionInfo = api.getFunctionInfo(binID); var transactionID = program.startTransaction("Associate Function Info"); @@ -214,28 +218,34 @@ private void associateFunctionInfo(Program program, BinaryID binID) throws ApiEx ghidraBoundariesMatchedFunction++; } + + program.endTransaction(transactionID, true); + + + var analysedProgram = new AnalysedProgram(program, binID, api.getAnalysisIDfromBinaryID(binID)); AtomicInteger ghidraFunctionCount = new AtomicInteger(); program.getFunctionManager().getFunctions(true).forEach( func -> { if (!func.isExternal() && !func.isThunk()){ ghidraFunctionCount.getAndIncrement(); - if (getFunctionIDFor(func).isEmpty()) { + if (analysedProgram.getIDForFunction(func).isEmpty()) { Msg.info(this, "Function %s not found in RevEng.AI".formatted(func.getSymbol().getName(false))); } } } ); - program.endTransaction(transactionID, true); - // Print summary Msg.showInfo(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Function loading summary", - ("Found %d functions from RevEng.AI. Your local Ghidra instance has %d/%d matching function " + - "boundaries. For better results, please start a new analysis from this plugin.").formatted( - functionInfo.size(), - ghidraBoundariesMatchedFunction, - ghidraFunctionCount.get() - )); + ("Found %d functions from RevEng.AI. Your local Ghidra instance has %d/%d matching function " + + "boundaries. For better results, please start a new analysis from this plugin.").formatted( + functionInfo.size(), + ghidraBoundariesMatchedFunction, + ghidraFunctionCount.get() + )); + + return analysedProgram; + } @@ -246,12 +256,12 @@ private void associateFunctionInfo(Program program, BinaryID binID) throws ApiEx /// * the type signature of the function /// /// It assumes that the initial load already happened, i.e. the functions have an associated FunctionID already. - /// The initial association happens in {@link #associateFunctionInfo(Program, BinaryID)} + /// The initial association happens in {@link #associateFunctionInfo(ProgramWithBinaryID)} /// - public void pullFunctionInfoFromAnalysis(ProgramWithBinaryID programWithBinaryID, TaskMonitor monitor) { - var transactionId = programWithBinaryID.program().startTransaction("RevEng.AI: Pull Function Info from Analysis"); + public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMonitor monitor) { + var transactionId = analysedProgram.program().startTransaction("RevEng.AI: Pull Function Info from Analysis"); - StringPropertyMap mangledNameMap = programWithBinaryID.program() + StringPropertyMap mangledNameMap = analysedProgram.program() .getUsrPropertyManager() .getStringPropertyMap(REAI_FUNCTION_MANGLED_MAP); @@ -260,7 +270,7 @@ public void pullFunctionInfoFromAnalysis(ProgramWithBinaryID programWithBinaryID ghidraRenamedFunctions = 0; int failedRenames = 0; - for (Function function : programWithBinaryID.program().getFunctionManager().getFunctions(true)) { + for (Function function : analysedProgram.program().getFunctionManager().getFunctions(true)) { if (monitor.isCancelled()) { continue; } @@ -271,14 +281,14 @@ public void pullFunctionInfoFromAnalysis(ProgramWithBinaryID programWithBinaryID continue; } - var fID = getFunctionIDFor(function); + var fID = analysedProgram.getIDForFunction(function); if (fID.isEmpty()) { Msg.info(this, "Function %s has no associated FunctionID, skipping".formatted(function.getName())); continue; } // Get the current name on the server side - FunctionDetails details = api.getFunctionDetails(fID.get()); + FunctionDetails details = api.getFunctionDetails(fID.get().functionID); var serverMangledName = details.functionName(); // Extract the mangled name from Ghidra @@ -295,8 +305,8 @@ public void pullFunctionInfoFromAnalysis(ProgramWithBinaryID programWithBinaryID // Get the type information on the server side Optional functionSignatureMessageOpt = api.getFunctionDataTypes( - programWithBinaryID.analysisID(), - fID.get() + analysedProgram.analysisID(), + fID.get().functionID() ) // Try getting the data types if they are available .flatMap(FunctionDataTypeStatus::data_types) @@ -333,7 +343,7 @@ public void pullFunctionInfoFromAnalysis(ProgramWithBinaryID programWithBinaryID if (!function.getSymbol().getName(false).equals(serverMangledName)) { Msg.info(this, "Renaming function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); var success = new SetFunctionNameCmd(function.getEntryPoint(), revEngDemangledName, SourceType.ANALYSIS) - .applyTo(programWithBinaryID.program()); + .applyTo(analysedProgram.program()); if (success) { ghidraRenamedFunctions++; } else { @@ -352,7 +362,7 @@ public void pullFunctionInfoFromAnalysis(ProgramWithBinaryID programWithBinaryID function.getEntryPoint(), functionSignatureMessageOpt.get(), SourceType.ANALYSIS - ).applyTo(programWithBinaryID.program(), monitor); + ).applyTo(analysedProgram.program(), monitor); if (success) { ghidraRenamedFunctions++; } else { @@ -366,7 +376,7 @@ public void pullFunctionInfoFromAnalysis(ProgramWithBinaryID programWithBinaryID } // Done iterating over all functions. If nothing changed, discard the transaction, to keep undo history clean - programWithBinaryID.program().endTransaction(transactionId, ghidraRenamedFunctions > 0); + analysedProgram.program().endTransaction(transactionId, ghidraRenamedFunctions > 0); if (failedRenames > 0){ Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Function Update Summary", ("Failed to update %d functions from RevEng.AI. Please check the error log for details.").formatted( @@ -377,26 +387,16 @@ public void pullFunctionInfoFromAnalysis(ProgramWithBinaryID programWithBinaryID /** * Get the FunctionID for a Ghidra Function, if there is one - * * There are two cases where a function ID is missing: * 1. Either the whole program has not been analyzed - * 2. Or the function was not found as part of the analysis on the server * (because its bounds were not included when the analysis was triggered) + * + * @deprecated Use {@link AnalysedProgram#getIDForFunction(Function)} instead. It forces the caller to prove that they know that the {@link Program} is indeed known on the server and associated by having to provide a {@link AnalysedProgram} instance. */ + @Deprecated public Optional getFunctionIDFor(Function function){ - return getKnownProgram(function.getProgram()) - .flatMap(knownProgram -> getFunctionIDFor(knownProgram, function)); - } - - public Optional getFunctionIDFor(ProgramWithBinaryID knownProgram, Function function){ - Optional functionIDMap = getFunctionIDPropertyMap(knownProgram); - return functionIDMap - .flatMap(map -> Optional.ofNullable(map.get(function.getEntryPoint()))) - .map(FunctionID::new); - } - - private Optional getFunctionIDPropertyMap(ProgramWithBinaryID program){ - return Optional.ofNullable(program.program().getUsrPropertyManager().getLongPropertyMap(REAI_FUNCTION_PROP_MAP)); + return getAnalysedProgram(function.getProgram()) + .flatMap(knownProgram -> knownProgram.getIDForFunction(function).map(fidWithStatus -> fidWithStatus.functionID)); } public Optional getFunctionMangledNamesMap(Program program) { @@ -464,15 +464,13 @@ public void removeProgramAssociation(Program program){ program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY).setLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID); } + /// @deprecated use {@link GhidraRevengService#getAnalysedProgram(Program)} instead + @Deprecated public boolean isProgramAnalysed(Program program){ return program.getUsrPropertyManager().getLongPropertyMap(REAI_FUNCTION_PROP_MAP) != null && program.getUsrPropertyManager().getStringPropertyMap(REAI_FUNCTION_MANGLED_MAP) != null; } - public boolean isKnownFunction(Function function){ - return getFunctionIDFor(function).isPresent(); - } - public static List exportFunctionBoundaries(Program program){ List result = new ArrayList<>(); Address imageBase = program.getImageBase(); @@ -537,11 +535,11 @@ public AnalysisStatus pollStatus(BinaryID bid) { } } - public String decompileFunctionViaAI(Function function, TaskMonitor monitor, AIDecompilationdWindow window) { + public String decompileFunctionViaAI(FunctionWithID functionWithID, TaskMonitor monitor, AIDecompilationdWindow window) { monitor.setMaximum(100 * 50); - var fID = getFunctionIDFor(function) - .orElseThrow(() -> new RuntimeException("Function has no associated FunctionID")); // Check if there is an existing process already, because the trigger API will fail with 400 if there is + var fID = functionWithID.functionID; + var function = functionWithID.function; if (api.pollAIDecompileStatus(fID).status().equals("uninitialised")){ // Trigger the decompilation api.triggerAIDecompilationForFunctionID(fID); @@ -585,15 +583,18 @@ public String decompileFunctionViaAI(Function function, TaskMonitor monitor, AID /// This method analyses a program by uploading it (if necessary), triggering an analysis, and _blocking_ /// until the analysis is complete. This is for scripts and tests, and must not be used on the UI thread - public ProgramWithBinaryID analyse(Program program, AnalysisOptionsBuilder analysisOptionsBuilder, TaskMonitor monitor) throws CancelledException, ApiException { + /// It does not upload the program, this must be done beforehand, and the hash must be associated via {@link AnalysisOptionsBuilder#hash(BinaryHash)} + public AnalysedProgram analyse(Program program, AnalysisOptionsBuilder analysisOptionsBuilder, TaskMonitor monitor) throws CancelledException, ApiException { // Check if we are on the swing thread var programWithBinaryID = startAnalysis(program, analysisOptionsBuilder); var finalStatus = waitForFinishedAnalysis(monitor, programWithBinaryID, null, null); // TODO: Check final status for errors, and do something appropriate on failure - registerFinishedAnalysisForProgram(programWithBinaryID, monitor); - return programWithBinaryID; + var analysedProgram = registerFinishedAnalysisForProgram(programWithBinaryID, monitor); + return analysedProgram; } + /// Get the ProgramWithBinaryID for a known program + /// This only guarantees an associated analysis, not that it is finished public Optional getKnownProgram(Program program) { return getBinaryIDFor(program).map(binID -> { var analysisID = api.getAnalysisIDfromBinaryID(binID); @@ -602,6 +603,18 @@ public Optional getKnownProgram(Program program) { ); } + /// {@link GhidraRevengService#isProgramAnalysed(Program)} and if so, return an AnalysedProgram + public Optional getAnalysedProgram(Program program) { + var kProg = getKnownProgram(program); + if (kProg.isEmpty()){ + return Optional.empty(); + } + if (isProgramAnalysed(kProg.get().program())){ + return Optional.of(new AnalysedProgram(kProg.get().program(), kProg.get().binaryID(), kProg.get().analysisID())); + } + return Optional.empty(); + } + public Optional getFunctionSignatureArtifact(ProgramWithBinaryID program, FunctionID functionID) { return api.getFunctionDataTypes(program.analysisID(), functionID).flatMap(FunctionDataTypeStatus::data_types); } @@ -915,9 +928,7 @@ public AnalysisStatus waitForFinishedAnalysis( public ProgramWithBinaryID startAnalysis(Program program, AnalysisOptionsBuilder analysisOptionsBuilder) throws ApiException { var binaryID = api.analyse(analysisOptionsBuilder); - AnalysisID analysisID = api.getAnalysisIDfromBinaryID(binaryID); - addBinaryIDtoProgramOptions(program, binaryID); - return new ProgramWithBinaryID(program, binaryID, analysisID); + return addBinaryIDtoProgramOptions(program, binaryID); } public Map getNameScores(java.util.Collection values) { @@ -1020,4 +1031,86 @@ public FunctionMatchingBatchResponse getFunctionMatchingForFunction(FunctionMatc public void batchRenameFunctions(FunctionsListRename functionsList) throws ApiException { api.batchRenameFunctions(functionsList); } + + /// Old Helper Datatype that encapsulates a Ghidra program with a binary ID and analysis ID + /// This only guarantees that the program has an associated analysis, but not that the analysis is finished + /// The id of this should also be stored in the program options, but this is currently not enforced yet to allow easier testing + public record ProgramWithBinaryID( + Program program, + BinaryID binaryID, + AnalysisID analysisID + ){} + + + + + /// All functions that require a program to have a finished analysis on the portal can use this to encode this assumption into the type system + /// This guarantees that Ghidra Functions that exist on the server can be mapped to Function IDs + /// This rules out the two cases: + /// * the program has no associated analysis on the server + /// * the program has an associated analysis, but analysis hasn't finished yet + public static class AnalysedProgram { + + private final Program program; + private final BinaryID binaryID; + private final AnalysisID analysisID; + + + /// The constructor is private to enforce the use of the static factory method and from the outer class + private AnalysedProgram( + Program program, + BinaryID binaryID, + AnalysisID analysisID + ) { + this.program = program; + this.binaryID = binaryID; + this.analysisID = analysisID; + + } + + public Program program() { + return program; + } + + public AnalysisID analysisID() { + return analysisID; + } + + private LongPropertyMap getFunctionIDPropertyMap(AnalysedProgram program){ + var map = program.program().getUsrPropertyManager().getLongPropertyMap(REAI_FUNCTION_PROP_MAP); + if (map == null){ + throw new IllegalStateException("Function ID property map not found for supposedly known program %s".formatted(program.program().getName())); + } + return map; + } + + public static AnalysedProgram fromAnalysisResult(LegacyAnalysisResult analysisResult, Program program) { + if (analysisResult.status() != AnalysisStatus.Complete) { + throw new IllegalArgumentException("AnalysisResult must have status Complete to create ProgramWithBinaryID"); + } + return new AnalysedProgram(program, analysisResult.binary_id(), analysisResult.analysis_id()); + } + + /// Only returns Optional.Empty if the function is not known on the server (e.g. because it's a thunk) + public Optional getIDForFunction(Function function) { + if (function.getProgram() != this.program){ + throw new IllegalArgumentException("Function %s does not belong to program %s".formatted(function, this.program.getName())); + } + LongPropertyMap functionIDMap = getFunctionIDPropertyMap(this); + var rawId = functionIDMap.get(function.getEntryPoint()); + return Optional + .ofNullable(rawId) + .map(FunctionID::new) + .map( + functionID -> new FunctionWithID(function, functionID) + ); + } + + } + + /// Holding this object serves as the proof that a Function has an associated FunctionID + public static record FunctionWithID( + Function function, + FunctionID functionID + ) {} } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/tasks/StartAnalysisTask.java b/src/main/java/ai/reveng/toolkit/ghidra/core/tasks/StartAnalysisTask.java index ad335f9..872aee3 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/tasks/StartAnalysisTask.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/tasks/StartAnalysisTask.java @@ -6,7 +6,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.listing.Program; import ghidra.util.exception.CancelledException; @@ -27,12 +26,6 @@ public class StartAnalysisTask extends Task { private final AnalysisLogConsumer log; private final PluginTool tool; - public ProgramWithBinaryID getProgramWithBinaryID() { - return programWithBinaryID; - } - - private ProgramWithBinaryID programWithBinaryID; - public StartAnalysisTask(Program program, AnalysisOptionsBuilder options, GhidraRevengService reService, @@ -55,6 +48,7 @@ public void run(TaskMonitor monitor) throws CancelledException { monitor.setMessage("Sending Analysis Request"); + GhidraRevengService.ProgramWithBinaryID programWithBinaryID; try { programWithBinaryID = reService.startAnalysis(program, options); } catch (ApiException e) { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/types/ProgramWithBinaryID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/types/ProgramWithBinaryID.java deleted file mode 100644 index bf64277..0000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/types/ProgramWithBinaryID.java +++ /dev/null @@ -1,17 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.types; - -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; -import ghidra.program.model.listing.Program; - - -/** - * Helper Datatype that encapsulates a Ghidra program with a binary ID and analysis ID - * All functions that require a program to be known on the portal can use this to encode this assumption into the type system - */ -public record ProgramWithBinaryID( - Program program, - BinaryID binaryID, - AnalysisID analysisID -) { -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java b/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java index f5409be..758fd2c 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java @@ -107,11 +107,11 @@ public void locationChanged(ProgramLocation loc) { var func = loc.getProgram().getFunctionManager().getFunctionContaining(loc.getAddress()); if (func != null) { function = func; - var kProg = revengService.getKnownProgram(loc.getProgram()); + var kProg = revengService.getAnalysedProgram(loc.getProgram()); kProg.ifPresent(p -> { - Optional f = revengService.getFunctionIDFor(p, func); + Optional f = p.getIDForFunction(func); f.ifPresent(functionID -> { - this.functionID = functionID; + this.functionID = functionID.functionID(); }); }); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java index 321eaa0..3a15879 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java @@ -17,11 +17,9 @@ import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisResultsLoaded; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; -import ai.reveng.toolkit.ghidra.binarysimilarity.ui.about.AboutDialog; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.analysiscreation.RevEngAIAnalysisOptionsDialog; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.misc.AnalysisLogComponent; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.recentanalyses.RecentAnalysisDialog; -import ai.reveng.toolkit.ghidra.binarysimilarity.ui.help.HelpDialog; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.types.*; @@ -29,8 +27,6 @@ import ai.reveng.toolkit.ghidra.core.services.function.export.ExportFunctionBoundariesServiceImpl; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; import ai.reveng.toolkit.ghidra.core.tasks.StartAnalysisTask; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; -import docking.ActionContext; import docking.action.DockingAction; import docking.action.builder.ActionBuilder; import docking.widgets.OptionDialog; @@ -279,21 +275,26 @@ private void setupActions() { protected void programActivated(Program program) { super.programActivated(program); - if (!revengService.isKnownProgram(program)){ - var maybeBinID = revengService.getBinaryIDFor(program); - if (maybeBinID.isEmpty()){ - Msg.info(this, "Program has no saved binary ID"); - return; - } - var binID = maybeBinID.get(); - AnalysisStatus status = revengService.pollStatus(binID); - var analysisID = revengService.getApi().getAnalysisIDfromBinaryID(binID); - tool.firePluginEvent( - new RevEngAIAnalysisStatusChangedEvent( - "programActivated", - new ProgramWithBinaryID(program, binID, analysisID), - status)); - } + var knownProgram = revengService.getKnownProgram(program); + if (knownProgram.isPresent()) { + log.info("Activated known program: {}", knownProgram.get()); + } else { + // Program is not known yet + var maybeBinID = revengService.getBinaryIDFor(program); + if (maybeBinID.isEmpty()){ + Msg.info(this, "Program has no saved binary ID"); + return; + } + var binID = maybeBinID.get(); + AnalysisStatus status = revengService.pollStatus(binID); + var analysisID = revengService.getApi().getAnalysisIDfromBinaryID(binID); + tool.firePluginEvent( + new RevEngAIAnalysisStatusChangedEvent( + "programActivated", + new GhidraRevengService.ProgramWithBinaryID(program, binID, analysisID), + status)); + } + } @Override @@ -311,12 +312,12 @@ public void processEvent(PluginEvent event) { try { // TODO: Can we get a better taskmonitor here? // Or should we never do something here that warrants a monitor in the first place? - revengService.registerFinishedAnalysisForProgram(program, TaskMonitor.DUMMY); + var analysedProgram = revengService.registerFinishedAnalysisForProgram(program, TaskMonitor.DUMMY); + tool.firePluginEvent(new RevEngAIAnalysisResultsLoaded("AnalysisManagementPlugin", analysedProgram)); } catch (Exception e) { Msg.error(this, "Error registering finished analysis for program " + program, e); return; } - tool.firePluginEvent(new RevEngAIAnalysisResultsLoaded("AnalysisManagementPlugin", program)); } } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java index 4750a64..3fbac91 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java @@ -127,7 +127,7 @@ private void setupActions() { ) .onAction(context -> { var program = tool.getService(ProgramManager.class).getCurrentProgram(); - if (!apiService.isProgramAnalysed(program)) { + if (apiService.getAnalysedProgram(program).isEmpty()) { Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Auto Unstrip", "Analysis must have completed before running auto unstrip"); return; @@ -159,16 +159,12 @@ private void setupActions() { ) .onAction(context -> { var program = tool.getService(ProgramManager.class).getCurrentProgram(); - if (!apiService.isProgramAnalysed(program)) { + var knownProgram = apiService.getAnalysedProgram(program); + if (knownProgram.isEmpty()){ Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Function Matching", "Analysis must have completed before running function matching"); return; } - var knownProgram = apiService.getKnownProgram(program); - if (knownProgram.isEmpty()){ - Msg.info(this, "Program has no saved binary ID"); - return; - } var functionMatchingDialog = new BinaryLevelFunctionMatchingDialog(tool, knownProgram.get()); tool.showDialog(functionMatchingDialog); @@ -184,25 +180,15 @@ private void setupActions() { // Exclude thunks and external functions because we do not support them in the portal && !func.isExternal() && !func.isThunk() - && apiService.isKnownProgram(context.getProgram()) - && apiService.isProgramAnalysed(context.getProgram()); + && apiService.getAnalysedProgram(context.getProgram()).isPresent(); }) .onAction(context -> { - var program = tool.getService(ProgramManager.class).getCurrentProgram(); - if (!apiService.isProgramAnalysed(program)) { - Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Match function", - "Analysis must have completed before running function matching"); - return; - } - var knownProgram = apiService.getKnownProgram(program); - if (knownProgram.isEmpty()){ - Msg.info(this, "Program has no saved binary ID"); - return; - } + // We know analysed program is present due to enabledWhen + var knownProgram = apiService.getAnalysedProgram(context.getProgram()).get(); var func = context.getProgram().getFunctionManager().getFunctionContaining(context.getAddress()); - var functionMatchingDialog = new FunctionLevelFunctionMatchingDialog(tool, knownProgram.get(), func); + var functionMatchingDialog = new FunctionLevelFunctionMatchingDialog(tool, knownProgram, func); tool.showDialog(functionMatchingDialog); }) .popupMenuPath(new String[] { "Match function" }) @@ -219,12 +205,13 @@ private void setupActions() { // Exclude thunks and external functions because we do not support them in the portal && !func.isExternal() && !func.isThunk() - && apiService.isKnownProgram(context.getProgram()) - && apiService.isProgramAnalysed(context.getProgram()); + && apiService.getAnalysedProgram(context.getProgram()).isPresent(); }) .onAction(context -> { var func = context.getProgram().getFunctionManager().getFunctionContaining(context.getAddress()); - if (!apiService.isKnownFunction(func)) { + var analysedProgram = apiService.getAnalysedProgram(context.getProgram()).get(); + var functionWithId = analysedProgram.getIDForFunction(func); + if (functionWithId.isEmpty()) { Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "AI Decompilation", "Function is not known to the RevEng.AI API." + "This can happen if the function boundaries do not match.\n" + @@ -234,7 +221,7 @@ private void setupActions() { // Spawn Task to decompile the function tool.getService(ReaiLoggingService.class).info("Requested AI Decompilation via Action for function " + func.getName()); decompiledWindow.setVisible(true); - decompiledWindow.refresh(func); + decompiledWindow.refresh(functionWithId.get()); }) .popupMenuPath(new String[] { "AI Decompilation" }) .popupMenuIcon(ReaiPluginPackage.REVENG_16) @@ -246,13 +233,13 @@ private void setupActions() { .enabledWhen(context -> { var func = context.getProgram().getFunctionManager().getFunctionContaining(context.getAddress()); return func != null - && apiService.isKnownProgram(context.getProgram()) - && apiService.isProgramAnalysed(context.getProgram()); + && apiService.getAnalysedProgram(context.getProgram()).isPresent(); }) .onAction(context -> { var func = context.getProgram().getFunctionManager().getFunctionContaining(context.getAddress()); - var functionID = apiService.getFunctionIDFor(func); - if (!apiService.isKnownFunction(func) || functionID.isEmpty()) { + var analysedProgram = apiService.getAnalysedProgram(context.getProgram()).get(); + var functionWithID = analysedProgram.getIDForFunction(func); + if (functionWithID.isEmpty()) { Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "View function in portal", "Function is not known to the RevEng.AI API." + "This can happen if the function boundaries do not match.\n" + @@ -260,7 +247,7 @@ private void setupActions() { return; } - apiService.openFunctionInPortal(functionID.get()); + apiService.openFunctionInPortal(functionWithID.get().functionID()); }) .popupMenuPath(new String[] { "View function in portal" }) .popupMenuIcon(ReaiPluginPackage.REVENG_16) diff --git a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java index a753349..065a656 100644 --- a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java +++ b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java @@ -5,7 +5,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ai.reveng.toolkit.ghidra.plugins.AnalysisManagementPlugin; import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; @@ -13,6 +12,7 @@ import org.junit.Test; import java.util.List; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.Assert.assertEquals; @@ -20,6 +20,10 @@ public class PortalAnalysisIntegrationTest extends RevEngMockableHeadedIntegrationTest { + /// TODO: This test currently tests more things at once than needed and could be split + /// * Test that the checks the event lifecycle works (status changed -> results loaded) + /// * Test that loading the function info/details works correctly + /// @Test public void testInfoLoading() throws Exception { @@ -28,13 +32,43 @@ public void testInfoLoading() throws Exception { @Override public List getFunctionInfo(BinaryID binaryID) { return List.of( - new FunctionInfo(new FunctionID(1), "portal_name", "portal_name_mangled", 0x4000L, 0x100) + new FunctionInfo(new FunctionID(1), "portal_name_demangled", "portal_name_mangled", 0x4000L, 0x100) + ); + } + + @Override + public Optional getFunctionDataTypes(AnalysisID analysisID, FunctionID functionID) { + return Optional.empty(); + } + + @Override + public AnalysisStatus status(AnalysisID analysisID) { + return AnalysisStatus.Complete; + } + + @Override + public FunctionDetails getFunctionDetails(FunctionID id) { + return new FunctionDetails( + id, + "portal_name_mangled", + 0x4000L, + 0x100L, + new AnalysisID(1), + new BinaryID(1), + "binary_name", + new BinaryHash("dummyhash") ); } }); var builder = new ProgramBuilder("mock", ProgramBuilder._8051, this); // Add an example function var exampleFunc = builder.createEmptyFunction(null, "0x4000", 0x100, Undefined.getUndefinedDataType(8)); + // We need to also create the memory where the function lives, `getFunctions` doesn't find it otherwise + builder.createMemory("test", "0x4000", 0x100); + Assert.assertNotNull(builder.getProgram().getFunctionManager().getFunctionAt(exampleFunc.getEntryPoint())); + assert builder.getProgram().getFunctionManager().getFunctionCount() == 1; + assert builder.getProgram().getFunctionManager().getFunctionAt(exampleFunc.getEntryPoint()) != null; + assert builder.getProgram().getFunctionManager().getFunctions(true).hasNext(); var program = builder.getProgram(); var defaultTool = env.showTool(program); @@ -42,7 +76,8 @@ public List getFunctionInfo(BinaryID binaryID) { env.addPlugin(AnalysisManagementPlugin.class); waitForSwing(); - var id = new ProgramWithBinaryID(program, new BinaryID(1), new AnalysisID(1)); + + var id = new GhidraRevengService.ProgramWithBinaryID(program, new BinaryID(1), new AnalysisID(1)); var service = defaultTool.getService(GhidraRevengService.class); service.addBinaryIDtoProgramOptions(program, id.binaryID()); @@ -68,7 +103,7 @@ public List getFunctionInfo(BinaryID binaryID) { // Check that we received the results loaded event, i.e. other plugins would have been notified assertTrue(receivedResultsLoadedEvent.get()); // Check that the function names have been updated to the one returned by the portal - assertEquals("portal_name", exampleFunc.getName()); + assertEquals("portal_name_demangled", exampleFunc.getName()); // Check the function ID has been stored in the program options var funcIDMap = service.getFunctionMap(program); @@ -76,7 +111,7 @@ public List getFunctionInfo(BinaryID binaryID) { var storedFunc = funcIDMap.get(new FunctionID(1)); Assert.assertNotNull(storedFunc); - assertEquals("portal_name", storedFunc.getName()); + assertEquals("portal_name_demangled", storedFunc.getName()); // Check the function mangled name has been stored var mangledNamesMap = service.getFunctionMangledNamesMap(program); diff --git a/src/test/java/ai/reveng/TestAnalysisLogComponent.java b/src/test/java/ai/reveng/TestAnalysisLogComponent.java index 4d7b7b2..e411912 100644 --- a/src/test/java/ai/reveng/TestAnalysisLogComponent.java +++ b/src/test/java/ai/reveng/TestAnalysisLogComponent.java @@ -1,11 +1,11 @@ package ai.reveng; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.misc.AnalysisLogComponent; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.plugins.AnalysisManagementPlugin; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ghidra.program.database.ProgramBuilder; import ghidra.util.task.Task; import ghidra.util.task.TaskMonitorComponent; @@ -19,11 +19,11 @@ public class TestAnalysisLogComponent extends RevEngMockableHeadedIntegrationTest { - private ProgramWithBinaryID getPlaceHolderID() throws Exception{ + private GhidraRevengService.ProgramWithBinaryID getPlaceHolderID() throws Exception{ var builder = new ghidra.program.database.ProgramBuilder("mock", ProgramBuilder._8051, this); // Add an example function var program = builder.getProgram(); - return new ProgramWithBinaryID( + return new GhidraRevengService.ProgramWithBinaryID( program, new BinaryID(1), new AnalysisID(1) @@ -59,7 +59,7 @@ public AnalysisStatus status(AnalysisID analysisID) { // The analysis log component should now be visible, and have a task var logComponent = defaultTool.getComponentProvider(AnalysisLogComponent.NAME); assertNotNull(logComponent); - Map trackedPrograms = (Map) getInstanceField("trackedPrograms", logComponent); + Map trackedPrograms = (Map) getInstanceField("trackedPrograms", logComponent); assertNotNull(trackedPrograms); assertTrue(trackedPrograms.containsKey(program)); } @@ -107,7 +107,7 @@ public List getFunctionInfo(BinaryID binaryID) { ); AnalysisLogComponent logComponent = (AnalysisLogComponent) defaultTool.getComponentProvider(AnalysisLogComponent.NAME); - var trackedPrograms = (Map) getInstanceField("trackedPrograms", logComponent); + var trackedPrograms = (Map) getInstanceField("trackedPrograms", logComponent); assertNotNull(trackedPrograms); assertTrue(trackedPrograms.containsKey(program)); waitForTasks(); From 56c516d2296ce81836820efdfedc127ea555afd3 Mon Sep 17 00:00:00 2001 From: Florian Magin Date: Fri, 21 Nov 2025 15:27:29 +0100 Subject: [PATCH 03/10] Include data type in test for pulling function info --- .../reveng/PortalAnalysisIntegrationTest.java | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java index 065a656..65ae9af 100644 --- a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java +++ b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java @@ -5,9 +5,13 @@ import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; +import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.*; import ai.reveng.toolkit.ghidra.plugins.AnalysisManagementPlugin; import ghidra.program.database.ProgramBuilder; +import ghidra.program.model.data.TypeDef; import ghidra.program.model.data.Undefined; +import ghidra.program.model.symbol.SourceType; +import ghidra.util.task.TaskMonitor; import org.junit.Assert; import org.junit.Test; @@ -38,7 +42,40 @@ public List getFunctionInfo(BinaryID binaryID) { @Override public Optional getFunctionDataTypes(AnalysisID analysisID, FunctionID functionID) { - return Optional.empty(); + var ds = new FunctionDataTypeStatus( + true, + Optional.of( + new FunctionDataTypeMessage( + new FunctionArtifact( + 0x4000L, + 0x100, + new FunctionHeader( + null, + "portal_name_demangled", + 0x4000L, + "int", + new FunctionArgument[]{ + new FunctionArgument(0, 8, + null, + "named_param_1", + "char *" + ), + } + + ), + new StackVariable[]{} + ), + new FunctionDependencies( + new Typedef[]{}, + new Struct[]{} + ) + ) + ), + "completed", + null, + functionID + ); + return Optional.of(ds); } @Override @@ -63,6 +100,12 @@ public FunctionDetails getFunctionDetails(FunctionID id) { var builder = new ProgramBuilder("mock", ProgramBuilder._8051, this); // Add an example function var exampleFunc = builder.createEmptyFunction(null, "0x4000", 0x100, Undefined.getUndefinedDataType(8)); + /// Tell Ghidra that the function signature source is just default, + /// as the logic in {@link GhidraRevengService#pullFunctionInfoFromAnalysis(GhidraRevengService.AnalysedProgram, TaskMonitor)} + /// relies on this to decide whether to update the function signature or not + var tId = builder.getProgram().startTransaction("Set function signature source"); + exampleFunc.setSignatureSource(SourceType.DEFAULT); + builder.getProgram().endTransaction(tId, true); // We need to also create the memory where the function lives, `getFunctions` doesn't find it otherwise builder.createMemory("test", "0x4000", 0x100); Assert.assertNotNull(builder.getProgram().getFunctionManager().getFunctionAt(exampleFunc.getEntryPoint())); @@ -105,6 +148,11 @@ public FunctionDetails getFunctionDetails(FunctionID id) { // Check that the function names have been updated to the one returned by the portal assertEquals("portal_name_demangled", exampleFunc.getName()); + assertEquals(SourceType.ANALYSIS, exampleFunc.getSignatureSource()); + var signature = exampleFunc.getSignature(true); + assertEquals("int portal_name_demangled(char * named_param_1)", signature.getPrototypeString()); + + // Check the function ID has been stored in the program options var funcIDMap = service.getFunctionMap(program); From 7afbc4023840c706ae073868f5d96d2521a10d26 Mon Sep 17 00:00:00 2001 From: Florian Magin Date: Fri, 21 Nov 2025 17:02:46 +0100 Subject: [PATCH 04/10] fixup! Separate the two concerns of fetching function infos/details --- build.gradle | 2 +- .../services/api/GhidraRevengService.java | 18 +++++++---- .../services/api/TypedApiImplementation.java | 13 +++++--- .../api/mocks/TypeGenerationMock.java | 3 +- .../services/api/types/FunctionDetails.java | 32 +++++++++---------- .../reveng/PortalAnalysisIntegrationTest.java | 4 ++- 6 files changed, 40 insertions(+), 32 deletions(-) diff --git a/build.gradle b/build.gradle index ffd2f71..6382770 100644 --- a/build.gradle +++ b/build.gradle @@ -65,7 +65,7 @@ dependencies { implementation 'org.json:json:20250107' implementation "com.google.guava:guava:33.2.0-jre" implementation group: 'com.fifesoft', name: 'rsyntaxtextarea', version: '3.5.2' - implementation "ai.reveng:sdk:2.37.4" + implementation "ai.reveng:sdk:2.42.0" testImplementation('junit:junit:4.13.1') testRuntimeOnly("org.junit.vintage:junit-vintage-engine:5.8.2") diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java index 057df03..b6a2e01 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java @@ -94,7 +94,7 @@ public URI getServer() { return this.apiInfo.hostURI(); } - public AnalysedProgram registerFinishedAnalysisForProgram(ProgramWithBinaryID programWithBinaryID, TaskMonitor monitor) throws ApiException { + public AnalysedProgram registerFinishedAnalysisForProgram(ProgramWithBinaryID programWithBinaryID, TaskMonitor monitor) { var status = api.status(programWithBinaryID.analysisID); if (!status.equals(AnalysisStatus.Complete)){ throw new IllegalStateException("Analysis %s is not complete yet, current status: %s" @@ -169,10 +169,15 @@ public Optional getBinaryIDfromOptions( /// analysis is associated with the program /// Other function information like the name and signature should be loaded in [#pullFunctionInfoFromAnalysis(ProgramWithBinaryID,TaskMonitor)] /// because this information can change on the server, and thus needs a dedicated method to refresh it - private AnalysedProgram associateFunctionInfo(ProgramWithBinaryID knownProgram) throws ApiException { + private AnalysedProgram associateFunctionInfo(ProgramWithBinaryID knownProgram) { var binID = knownProgram.binaryID(); var program = knownProgram.program(); - List functionInfo = api.getFunctionInfo(binID); + List functionInfo = null; + try { + functionInfo = api.getFunctionInfo(binID); + } catch (ApiException e) { + throw new RuntimeException(e); + } var transactionID = program.startTransaction("Associate Function Info"); // Create the FunctionID map @@ -289,10 +294,9 @@ public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMo // Get the current name on the server side FunctionDetails details = api.getFunctionDetails(fID.get().functionID); - var serverMangledName = details.functionName(); // Extract the mangled name from Ghidra - var revEngMangledName = details.functionName(); + var revEngMangledName = details.mangledFunctionName(); // TODO: This is currently just a placeholder until the server provides demangled names at this endpoint! var revEngDemangledName = details.demangledName(); @@ -324,7 +328,7 @@ public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMo })); - mangledNameMap.add(function.getEntryPoint(), serverMangledName); + mangledNameMap.add(function.getEntryPoint(), revEngMangledName); /// Source types: /// DEFAULT: placeholder name automatically assigned by Ghidra when it doesn’t know the real name. @@ -340,7 +344,7 @@ public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMo // The following check should never fail because it is a default name, // and we checked above that the server name is not a default name // but just to be safe and make that assumption explicit we check it explicitly - if (!function.getSymbol().getName(false).equals(serverMangledName)) { + if (!function.getSymbol().getName(false).equals(revEngDemangledName)) { Msg.info(this, "Renaming function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); var success = new SetFunctionNameCmd(function.getEntryPoint(), revEngDemangledName, SourceType.ANALYSIS) .applyTo(analysedProgram.program()); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java index 660b812..188f1e1 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java @@ -519,11 +519,14 @@ public AnalysisResult getInfoForAnalysis(AnalysisID id) { */ @Override public FunctionDetails getFunctionDetails(FunctionID id) { - var request = requestBuilderForEndpoint("functions", String.valueOf(id.value())) - .GET() - .build(); - var response = sendVersion2Request(request); - return FunctionDetails.fromJSON(response.getJsonData()); + BaseResponseFunctionsDetailResponse dets = null; + try { + dets = functionsCoreApi.getFunctionDetails((int) id.value()); + } catch (ApiException e) { + throw new RuntimeException(e); + } + return FunctionDetails.fromServerResponse(dets.getData()); + } @Override diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/TypeGenerationMock.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/TypeGenerationMock.java index 7d74c75..3db24b2 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/TypeGenerationMock.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/TypeGenerationMock.java @@ -59,7 +59,8 @@ public FunctionDetails getFunctionDetails(FunctionID id) { new AnalysisID(1337), new BinaryID(1337), "placeholder_for_%s".formatted(id), - new BinaryHash("placeholder_for_%s".formatted(id)) + new BinaryHash("placeholder_for_%s".formatted(id)), + "demangled_placeholder_for_%s".formatted(id) ); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java index 9ea14f5..51fbdc0 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java @@ -1,36 +1,34 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; -import org.json.JSONObject; +import ai.reveng.model.FunctionsDetailResponse; /** * Record representing detailed function information from the RevEng.AI API */ public record FunctionDetails( FunctionID functionId, - String functionName, + String mangledFunctionName, Long functionVaddr, Long functionSize, AnalysisID analysisId, BinaryID binaryId, String binaryName, - BinaryHash sha256Hash + BinaryHash sha256Hash, + String demangledName ) { - public static FunctionDetails fromJSON(JSONObject json) { + + public static FunctionDetails fromServerResponse(FunctionsDetailResponse response) { return new FunctionDetails( - new FunctionID(json.getInt("function_id")), - json.getString("function_name_mangled"), - json.getLong("function_vaddr"), - json.getLong("function_size"), - new AnalysisID(json.getInt("analysis_id")), - new BinaryID(json.getInt("binary_id")), - json.getString("binary_name"), - new BinaryHash(json.getString("sha_256_hash")) + new FunctionID(response.getFunctionId()), + response.getFunctionNameMangled(), + response.getFunctionVaddr(), + response.getFunctionSize().longValue(), + new AnalysisID(response.getAnalysisId()), + new BinaryID(response.getBinaryId()), + response.getBinaryName(), + new BinaryHash(response.getSha256Hash()), + response.getFunctionName() ); } - - public String demangledName() { - // This is currently a fake placeholder for the future field that will contain the demangled name - return functionName; - } } diff --git a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java index 65ae9af..5624cb9 100644 --- a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java +++ b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java @@ -1,5 +1,6 @@ package ai.reveng; +import ai.reveng.model.FunctionsDetailResponse; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisResultsLoaded; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; @@ -93,7 +94,8 @@ public FunctionDetails getFunctionDetails(FunctionID id) { new AnalysisID(1), new BinaryID(1), "binary_name", - new BinaryHash("dummyhash") + new BinaryHash("dummyhash"), + "portal_name_demangled" ); } }); From 53fedc57544301d739d79b469b89cf782434e6ed Mon Sep 17 00:00:00 2001 From: Florian Magin Date: Fri, 21 Nov 2025 17:44:19 +0100 Subject: [PATCH 05/10] Improve analysis lifecycle handling and support multiple open/tracked programs waiting for analysis to finish --- .../ui/misc/AnalysisLogComponent.java | 37 +++++++--- .../ghidra/core/AnalysisLogConsumer.java | 4 +- .../services/api/GhidraRevengService.java | 2 +- .../plugins/AnalysisManagementPlugin.java | 67 +++++++++++++------ .../ai/reveng/TestAnalysisLogComponent.java | 21 +++--- 5 files changed, 92 insertions(+), 39 deletions(-) diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java index 497134a..2dc073b 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java @@ -6,6 +6,7 @@ import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ghidra.framework.plugintool.ComponentProviderAdapter; import ghidra.framework.plugintool.PluginTool; +import ghidra.program.model.listing.Program; import ghidra.util.exception.CancelledException; import ghidra.util.task.Task; import ghidra.util.task.TaskBuilder; @@ -23,9 +24,11 @@ public class AnalysisLogComponent extends ComponentProviderAdapter implements An private final JScrollPane scroll; private TaskMonitorComponent taskMonitorComponent; private final JPanel mainPanel; - private final Map trackedPrograms = new ConcurrentHashMap<>(); + private final Map trackedPrograms = new ConcurrentHashMap<>(); + private final Map storedLogs = new ConcurrentHashMap<>(); public static String NAME = ReaiPluginPackage.WINDOW_PREFIX + "Analysis Log"; + private Program activeProgram; public AnalysisLogComponent(PluginTool tool) { super(tool, NAME, ReaiPluginPackage.NAME); @@ -65,6 +68,18 @@ public TaskMonitor getTaskMonitor() { return taskMonitorComponent; } + public void programActivated(Program program) { + // Switch the active log to the given program + this.activeProgram = program; + if (trackedPrograms.containsKey(program)) { + // If we are tracking this program, show the task monitor + taskMonitorComponent.setVisible(true); + } else { + taskMonitorComponent.setVisible(false); + } + setLogs(storedLogs.getOrDefault(program, "No analysis logs available for the active program.")); + } + public void processEvent(RevEngAIAnalysisStatusChangedEvent event) { // We don't need to display the log window when the user selects an existing analysis because it will be an // already completed analysis. @@ -77,11 +92,11 @@ public void processEvent(RevEngAIAnalysisStatusChangedEvent event) { switch (event.getStatus()) { case Complete, Error -> {} default -> { - if (!trackedPrograms.containsKey(event.getProgramWithBinaryID())){ + if (!trackedPrograms.containsKey(event.getProgram())){ // We aren't tracking this program yet, so we start a new task for it var task = new AnalysisMonitoringTask(event.getProgramWithBinaryID(), this); var builder = TaskBuilder.withTask(task); - trackedPrograms.put(event.getProgramWithBinaryID(), task); + trackedPrograms.put(event.getProgram(), task); builder.launchInBackground(getTaskMonitor()); } } @@ -90,8 +105,13 @@ public void processEvent(RevEngAIAnalysisStatusChangedEvent event) { } @Override - public void consumeLogs(String logs) { - this.setLogs(logs); + public void consumeLogs(String logs, GhidraRevengService.ProgramWithBinaryID programWithBinaryID) { + storedLogs.put(programWithBinaryID.program(), logs); + // If this is the currently active program, update the log display + + if (activeProgram == programWithBinaryID.program()) { + setLogs(logs); + } } @@ -110,10 +130,11 @@ public AnalysisMonitoringTask(GhidraRevengService.ProgramWithBinaryID programWit public void run(TaskMonitor monitor) throws CancelledException { // Wait for the analysis var service = tool.getService(GhidraRevengService.class); - // TODO: Get the log consumer for this specific program service.waitForFinishedAnalysis(monitor, program, this.logConsumer, tool); - // Remove ourselfs from the tracked programs when done - trackedPrograms.remove(program); + /// waitForFinishedAnalysis has already sent the completion event to all plugins + /// so this task just needs to clean up the tracked programs map + // Remove ourselves from the tracked programs when done + trackedPrograms.remove(program.program()); taskMonitorComponent.setVisible(false); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/AnalysisLogConsumer.java b/src/main/java/ai/reveng/toolkit/ghidra/core/AnalysisLogConsumer.java index 17e196d..e1c015f 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/AnalysisLogConsumer.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/AnalysisLogConsumer.java @@ -1,6 +1,8 @@ package ai.reveng.toolkit.ghidra.core; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; + public interface AnalysisLogConsumer { - void consumeLogs(String logs); + void consumeLogs(String logs, GhidraRevengService.ProgramWithBinaryID programWithBinaryID); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java index b6a2e01..0b70b9c 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java @@ -904,7 +904,7 @@ public AnalysisStatus waitForFinishedAnalysis( // Analysis log endpoint only starts to return data after the analysis is processing String logs = this.getAnalysisLog(programWithID.analysisID()); if (logger != null) { - logger.consumeLogs(logs); + logger.consumeLogs(logs, programWithID); } var logsLines = logs.lines().toList(); var lastLine = logsLines.get(logsLines.size() - 1); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java index 3a15879..f2bf6e3 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java @@ -271,30 +271,59 @@ private void setupActions() { .buildAndInstall(tool); } - @Override - protected void programActivated(Program program) { - super.programActivated(program); - + @Override + protected void programOpened(Program program) { + super.programOpened(program); + // When a program is opened, we check if it has an associated analysis (not necessarily finished yet) var knownProgram = revengService.getKnownProgram(program); if (knownProgram.isPresent()) { - log.info("Activated known program: {}", knownProgram.get()); - } else { - // Program is not known yet - var maybeBinID = revengService.getBinaryIDFor(program); - if (maybeBinID.isEmpty()){ - Msg.info(this, "Program has no saved binary ID"); - return; + log.info("Opened known program: {}", knownProgram.get()); + var analysedProgram = revengService.getAnalysedProgram(program); + if (analysedProgram.isPresent()) { + // Nothing to do, we already have loaded the function IDs and similar + log.info("Loaded analysed program: {}", analysedProgram); + } else { + // There is an associated program that hasn't been fully loaded yet + // This can happen if the analysis was started in a previous session but hadn't finished when closing Ghidra + // Either the analysis is finished already now, or we want to actively wait for it to finish + log.info("Detected known program that hasn't been fully loaded yet: {}", knownProgram.get()); + var status = revengService.getApi().status(knownProgram.get().analysisID()); + switch (status) { + case Complete -> { + tool.firePluginEvent( + new RevEngAIAnalysisStatusChangedEvent( + "programOpened", + knownProgram.get(), + status)); + } + case Queued, Processing -> { + // This is the same code as above, but it has the implicit assumption that someone + /// Currently this is done by {@link AnalysisLogComponent#processEvent(RevEngAIAnalysisStatusChangedEvent)} + tool.firePluginEvent( + new RevEngAIAnalysisStatusChangedEvent( + "programOpened", + knownProgram.get(), + status)); + } + + + case Error -> { + // The analysis failed on the server side + Msg.showError(this, null, "Analysis Error", + "The RevEng.AI analysis for the program " + knownProgram.get() + " is in error state on the server side."); + } + } } - var binID = maybeBinID.get(); - AnalysisStatus status = revengService.pollStatus(binID); - var analysisID = revengService.getApi().getAnalysisIDfromBinaryID(binID); - tool.firePluginEvent( - new RevEngAIAnalysisStatusChangedEvent( - "programActivated", - new GhidraRevengService.ProgramWithBinaryID(program, binID, analysisID), - status)); + } else { + log.info("Opened unknown program: {}", program.getName()); } + } + @Override + protected void programActivated(Program program) { + super.programActivated(program); + // Any ComponentProviders that need to refresh based on the current program should be notified here + analysisLogComponent.programActivated(program); } @Override diff --git a/src/test/java/ai/reveng/TestAnalysisLogComponent.java b/src/test/java/ai/reveng/TestAnalysisLogComponent.java index e411912..633d517 100644 --- a/src/test/java/ai/reveng/TestAnalysisLogComponent.java +++ b/src/test/java/ai/reveng/TestAnalysisLogComponent.java @@ -7,6 +7,7 @@ import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ghidra.program.database.ProgramBuilder; +import ghidra.program.model.listing.Program; import ghidra.util.task.Task; import ghidra.util.task.TaskMonitorComponent; import org.junit.Test; @@ -47,11 +48,11 @@ public AnalysisStatus status(AnalysisID analysisID) { // defaultTool.getService(GhidraRevengService.class); env.addPlugin(AnalysisManagementPlugin.class); - var program = getPlaceHolderID(); + var programWithID = getPlaceHolderID(); defaultTool.firePluginEvent( new RevEngAIAnalysisStatusChangedEvent( "Test", - program, + programWithID, AnalysisStatus.Processing ) ); @@ -59,22 +60,22 @@ public AnalysisStatus status(AnalysisID analysisID) { // The analysis log component should now be visible, and have a task var logComponent = defaultTool.getComponentProvider(AnalysisLogComponent.NAME); assertNotNull(logComponent); - Map trackedPrograms = (Map) getInstanceField("trackedPrograms", logComponent); + Map trackedPrograms = (Map) getInstanceField("trackedPrograms", logComponent); assertNotNull(trackedPrograms); - assertTrue(trackedPrograms.containsKey(program)); + assertTrue(trackedPrograms.containsKey(programWithID.program())); } @Test public void testFullAnalysisFlow() throws Exception { var defaultTool = env.showTool(); - var program = getPlaceHolderID(); + var programWithID = getPlaceHolderID(); addMockedService(defaultTool, new UnimplementedAPI() { private final Map statusMap = new HashMap<>(); { - statusMap.put(program.analysisID(), AnalysisStatus.Queued); + statusMap.put(programWithID.analysisID(), AnalysisStatus.Queued); } @Override @@ -101,19 +102,19 @@ public List getFunctionInfo(BinaryID binaryID) { defaultTool.firePluginEvent( new RevEngAIAnalysisStatusChangedEvent( "Test", - program, + programWithID, AnalysisStatus.Queued ) ); AnalysisLogComponent logComponent = (AnalysisLogComponent) defaultTool.getComponentProvider(AnalysisLogComponent.NAME); - var trackedPrograms = (Map) getInstanceField("trackedPrograms", logComponent); + var trackedPrograms = (Map) getInstanceField("trackedPrograms", logComponent); assertNotNull(trackedPrograms); - assertTrue(trackedPrograms.containsKey(program)); + assertTrue(trackedPrograms.containsKey(programWithID.program())); waitForTasks(); // Check that it is cleared out again after the task finished - assertFalse(trackedPrograms.containsKey(program)); + assertFalse(trackedPrograms.containsKey(programWithID.program())); // Check that the taskmonitor is hidden again and isn't sitting there forever TaskMonitorComponent taskMonitorComponent = (TaskMonitorComponent) getInstanceField("taskMonitorComponent", logComponent); assertFalse(taskMonitorComponent.isVisible()); From 9dcdfbd5044acc407c49230515520581305d9987 Mon Sep 17 00:00:00 2001 From: Florian Magin Date: Fri, 21 Nov 2025 20:21:48 +0100 Subject: [PATCH 06/10] Start Deprecating BinaryID --- .../ui/autounstrip/AutoUnstripDialog.java | 2 +- .../ui/misc/AnalysisLogComponent.java | 14 +- .../recentanalyses/RecentAnalysisDialog.java | 5 +- .../ghidra/core/AnalysisLogConsumer.java | 2 +- .../RevEngAIAnalysisStatusChangedEvent.java | 19 +- .../services/api/GhidraRevengService.java | 238 +++++++++++------- .../services/api/TypedApiImplementation.java | 47 ++-- .../core/services/api/TypedApiInterface.java | 22 +- .../core/services/api/mocks/MockApi.java | 24 +- .../api/mocks/ProcessingLimboApi.java | 35 --- .../services/api/mocks/SimpleMatchesApi.java | 15 -- .../api/mocks/TypeGenerationMock.java | 66 ----- .../services/api/mocks/UnimplementedAPI.java | 10 - .../core/services/api/types/AnalysisID.java | 3 + .../services/api/types/AnalysisResult.java | 34 +-- .../core/services/api/types/BinaryID.java | 1 + .../services/api/types/FunctionDetails.java | 2 - .../services/api/types/FunctionMatch.java | 3 - .../services/api/types/InvalidBinaryID.java | 21 -- .../api/types/LegacyAnalysisResult.java | 2 + .../ghidra/core/tasks/StartAnalysisTask.java | 6 +- .../devplugin/RevEngMetadataProvider.java | 6 +- .../plugins/AnalysisManagementPlugin.java | 33 ++- .../plugins/BinarySimilarityPlugin.java | 6 +- .../toolkit/ghidra/plugins/DevPlugin.java | 2 +- .../ghidra/plugins/ReaiPluginPackage.java | 4 + .../java/ConvertBinSyncArtifactTests.java | 70 +++++- .../ai/reveng/AIDecompilerComponentTest.java | 18 +- .../reveng/PortalAnalysisIntegrationTest.java | 9 +- .../ai/reveng/TestAnalysisLogComponent.java | 7 +- .../ai/reveng/TestUpgradeFromBinaryID.java | 64 +++++ src/test/java/ai/reveng/UnstripTest.java | 23 +- 32 files changed, 402 insertions(+), 411 deletions(-) delete mode 100644 src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/ProcessingLimboApi.java delete mode 100644 src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/SimpleMatchesApi.java delete mode 100644 src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/TypeGenerationMock.java delete mode 100644 src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/InvalidBinaryID.java create mode 100644 src/test/java/ai/reveng/TestUpgradeFromBinaryID.java diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java index b480b69..ccf4260 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java @@ -48,7 +48,7 @@ public class AutoUnstripDialog extends RevEngDialogComponentProvider { private record RenameResult(String virtualAddress, String originalName, String newName) { } - public AutoUnstripDialog(PluginTool tool, GhidraRevengService.ProgramWithBinaryID analysisID) { + public AutoUnstripDialog(PluginTool tool, GhidraRevengService.ProgramWithID analysisID) { super(ReaiPluginPackage.WINDOW_PREFIX + "Auto Unstrip", true); this.analysisID = analysisID.analysisID(); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java index 2dc073b..6ed88e1 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java @@ -105,11 +105,11 @@ public void processEvent(RevEngAIAnalysisStatusChangedEvent event) { } @Override - public void consumeLogs(String logs, GhidraRevengService.ProgramWithBinaryID programWithBinaryID) { - storedLogs.put(programWithBinaryID.program(), logs); + public void consumeLogs(String logs, GhidraRevengService.ProgramWithID programWithID) { + storedLogs.put(programWithID.program(), logs); // If this is the currently active program, update the log display - if (activeProgram == programWithBinaryID.program()) { + if (activeProgram == programWithID.program()) { setLogs(logs); } } @@ -117,12 +117,12 @@ public void consumeLogs(String logs, GhidraRevengService.ProgramWithBinaryID pro class AnalysisMonitoringTask extends Task { - private final GhidraRevengService.ProgramWithBinaryID program; + private final GhidraRevengService.ProgramWithID program; private final AnalysisLogConsumer logConsumer; - public AnalysisMonitoringTask(GhidraRevengService.ProgramWithBinaryID programWithBinaryID, AnalysisLogConsumer logConsumer) { - super(programWithBinaryID.toString(), true, false, false); - program = programWithBinaryID; + public AnalysisMonitoringTask(GhidraRevengService.ProgramWithID programWithID, AnalysisLogConsumer logConsumer) { + super(programWithID.toString(), true, false, false); + program = programWithID; this.logConsumer = logConsumer; } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java index a01e1ab..c3a5e9b 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java @@ -64,7 +64,8 @@ public void mouseClicked(MouseEvent e) { if ("Analysis ID".equals(columnName)) { LegacyAnalysisResult result = recentAnalysesTable.getModel().getRowObject(row); if (result != null) { - ghidraRevengService.openPortalFor(result); + var analysisID = ghidraRevengService.getApi().getAnalysisIDfromBinaryID(result.binary_id()); + ghidraRevengService.openPortalFor(analysisID); } } } @@ -97,7 +98,7 @@ public void mouseClicked(MouseEvent e) { private void pickAnalysis(LegacyAnalysisResult result) { var service = tool.getService(GhidraRevengService.class); var analysisID = service.getApi().getAnalysisIDfromBinaryID(result.binary_id()); - var programWithID = new GhidraRevengService.ProgramWithBinaryID(program, result.binary_id(), analysisID); + var programWithID = new GhidraRevengService.ProgramWithID(program, analysisID); tool.firePluginEvent( new RevEngAIAnalysisStatusChangedEvent( diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/AnalysisLogConsumer.java b/src/main/java/ai/reveng/toolkit/ghidra/core/AnalysisLogConsumer.java index e1c015f..91891ed 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/AnalysisLogConsumer.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/AnalysisLogConsumer.java @@ -4,5 +4,5 @@ public interface AnalysisLogConsumer { - void consumeLogs(String logs, GhidraRevengService.ProgramWithBinaryID programWithBinaryID); + void consumeLogs(String logs, GhidraRevengService.ProgramWithID programWithID); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java b/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java index 49ac0ae..4bbb793 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java @@ -21,38 +21,35 @@ */ public class RevEngAIAnalysisStatusChangedEvent extends PluginEvent { private final AnalysisStatus status; - private final GhidraRevengService.ProgramWithBinaryID programWithBinaryID; + private final GhidraRevengService.ProgramWithID programWithID; - public RevEngAIAnalysisStatusChangedEvent(String sourceName, GhidraRevengService.ProgramWithBinaryID programWithBinaryID, AnalysisStatus status) { + public RevEngAIAnalysisStatusChangedEvent(String sourceName, GhidraRevengService.ProgramWithID programWithID, AnalysisStatus status) { super(sourceName, "RevEngAI Analysis Finished"); - if (status == null || programWithBinaryID == null) { + if (status == null || programWithID == null) { throw new IllegalArgumentException("args cannot be null"); } this.status = status; - this.programWithBinaryID = programWithBinaryID; + this.programWithID = programWithID; } public AnalysisStatus getStatus() { return status; } - public GhidraRevengService.ProgramWithBinaryID getProgramWithBinaryID() { - return programWithBinaryID; + public GhidraRevengService.ProgramWithID getProgramWithBinaryID() { + return programWithID; } public Program getProgram() { - return programWithBinaryID.program(); + return programWithID.program(); } - public BinaryID getBinaryID() { - return programWithBinaryID.binaryID(); - } @Override public String toString() { return "RevEngAIAnalysisStatusChangedEvent{" + "status=" + status + - ", programWithBinaryID=" + programWithBinaryID + + ", programWithBinaryID=" + programWithID + '}'; } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java index 0b70b9c..84d9a7c 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java @@ -48,6 +48,8 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage.OPTION_KEY_ANALYSIS_ID; + /** * Implements a Ghidra compatible interface on top of the RevEngAI REST API @@ -65,7 +67,7 @@ public class GhidraRevengService { private static final String REAI_FUNCTION_PROP_MAP = "RevEngAI_FunctionID_Map"; private static final String REAI_FUNCTION_MANGLED_MAP = "RevEngAI_FunctionMangledNames_Map"; - private Map statusCache = new HashMap<>(); + private Map statusCache = new HashMap<>(); private TypedApiInterface api; private ApiInfo apiInfo; @@ -94,25 +96,32 @@ public URI getServer() { return this.apiInfo.hostURI(); } - public AnalysedProgram registerFinishedAnalysisForProgram(ProgramWithBinaryID programWithBinaryID, TaskMonitor monitor) { - var status = api.status(programWithBinaryID.analysisID); + public AnalysedProgram registerFinishedAnalysisForProgram(ProgramWithID programWithID, TaskMonitor monitor) { + var status = status(programWithID); if (!status.equals(AnalysisStatus.Complete)){ throw new IllegalStateException("Analysis %s is not complete yet, current status: %s" - .formatted(programWithBinaryID.analysisID(), status)); + .formatted(programWithID.analysisID(), status)); } - statusCache.put(programWithBinaryID.binaryID(), AnalysisStatus.Complete); + statusCache.put(programWithID.analysisID, AnalysisStatus.Complete); - var analysedProgram = associateFunctionInfo(programWithBinaryID); + var analysedProgram = associateFunctionInfo(programWithID); pullFunctionInfoFromAnalysis(analysedProgram, monitor); return analysedProgram; } - public ProgramWithBinaryID addBinaryIDtoProgramOptions(Program program, BinaryID binID){ + @Deprecated + public ProgramWithID addBinaryIDtoProgramOptions(Program program, BinaryID binID){ + var analysisID = api.getAnalysisIDfromBinaryID(binID); + return addAnalysisIDtoProgramOptions(program, analysisID); + } + + /// This method is only for mocking purposes + public ProgramWithID addAnalysisIDtoProgramOptions(Program program, AnalysisID analysisID){ var transactionId = program.startTransaction("Associate Binary ID with Program"); program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY) - .setLong(ReaiPluginPackage.OPTION_KEY_BINID, binID.value()); + .setLong(OPTION_KEY_ANALYSIS_ID, analysisID.id()); program.endTransaction(transactionId, true); - return new ProgramWithBinaryID(program, binID, api.getAnalysisIDfromBinaryID(binID)); + return new ProgramWithID(program, analysisID); } /** @@ -122,26 +131,62 @@ public ProgramWithBinaryID addBinaryIDtoProgramOptions(Program program, BinaryID * @param program * @return */ + @Deprecated public Optional getBinaryIDFor(Program program) { - Optional binID; - try { - binID = getBinaryIDfromOptions(program); - } catch (InvalidBinaryID e) { - Msg.error(this, "Invalid Binary ID found in program options: %s".formatted(e.getMessage())); - return Optional.empty(); - } - return binID; - + return getBinaryIDfromOptions(program); } - public Optional getAnalysisIDFor(Program program){ - return getBinaryIDFor(program).map(binID -> api.getAnalysisIDfromBinaryID(binID)); + private Optional getAnalysisIDFor(Program program){ + var optAnalysisID = getAnalysisIDFromOptions(program); + if (optAnalysisID.isPresent()){ + return optAnalysisID; + } + // Fallback to getting it from the BinaryID, if one exists + var legacyBinaryID = getBinaryIDFor(program); + if (legacyBinaryID.isPresent()) { + // We have a legacy binary ID, upgrade to AnalysisID + var analysisID = api.getAnalysisIDfromBinaryID(legacyBinaryID.get()); + addAnalysisIDtoProgramOptions(program, analysisID); + program.withTransaction("Remove legacy BinaryID from program options", () -> + program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY) + .setLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID) + ); + return Optional.of(analysisID); + } + return Optional.empty(); } + private Optional getAnalysisIDFromOptions( + Program program + ) { + long bid = program.getOptions( + ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(OPTION_KEY_ANALYSIS_ID, + ReaiPluginPackage.INVALID_ANALYSIS_ID); + if (bid == ReaiPluginPackage.INVALID_ANALYSIS_ID) { + return Optional.empty(); + } + var analysisID = new AnalysisID((int) bid); + if (!statusCache.containsKey(analysisID)) { + // Check that it's really valid in the context of the currently configured API + try { + var status = api.status(analysisID); + statusCache.put(analysisID, status); + } catch (APIAuthenticationException | ApiException e) { + Msg.showError(this, null, "Invalid Analysis ID", + ("The Analysis ID %s stored in the program options is invalid for the currently configured RevEng.AI server %s. " + + "This could be an intermittent error, or you switched the servers") + .formatted(analysisID, this.apiInfo.hostURI()), e); + return Optional.empty(); + } + // Now it's certain that it is a valid binary ID + } + return Optional.of(analysisID); + } - public Optional getBinaryIDfromOptions( + @Deprecated + private Optional getBinaryIDfromOptions( Program program - ) throws InvalidBinaryID { + ) { long bid = program.getOptions( ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID); @@ -149,16 +194,21 @@ public Optional getBinaryIDfromOptions( return Optional.empty(); } var binID = new BinaryID((int) bid); - if (!statusCache.containsKey(binID)) { - // Check that it's really valid in the context of the currently configured API - try { - var status = api.status(binID); - statusCache.put(binID, status); - } catch (APIAuthenticationException | ApiException e) { - throw new InvalidBinaryID(binID, this.apiInfo); - } - // Now it's certain that it is a valid binary ID + // Check that it's really valid in the context of the currently configured API + AnalysisStatus status; + try { + status = api.status(binID); + } catch (APIAuthenticationException | ApiException e) { + Msg.showError(this, null, "Invalid Binary ID", + ("The Binary ID %s stored in the program options is invalid for the currently configured RevEng.AI server %s. " + + "This could be an intermittent error, or you switched servers") + .formatted(binID, this.apiInfo.hostURI()), e); + return Optional.empty(); } + var analysisID = api.getAnalysisIDfromBinaryID(binID); + statusCache.put(analysisID, status); + + // Now it's certain that it is a valid binary ID return Optional.of(binID); } @@ -167,17 +217,13 @@ public Optional getBinaryIDfromOptions( /// This method should only concern itself with associating the FunctionID with the Ghidra Function /// This property is immutable within an Analysis: The function ID will never change unless an entirely different /// analysis is associated with the program - /// Other function information like the name and signature should be loaded in [#pullFunctionInfoFromAnalysis(ProgramWithBinaryID,TaskMonitor)] + /// Other function information like the name and signature should be loaded in [#pullFunctionInfoFromAnalysis(AnalysedProgram ,TaskMonitor)] /// because this information can change on the server, and thus needs a dedicated method to refresh it - private AnalysedProgram associateFunctionInfo(ProgramWithBinaryID knownProgram) { - var binID = knownProgram.binaryID(); + private AnalysedProgram associateFunctionInfo(ProgramWithID knownProgram) { + var analysisID = knownProgram.analysisID(); var program = knownProgram.program(); List functionInfo = null; - try { - functionInfo = api.getFunctionInfo(binID); - } catch (ApiException e) { - throw new RuntimeException(e); - } + functionInfo = api.getFunctionInfo(analysisID); var transactionID = program.startTransaction("Associate Function Info"); // Create the FunctionID map @@ -227,7 +273,7 @@ private AnalysedProgram associateFunctionInfo(ProgramWithBinaryID knownProgram) program.endTransaction(transactionID, true); - var analysedProgram = new AnalysedProgram(program, binID, api.getAnalysisIDfromBinaryID(binID)); + var analysedProgram = new AnalysedProgram(program, analysisID); AtomicInteger ghidraFunctionCount = new AtomicInteger(); program.getFunctionManager().getFunctions(true).forEach( func -> { @@ -261,7 +307,7 @@ private AnalysedProgram associateFunctionInfo(ProgramWithBinaryID knownProgram) /// * the type signature of the function /// /// It assumes that the initial load already happened, i.e. the functions have an associated FunctionID already. - /// The initial association happens in {@link #associateFunctionInfo(ProgramWithBinaryID)} + /// The initial association happens in {@link #associateFunctionInfo(ProgramWithID)} /// public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMonitor monitor) { var transactionId = analysedProgram.program().startTransaction("RevEng.AI: Pull Function Info from Analysis"); @@ -437,40 +483,29 @@ public Optional getFunctionFor(FunctionInfo functionInfo, Program prog return Optional.ofNullable(func); } + @Deprecated public List searchForHash(BinaryHash hash){ return api.search(hash); } - public boolean isKnownProgram(Program program){ - var storedBinID = program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID); - return storedBinID != ReaiPluginPackage.INVALID_BINARY_ID; - } - public void removeProgramAssociation(Program program){ - BinaryID binID; - try { - var maybebinID = getBinaryIDfromOptions(program); - if (maybebinID.isEmpty()){ - Msg.warn(this, "No binary ID found for program, cannot remove association"); - return; - } - binID = maybebinID.get(); - } catch (InvalidBinaryID e) { - // The program has an invalid binary ID, which can happen if the server was changed - // This is a very good reason to remove the association, so we unpack the id from the error - binID = e.getBinaryID(); - } // Clear all function ID data - program.getUsrPropertyManager().removePropertyMap(REAI_FUNCTION_PROP_MAP); program.getUsrPropertyManager().removePropertyMap(REAI_FUNCTION_MANGLED_MAP); - statusCache.remove(binID); - program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY).setLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID); + var reaiOptions = program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY); + reaiOptions.setLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID); + reaiOptions.setLong(OPTION_KEY_ANALYSIS_ID, ReaiPluginPackage.INVALID_ANALYSIS_ID); + // Clear the entire cache. Getting the correct ID is not worth the effort in terms of edge cases to handle + // because this method should still work even if the analysis ID or binary ID that was associated is invalid + statusCache.clear(); + } - /// @deprecated use {@link GhidraRevengService#getAnalysedProgram(Program)} instead - @Deprecated - public boolean isProgramAnalysed(Program program){ + /// This method is private to the service, because it only concerns itself with how the service determines + /// this internally + /// Plugin code that wants to know if a program is known should use {@link #getAnalysedProgram(Program)} and check + /// if the result is present + private boolean isProgramAnalysed(Program program){ return program.getUsrPropertyManager().getLongPropertyMap(REAI_FUNCTION_PROP_MAP) != null && program.getUsrPropertyManager().getStringPropertyMap(REAI_FUNCTION_MANGLED_MAP) != null; } @@ -531,6 +566,7 @@ public BinaryHash upload(Path path) { } } + @Deprecated public AnalysisStatus pollStatus(BinaryID bid) { try { return api.status(bid); @@ -539,6 +575,22 @@ public AnalysisStatus pollStatus(BinaryID bid) { } } + /// Use this method if you just have an AnalysisID and it is not clear yet if it can be accessed + public AnalysisStatus pollStatus(AnalysisID id) throws ApiException { + return api.status(id); + } + + public AnalysisStatus status(ProgramWithID program) { + try { + return api.status(program.analysisID()); + } catch (ApiException e) { + // This should never happen given that `ProgramWithID` guarantees a valid analysis ID + throw new RuntimeException(e); + } + } + + + public String decompileFunctionViaAI(FunctionWithID functionWithID, TaskMonitor monitor, AIDecompilationdWindow window) { monitor.setMaximum(100 * 50); // Check if there is an existing process already, because the trigger API will fail with 400 if there is @@ -594,32 +646,32 @@ public AnalysedProgram analyse(Program program, AnalysisOptionsBuilder analysisO var finalStatus = waitForFinishedAnalysis(monitor, programWithBinaryID, null, null); // TODO: Check final status for errors, and do something appropriate on failure var analysedProgram = registerFinishedAnalysisForProgram(programWithBinaryID, monitor); + if (getKnownProgram(program).isEmpty()){ + throw new IllegalStateException("Program is not known after finished analysis. Something seriously went wrong."); + } return analysedProgram; } - /// Get the ProgramWithBinaryID for a known program + /// Get the {@link ProgramWithID} for a known program /// This only guarantees an associated analysis, not that it is finished - public Optional getKnownProgram(Program program) { - return getBinaryIDFor(program).map(binID -> { - var analysisID = api.getAnalysisIDfromBinaryID(binID); - return new ProgramWithBinaryID(program, binID, analysisID); - } - ); + public Optional getKnownProgram(Program program) { + var analysisID = getAnalysisIDFor(program); + return analysisID.map(id -> new ProgramWithID(program, id)); } - /// {@link GhidraRevengService#isProgramAnalysed(Program)} and if so, return an AnalysedProgram + /// Get the {@link AnalysedProgram} for a known and analysed program public Optional getAnalysedProgram(Program program) { var kProg = getKnownProgram(program); if (kProg.isEmpty()){ return Optional.empty(); } if (isProgramAnalysed(kProg.get().program())){ - return Optional.of(new AnalysedProgram(kProg.get().program(), kProg.get().binaryID(), kProg.get().analysisID())); + return Optional.of(new AnalysedProgram(kProg.get().program(), kProg.get().analysisID())); } return Optional.empty(); } - public Optional getFunctionSignatureArtifact(ProgramWithBinaryID program, FunctionID functionID) { + public Optional getFunctionSignatureArtifact(ProgramWithID program, FunctionID functionID) { return api.getFunctionDataTypes(program.analysisID(), functionID).flatMap(FunctionDataTypeStatus::data_types); } @@ -820,11 +872,15 @@ public void openPortalFor(FunctionID f){ } public void openPortalFor(AnalysisResult analysisResult) { - openPortal("analyses", String.valueOf(analysisResult.analysisID().id())); + openPortalFor(analysisResult.analysisID()); + } + + public void openPortalFor(ProgramWithID programWithID) { + openPortalFor(programWithID.analysisID()); } - public void openPortalFor(LegacyAnalysisResult analysisResult) { - openPortal("analyses", String.valueOf(analysisResult.binary_id().value())); + public void openPortalFor(AnalysisID analysisID) { + openPortal("analyses", String.valueOf(analysisID.id())); } public void openPortal(String... subPath) { @@ -889,7 +945,7 @@ public List getActiveAnalysisIDsFilter() { */ public AnalysisStatus waitForFinishedAnalysis( TaskMonitor monitor, - ProgramWithBinaryID programWithID, + ProgramWithID programWithID, @Nullable AnalysisLogConsumer logger, @Nullable PluginTool tool @@ -899,7 +955,7 @@ public AnalysisStatus waitForFinishedAnalysis( // TODO: In the future this can be made smarter and e.g. wait longer if the analysis log hasn't changed AnalysisStatus lastStatus = null; while (true) { - AnalysisStatus currentStatus = this.api.status(programWithID.analysisID()); + AnalysisStatus currentStatus = this.status(programWithID); if (currentStatus != AnalysisStatus.Queued) { // Analysis log endpoint only starts to return data after the analysis is processing String logs = this.getAnalysisLog(programWithID.analysisID()); @@ -930,9 +986,9 @@ public AnalysisStatus waitForFinishedAnalysis( } } - public ProgramWithBinaryID startAnalysis(Program program, AnalysisOptionsBuilder analysisOptionsBuilder) throws ApiException { - var binaryID = api.analyse(analysisOptionsBuilder); - return addBinaryIDtoProgramOptions(program, binaryID); + public ProgramWithID startAnalysis(Program program, AnalysisOptionsBuilder analysisOptionsBuilder) throws ApiException { + var analysisID = api.analyse(analysisOptionsBuilder); + return addAnalysisIDtoProgramOptions(program, analysisID); } public Map getNameScores(java.util.Collection values) { @@ -1039,9 +1095,8 @@ public void batchRenameFunctions(FunctionsListRename functionsList) throws ApiEx /// Old Helper Datatype that encapsulates a Ghidra program with a binary ID and analysis ID /// This only guarantees that the program has an associated analysis, but not that the analysis is finished /// The id of this should also be stored in the program options, but this is currently not enforced yet to allow easier testing - public record ProgramWithBinaryID( + public record ProgramWithID( Program program, - BinaryID binaryID, AnalysisID analysisID ){} @@ -1056,18 +1111,16 @@ public record ProgramWithBinaryID( public static class AnalysedProgram { private final Program program; - private final BinaryID binaryID; private final AnalysisID analysisID; - /// The constructor is private to enforce the use of the static factory method and from the outer class + /// The constructor is private to enforce that only the GhidraRevengService class + /// can create instances of this class, ensuring the guarantees hold private AnalysedProgram( Program program, - BinaryID binaryID, AnalysisID analysisID ) { this.program = program; - this.binaryID = binaryID; this.analysisID = analysisID; } @@ -1088,13 +1141,6 @@ private LongPropertyMap getFunctionIDPropertyMap(AnalysedProgram program){ return map; } - public static AnalysedProgram fromAnalysisResult(LegacyAnalysisResult analysisResult, Program program) { - if (analysisResult.status() != AnalysisStatus.Complete) { - throw new IllegalArgumentException("AnalysisResult must have status Complete to create ProgramWithBinaryID"); - } - return new AnalysedProgram(program, analysisResult.binary_id(), analysisResult.analysis_id()); - } - /// Only returns Optional.Empty if the function is not known on the server (e.g. because it's a thunk) public Optional getIDForFunction(Function function) { if (function.getProgram() != this.program){ diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java index 188f1e1..f48ff91 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java @@ -62,6 +62,7 @@ public class TypedApiImplementation implements TypedApiInterface { private final FunctionsAiDecompilationApi functionsAiDecompilationApi; // Cache for binary ID to analysis ID mappings + @Deprecated private final Map binaryToAnalysisCache = new HashMap<>(); // Cache for analysis basic info to avoid repeated API calls @@ -139,6 +140,7 @@ public BinaryHash upload(Path binPath) throws FileNotFoundException, ApiExceptio Not all parameters are required, for example /search?search=sha_256_hash: only searches for binaries and collection with hashes like . */ + @Deprecated public List search(BinaryHash hash) { Map params = new HashMap<>(); params.put("sha256_hash", hash.sha256()); @@ -195,13 +197,13 @@ private JSONObject sendRequest(HttpRequest request) throws APIAuthenticationExce } @Override - public BinaryID analyse(AnalysisOptionsBuilder builder) throws ApiException { - var analysisRequest = builder.toAnalysisCreateRequest(); + public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + var analysisRequest = options.toAnalysisCreateRequest(); var result = this.analysisCoreApi.createAnalysis(analysisRequest); - - return new BinaryID(result.getData().getBinaryId()); + return new AnalysisID(result.getData().getAnalysisId()); } + @Deprecated @Override public AnalysisStatus status(BinaryID binaryID) throws ApiException { var analysisID = this.getAnalysisIDfromBinaryID(binaryID); @@ -212,18 +214,20 @@ public AnalysisStatus status(BinaryID binaryID) throws ApiException { } @Override - public AnalysisStatus status(AnalysisID analysisID) { - var request = requestBuilderForEndpoint("analyses/%s/status".formatted(analysisID.id())) - .GET() - .build(); - return AnalysisStatus.valueOf(sendVersion2Request(request).getJsonData().getString("analysis_status")); + public AnalysisStatus status(AnalysisID analysisID) throws ApiException { + var status = analysisCoreApi.getAnalysisStatus(analysisID.id()); + return AnalysisStatus.valueOf(status.getData().getAnalysisStatus()); } @Override - public List getFunctionInfo(BinaryID binaryID) throws ApiException { - var analysisID = this.getAnalysisIDfromBinaryID(binaryID); + public List getFunctionInfo(AnalysisID analysisID) { - var response = this.analysesResultsMetadataApi.getFunctionsList(analysisID.id(), null, null, null); + BaseResponseAnalysisFunctions response = null; + try { + response = this.analysesResultsMetadataApi.getFunctionsList(analysisID.id(), null, null, null); + } catch (ApiException e) { + throw new RuntimeException("Could not find analysis with ID: " + analysisID.id(), e); + } return response.getData().getFunctions().stream().map(f -> ( new FunctionInfo( @@ -340,6 +344,7 @@ private HttpRequest.Builder requestBuilderForEndpoint(String... endpointPaths){ * @return the analysis id */ @Override + @Deprecated public AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID){ // Check cache first AnalysisID cachedResult = binaryToAnalysisCache.get(binaryID); @@ -505,11 +510,19 @@ public List getNameScores(List matches, Boolea */ @Override public AnalysisResult getInfoForAnalysis(AnalysisID id) { - var request = requestBuilderForEndpoint("analyses", String.valueOf(id.id())) - .GET() - .build(); - var response = sendVersion2Request(request); - return AnalysisResult.fromJSONObject(this, response.getJsonData()); + try { + var response = analysisCoreApi.getAnalysisBasicInfo(id.id()); + var data = response.getData(); + if (data == null) { + throw new RuntimeException("Unexpected null data for analysis ID: " + id.id()); + } + return new AnalysisResult( + id, + data + ); + } catch (ApiException e) { + throw new IllegalArgumentException("Could not find analysis with ID: " + id.id()); + } } /** diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java index d7cae31..12ea84f 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java @@ -3,7 +3,6 @@ import java.io.FileNotFoundException; import java.nio.file.Path; import java.util.List; -import java.util.Map; import java.util.Optional; import ai.reveng.model.*; @@ -29,26 +28,27 @@ */ public interface TypedApiInterface { // Analysis - BinaryID analyse(AnalysisOptionsBuilder binHash) throws ApiException; +// @Deprecated +// BinaryID legacyAnalyse(AnalysisOptionsBuilder binHash) throws ApiException; - - default Object delete(BinaryID binID) { - throw new UnsupportedOperationException("delete not implemented yet"); + default AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + throw new UnsupportedOperationException("analyse not implemented yet"); } - - default AnalysisStatus status(AnalysisID analysisID) { + default AnalysisStatus status(AnalysisID analysisID) throws ApiException { throw new UnsupportedOperationException("status not implemented yet"); } - default List getFunctionInfo(BinaryID binaryID) throws ApiException { + default List getFunctionInfo(AnalysisID analysisID) { throw new UnsupportedOperationException("getFunctionInfo not implemented yet"); } - default List recentAnalyses() { - throw new UnsupportedOperationException("recentAnalyses not implemented yet"); + @Deprecated + default List getFunctionInfo(BinaryID binID) throws ApiException { + return getFunctionInfo(getAnalysisIDfromBinaryID(binID)); } + @Deprecated default AnalysisStatus status(BinaryID binID) throws ApiException { throw new UnsupportedOperationException("status not implemented yet"); }; @@ -56,6 +56,7 @@ default AnalysisStatus status(BinaryID binID) throws ApiException { /** * https://docs.reveng.ai/#/Utility/get_search */ + @Deprecated default List search(BinaryHash hash) { throw new UnsupportedOperationException("search not implemented yet"); } @@ -98,6 +99,7 @@ default Optional getFunctionDataTypes(AnalysisID analysi } + @Deprecated default AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { throw new UnsupportedOperationException("getAnalysisIDfromBinaryID not implemented yet"); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/MockApi.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/MockApi.java index ceb8e29..90f1bf5 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/MockApi.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/MockApi.java @@ -1,20 +1,16 @@ package ai.reveng.toolkit.ghidra.core.services.api.mocks; -import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; -import ai.reveng.toolkit.ghidra.core.services.api.ModelName; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIAuthenticationException; import org.json.JSONObject; -import javax.annotation.Nullable; import java.io.FileNotFoundException; -import java.math.BigDecimal; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.Map; +@Deprecated public class MockApi implements TypedApiInterface { @Override public BinaryHash upload(Path binPath) throws FileNotFoundException { @@ -22,16 +18,7 @@ public BinaryHash upload(Path binPath) throws FileNotFoundException { } @Override - public Object delete(BinaryID binID) { - return TypedApiInterface.super.delete(binID); - } - - @Override - public List recentAnalyses() { - return TypedApiInterface.super.recentAnalyses(); - } - - @Override + @Deprecated public List search(BinaryHash hash) { if (hash.equals(new BinaryHash("b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6"))) { return List.of(new LegacyAnalysisResult( @@ -55,11 +42,6 @@ public AnalysisStatus status(BinaryID binID) { return AnalysisStatus.Complete; } - @Override - public BinaryID analyse(AnalysisOptionsBuilder binHash) { - return new BinaryID(17920); - } - @Override public String getAnalysisLogs(AnalysisID analysisID) { return ""; @@ -75,7 +57,7 @@ public void renameFunction(FunctionID id, String newName) { } @Override - public List getFunctionInfo(BinaryID binaryID) { + public List getFunctionInfo(AnalysisID analysisID) { var r = """ { "success": true, diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/ProcessingLimboApi.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/ProcessingLimboApi.java deleted file mode 100644 index 8d9a66f..0000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/ProcessingLimboApi.java +++ /dev/null @@ -1,35 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.mocks; - -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; - -/** - * An API mock to simulate a server that is never finished with processing a binary. - */ -public class ProcessingLimboApi extends UnimplementedAPI { - - private final AnalysisStatus status; - - private int logCounter = 0; - - public ProcessingLimboApi() { - super(); - status = AnalysisStatus.Processing; - } - - public ProcessingLimboApi(AnalysisStatus status) { - super(); - this.status = status; - } - - @Override - public AnalysisStatus status(BinaryID binID) { - return status; - } - - @Override - public String getAnalysisLogs(AnalysisID analysisID) { - return "Analysis Logs: " + logCounter++; - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/SimpleMatchesApi.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/SimpleMatchesApi.java deleted file mode 100644 index aaa5466..0000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/SimpleMatchesApi.java +++ /dev/null @@ -1,15 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.mocks; - -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; - -/** - * An API mock to simulate a server that is never finished with processing a binary. - */ -public class SimpleMatchesApi extends UnimplementedAPI { - @Override - public AnalysisStatus status(BinaryID binID) { - return AnalysisStatus.Complete; - } - -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/TypeGenerationMock.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/TypeGenerationMock.java deleted file mode 100644 index 3db24b2..0000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/TypeGenerationMock.java +++ /dev/null @@ -1,66 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.mocks; - -import ai.reveng.toolkit.ghidra.core.services.api.types.*; - -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -public class TypeGenerationMock extends UnimplementedAPI { - - Set generatedFunctions = new HashSet<>(); - @Override - public DataTypeList generateFunctionDataTypes(AnalysisID analysisID, List functionIDS) { - var statuses = functionIDS.stream() - .map(id -> new FunctionDataTypeStatus( - false, - Optional.empty(), - "UNKNOWN", - null, - id - )) - .toList(); - return new DataTypeList( - functionIDS.size(), 0, statuses.toArray(new FunctionDataTypeStatus[0]) - ); - } - - @Override - public DataTypeList getFunctionDataTypes(List functionIDS) { - for (FunctionID functionID : functionIDS) { - if (generatedFunctions.contains(functionID)) continue; - generatedFunctions.add(functionID); - break; - } - - var statuses = functionIDS.stream() - .map(id -> new FunctionDataTypeStatus( - generatedFunctions.contains(id), - Optional.empty(), - generatedFunctions.contains(id) ? "completed" : "UNKNOWN", - null, - id - )) - .toList(); - - return new DataTypeList( - functionIDS.size(), 0, statuses.toArray(new FunctionDataTypeStatus[0]) - ); - } - - @Override - public FunctionDetails getFunctionDetails(FunctionID id) { - return new FunctionDetails( - id, - "placeholder_for_%s".formatted(id), - 0L, - 10L, - new AnalysisID(1337), - new BinaryID(1337), - "placeholder_for_%s".formatted(id), - new BinaryHash("placeholder_for_%s".formatted(id)), - "demangled_placeholder_for_%s".formatted(id) - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java index cff3521..b31821e 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java @@ -28,11 +28,6 @@ protected AnalysisStatus getNextStatus(AnalysisStatus previousStatus) { }; } - @Override - public BinaryID analyse(AnalysisOptionsBuilder binHash) { - return new BinaryID(1337); - } - @Override public String getAnalysisLogs(AnalysisID analysisID) { return "ANALYSIS LOGS"; @@ -61,9 +56,4 @@ public BinaryHash upload(Path binPath) { } } - - @Override - public AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { - return new AnalysisID(1337); - } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisID.java index 9441dce..2cdacdc 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisID.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisID.java @@ -1,4 +1,7 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +/// This is a special box type for an analysis ID +/// It enforces that the integer is specifically an analysis ID, +/// and it implies that the user has access to this ID public record AnalysisID(int id) { } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java index d7eb50e..a0e58ed 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java @@ -1,33 +1,23 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.invoker.ApiException; +import ai.reveng.model.BaseResponseBasic; +import ai.reveng.model.Basic; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; -/* - */ +/// This is a remnant of an older class that contained the analysis result data directly. +/// Now it's a wrapper around the generated Basic class with some shim methods for convenience. public record AnalysisResult( AnalysisID analysisID, - String binary_name, - String creation, - Integer model_id, - String model_name, - BinaryHash sha_256_hash, - AnalysisStatus status -// AnalysisScope analysis_scope, + Basic base_response_basic ) { - public static AnalysisResult fromJSONObject(TypedApiInterface api, JSONObject json) { -// throw new UnsupportedOperationException("fromJSONObject not implemented yet"); - var analysisId = new AnalysisID(json.getInt("analysis_id")); - var analysisStatus = api.status(analysisId); - return new AnalysisResult( - analysisId, - json.getString("binary_name"), - json.getString("creation"), - json.has("model_id") ? json.getInt("model_id") : null, - json.getString("model_name"), - new BinaryHash(json.getString("sha_256_hash")), - analysisStatus - ); + public BinaryHash sha_256_hash() { + return new BinaryHash(base_response_basic().getSha256Hash()); + } + + public String binary_name() { + return base_response_basic.getBinaryName(); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryID.java index af19892..17a6174 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryID.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryID.java @@ -6,6 +6,7 @@ * They are called binary ID in the API doc, but they should be thought of as _analysis_ ids * for a single binary (identified by hash), there can be multiple analyses, which are distinguished by this ID */ +@Deprecated public record BinaryID(int value) implements Comparable { public BinaryID { if (value < 0) { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java index 51fbdc0..4f1c825 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java @@ -11,7 +11,6 @@ public record FunctionDetails( Long functionVaddr, Long functionSize, AnalysisID analysisId, - BinaryID binaryId, String binaryName, BinaryHash sha256Hash, String demangledName @@ -25,7 +24,6 @@ public static FunctionDetails fromServerResponse(FunctionsDetailResponse respons response.getFunctionVaddr(), response.getFunctionSize().longValue(), new AnalysisID(response.getAnalysisId()), - new BinaryID(response.getBinaryId()), response.getBinaryName(), new BinaryHash(response.getSha256Hash()), response.getFunctionName() diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionMatch.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionMatch.java index 171f8ce..66894fe 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionMatch.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionMatch.java @@ -9,7 +9,6 @@ * @param nearest_neighbor_function_name * @param nearest_neighbor_binary_name * @param nearest_neighbor_sha_256_hash - * @param nearest_neighbor_binary_id * @param nearest_neighbor_debug * @param similarity */ @@ -19,7 +18,6 @@ public record FunctionMatch( String nearest_neighbor_function_name, String nearest_neighbor_binary_name, BinaryHash nearest_neighbor_sha_256_hash, - BinaryID nearest_neighbor_binary_id, Boolean nearest_neighbor_debug, double similarity ) { @@ -30,7 +28,6 @@ public static FunctionMatch fromJSONObject(JSONObject json) { json.getString("nearest_neighbor_function_name"), json.getString("nearest_neighbor_binary_name"), new BinaryHash(json.getString("nearest_neighbor_sha_256_hash")), - new BinaryID(json.getInt("nearest_neighbor_binary_id")), json.has("nearest_neighbor_debug") ? json.getBoolean("nearest_neighbor_debug") : null, // This is called confidence for legacy reasons, but it is actually the similarity json.getDouble("confidence") diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/InvalidBinaryID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/InvalidBinaryID.java deleted file mode 100644 index cc7453d..0000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/InvalidBinaryID.java +++ /dev/null @@ -1,21 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - - - -/** - * Exception thrown when a binary ID is invalid or not accessible under a certain config - */ -public class InvalidBinaryID extends Exception{ - private final BinaryID binaryID; - private final ApiInfo config; - - public InvalidBinaryID(BinaryID binaryID, ApiInfo config) { - super("Binary ID " + binaryID + " is invalid under config " + config); - this.binaryID = binaryID; - this.config = config; - } - - public BinaryID getBinaryID() { - return binaryID; - } -}; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java index c2583e4..bd1118f 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java @@ -7,8 +7,10 @@ {"analysis_scope":"PRIVATE","binary_id":27665,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:57:18 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27664,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:55:28 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27663,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:53:33 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27662,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:32:34 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27661,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:24:44 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27660,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:23:54 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27633,"binary_name":"true","creation":"Thu, 18 Apr 2024 17:34:42 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27632,"binary_name":"true","creation":"Thu, 18 Apr 2024 17:33:36 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27631,"binary_name":"ls","creation":"Thu, 18 Apr 2024 17:17:48 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"2e7ef3da2b295c77820a0782b00c9d607cd48d5c8f7458a76b0921bec20a30ae","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27630,"binary_name":"ls","creation":"Thu, 18 Apr 2024 17:16:48 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"2e7ef3da2b295c77820a0782b00c9d607cd48d5c8f7458a76b0921bec20a30ae","status":"Error"}]} */ +@Deprecated public record LegacyAnalysisResult( AnalysisID analysis_id, + @Deprecated BinaryID binary_id, String binary_name, String creation, diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/tasks/StartAnalysisTask.java b/src/main/java/ai/reveng/toolkit/ghidra/core/tasks/StartAnalysisTask.java index 872aee3..8e4d41a 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/tasks/StartAnalysisTask.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/tasks/StartAnalysisTask.java @@ -48,9 +48,9 @@ public void run(TaskMonitor monitor) throws CancelledException { monitor.setMessage("Sending Analysis Request"); - GhidraRevengService.ProgramWithBinaryID programWithBinaryID; + GhidraRevengService.ProgramWithID programWithID; try { - programWithBinaryID = reService.startAnalysis(program, options); + programWithID = reService.startAnalysis(program, options); } catch (ApiException e) { monitor.setMessage("Analysis Request Failed"); return; @@ -58,7 +58,7 @@ public void run(TaskMonitor monitor) throws CancelledException { tool.firePluginEvent(new RevEngAIAnalysisStatusChangedEvent( "StartAnalysisTask", - programWithBinaryID, + programWithID, AnalysisStatus.Queued) ); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java b/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java index 758fd2c..86f6e77 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java @@ -17,7 +17,6 @@ public class RevEngMetadataProvider extends ComponentProviderAdapter { private final JPanel panel; private final JTextField serverField; - BinaryID binaryID; AnalysisID analysisID; FunctionID functionID; Function function; @@ -69,7 +68,6 @@ public JComponent getComponent() { } private void clear() { - binaryID = null; analysisID = null; functionID = null; function = null; @@ -82,8 +80,7 @@ public void setProgram(Program program) { var kProg = api.getKnownProgram(program); kProg.ifPresent(p -> { - binaryID = p.binaryID(); - analysisID = api.getApi().getAnalysisIDfromBinaryID(binaryID); + analysisID = p.analysisID(); }); refresh(); @@ -93,7 +90,6 @@ public void setProgram(Program program) { * Updates the component with the latest information */ private void refresh() { - binaryIDField.setText(binaryID == null ? "" : binaryID.toString()); analysisIDField.setText(analysisID == null ? "" : analysisID.toString()); functionIDField.setText(functionID == null ? "" : functionID.toString()); functionField.setText(function == null ? "" : function.toString()); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java index f2bf6e3..2573bf7 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java @@ -130,10 +130,7 @@ private void setupActions() { // Disable the action if no program is open return false; } - boolean isKnown = revengService.isKnownProgram(currentProgram); - boolean shouldEnable = !isKnown; - - return shouldEnable; + return revengService.getKnownProgram(currentProgram).isEmpty(); }) .onAction(context -> { var program = tool.getService(ProgramManager.class).getCurrentProgram(); @@ -177,8 +174,7 @@ private void setupActions() { if (currentProgram == null) { return false; } - boolean isKnown = revengService.isKnownProgram(currentProgram); - return !isKnown; + return revengService.getKnownProgram(currentProgram).isEmpty(); }) .onAction(context -> { var currentProgram = tool.getService(ProgramManager.class).getCurrentProgram(); @@ -200,12 +196,12 @@ private void setupActions() { // Disable the action if no program is open return false; } - return revengService.isKnownProgram(currentProgram); + return revengService.getKnownProgram(currentProgram).isPresent(); }) .onAction(context -> { var program = tool.getService(ProgramManager.class).getCurrentProgram(); - var analysisID = this.revengService.getAnalysisIDFor(program); - var displayText = analysisID.map(id -> "analysis " + id.id()).get(); + var knownProgram = this.revengService.getKnownProgram(program); + var displayText = knownProgram.map(p -> "analysis " + p.analysisID().id()).orElseThrow(); var result = OptionDialog.showOptionDialogWithCancelAsDefaultButton( tool.getToolFrame(), @@ -235,18 +231,17 @@ private void setupActions() { // Disable the action if no program is open return false; } - return revengService.isKnownProgram(currentProgram); + return revengService.getKnownProgram(currentProgram).isPresent(); }) .onAction(context -> { var currentProgram = tool.getService(ProgramManager.class).getCurrentProgram(); - var binID = revengService.getBinaryIDFor(currentProgram).orElseThrow(); - var analysisID = revengService.getApi().getAnalysisIDfromBinaryID(binID); - var logs = revengService.getAnalysisLog(analysisID); + var knownProgram = revengService.getKnownProgram(currentProgram).orElseThrow(); + var logs = revengService.getAnalysisLog(knownProgram.analysisID()); analysisLogComponent.setLogs(logs); - AnalysisStatus status = revengService.pollStatus(binID); + AnalysisStatus status = revengService.status(knownProgram); tool.getService(ReaiLoggingService.class).info("Check Status: " + status); Msg.showInfo(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Check status", - "Status of analysis " + analysisID.id() + ": " + status); + "Status of analysis " + knownProgram + ": " + status); }) .menuPath(new String[] { ReaiPluginPackage.MENU_GROUP_NAME, "Analysis", "Check status" }) .menuGroup(REAI_ANALYSIS_MANAGEMENT_MENU_GROUP, "400") @@ -259,12 +254,12 @@ private void setupActions() { // Disable the action if no program is open return false; } - return revengService.isKnownProgram(currentProgram); + return revengService.getKnownProgram(currentProgram).isPresent(); }) .onAction(context -> { var currentProgram = tool.getService(ProgramManager.class).getCurrentProgram(); - var binID = revengService.getBinaryIDFor(currentProgram).orElseThrow(); - revengService.openPortal("analyses", String.valueOf(binID.value())); + var knownProgram = revengService.getKnownProgram(currentProgram).orElseThrow(); + revengService.openPortalFor(knownProgram); }) .menuPath(new String[] { ReaiPluginPackage.MENU_GROUP_NAME, "Analysis", "View in portal" }) .menuGroup(REAI_ANALYSIS_MANAGEMENT_MENU_GROUP, "400") @@ -287,7 +282,7 @@ protected void programOpened(Program program) { // This can happen if the analysis was started in a previous session but hadn't finished when closing Ghidra // Either the analysis is finished already now, or we want to actively wait for it to finish log.info("Detected known program that hasn't been fully loaded yet: {}", knownProgram.get()); - var status = revengService.getApi().status(knownProgram.get().analysisID()); + var status = revengService.status(knownProgram.get()); switch (status) { case Complete -> { tool.firePluginEvent( diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java index 3fbac91..145583a 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java @@ -77,7 +77,7 @@ protected void locationChanged(ProgramLocation loc) { // If no program, or not attached to a complete analysis, do not trigger location change events var program = loc.getProgram(); - if (program == null || !apiService.isKnownProgram(program) || !apiService.isProgramAnalysed(program)) { + if (program == null || apiService.getAnalysedProgram(program).isEmpty()) { return; } @@ -119,7 +119,7 @@ private void setupActions() { .enabledWhen(context -> { var program = tool.getService(ProgramManager.class).getCurrentProgram(); if (program != null) { - return apiService.isKnownProgram(program); + return apiService.getKnownProgram(program).isPresent(); } else { return false; } @@ -151,7 +151,7 @@ private void setupActions() { .enabledWhen(context -> { var program = tool.getService(ProgramManager.class).getCurrentProgram(); if (program != null) { - return apiService.isKnownProgram(program); + return apiService.getAnalysedProgram(program).isPresent(); } else { return false; } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/DevPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/DevPlugin.java index cd9fd16..918a5b1 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/DevPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/DevPlugin.java @@ -50,7 +50,7 @@ public DevPlugin(PluginTool tool) { .onAction(e -> { GhidraRevengService reAIService = tool.getService(GhidraRevengService.class); var api = reAIService.getApi(); - AnalysisID analysisID = reAIService.getAnalysisIDFor(currentProgram).get(); + AnalysisID analysisID = reAIService.getAnalysedProgram(currentProgram).orElseThrow().analysisID(); var functionMap = reAIService.getFunctionMap(currentProgram); var task = new Task("Generate Signatures", true, true, true) { @Override diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/ReaiPluginPackage.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/ReaiPluginPackage.java index 2c0a5f5..2b819b7 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/ReaiPluginPackage.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/ReaiPluginPackage.java @@ -22,12 +22,16 @@ public class ReaiPluginPackage extends PluginPackage { public static final String OPTION_KEY_HOSTNAME = PREFIX + "Hostname"; public static final String OPTION_KEY_PORTAL_HOSTNAME = PREFIX + "Portal Hostname"; public static final String OPTION_KEY_MODEL = PREFIX + "Model"; + @Deprecated public static final String OPTION_KEY_BINID = PREFIX + "Binary ID"; + public static final String OPTION_KEY_ANALYSIS_ID = PREFIX + "Analysis ID"; public static final String REAI_OPTIONS_CATEGORY = "RevEngAI Options"; + @Deprecated public static final Integer INVALID_BINARY_ID = -1; + public static final Integer INVALID_ANALYSIS_ID = -1; public static final Icon REVENG_16 = ResourceManager.loadImage("images/reveng_16.png"); diff --git a/src/test/java/ConvertBinSyncArtifactTests.java b/src/test/java/ConvertBinSyncArtifactTests.java index f2c8fee..2a622c5 100644 --- a/src/test/java/ConvertBinSyncArtifactTests.java +++ b/src/test/java/ConvertBinSyncArtifactTests.java @@ -1,28 +1,20 @@ import ai.reveng.toolkit.ghidra.binarysimilarity.cmds.ComputeTypeInfoTask; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.V2Response; -import ai.reveng.toolkit.ghidra.core.services.api.mocks.TypeGenerationMock; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionDataTypeStatus; +import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; +import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.services.api.types.DataTypeList; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionID; import ghidra.program.model.data.CategoryPath; import ghidra.program.model.data.DataType; import ghidra.program.model.data.DataTypeDependencyException; import ghidra.program.model.data.Structure; -import ghidra.test.AbstractGhidraHeadlessIntegrationTest; import ghidra.util.Msg; import ghidra.util.exception.CancelledException; import ghidra.util.task.TaskMonitor; -import org.json.JSONObject; import org.junit.Ignore; import org.junit.Test; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -169,4 +161,60 @@ public void testDataTypeGenerationTask() throws CancelledException { } + public static class TypeGenerationMock extends UnimplementedAPI { + + Set generatedFunctions = new HashSet<>(); + @Override + public DataTypeList generateFunctionDataTypes(AnalysisID analysisID, List functionIDS) { + var statuses = functionIDS.stream() + .map(id -> new FunctionDataTypeStatus( + false, + Optional.empty(), + "UNKNOWN", + null, + id + )) + .toList(); + return new DataTypeList( + functionIDS.size(), 0, statuses.toArray(new FunctionDataTypeStatus[0]) + ); + } + + @Override + public DataTypeList getFunctionDataTypes(List functionIDS) { + for (FunctionID functionID : functionIDS) { + if (generatedFunctions.contains(functionID)) continue; + generatedFunctions.add(functionID); + break; + } + + var statuses = functionIDS.stream() + .map(id -> new FunctionDataTypeStatus( + generatedFunctions.contains(id), + Optional.empty(), + generatedFunctions.contains(id) ? "completed" : "UNKNOWN", + null, + id + )) + .toList(); + + return new DataTypeList( + functionIDS.size(), 0, statuses.toArray(new FunctionDataTypeStatus[0]) + ); + } + + @Override + public FunctionDetails getFunctionDetails(FunctionID id) { + return new FunctionDetails( + id, + "placeholder_for_%s".formatted(id), + 0L, + 10L, + new AnalysisID(1337), + "placeholder_for_%s".formatted(id), + new BinaryHash("placeholder_for_%s".formatted(id)), + "demangled_placeholder_for_%s".formatted(id) + ); + } + } } diff --git a/src/test/java/ai/reveng/AIDecompilerComponentTest.java b/src/test/java/ai/reveng/AIDecompilerComponentTest.java index 87a4562..7f7ec1f 100644 --- a/src/test/java/ai/reveng/AIDecompilerComponentTest.java +++ b/src/test/java/ai/reveng/AIDecompilerComponentTest.java @@ -1,6 +1,8 @@ package ai.reveng; +import ai.reveng.invoker.ApiException; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.aidecompiler.AIDecompilationdWindow; +import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.plugins.BinarySimilarityPlugin; @@ -36,7 +38,7 @@ public AnalysisStatus status(AnalysisID analysisID) { } @Override - public List getFunctionInfo(BinaryID binaryID) { + public List getFunctionInfo(AnalysisID analysisID) { return List.of( new FunctionInfo( new FunctionID(1), @@ -83,6 +85,11 @@ public AIDecompilationStatus pollAIDecompileStatus(FunctionID functionID) { } + @Override + public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + return new AnalysisID(1); + } + @Override public boolean triggerAIDecompilationForFunctionID(FunctionID functionID) { return true; @@ -155,7 +162,7 @@ public void testAIDecompFeedbackMechanism() throws Exception { var func2 = builder.createEmptyFunction(null, "0x2000", 10, Undefined.getUndefinedDataType(4)); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); - +// service.getAnalysedProgram(programWithID); env.showTool(programWithID.program()); waitForSwing(); @@ -190,13 +197,18 @@ static class RatingsAPI extends UnimplementedAPI { public RatingsAPI() { } + @Override + public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + return new AnalysisID(1); + } + @Override public AnalysisStatus status(AnalysisID analysisID) { return AnalysisStatus.Complete; } @Override - public List getFunctionInfo(BinaryID binaryID) { + public List getFunctionInfo(AnalysisID analysisID) { return List.of( new FunctionInfo( new FunctionID(1), diff --git a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java index 5624cb9..fd0da28 100644 --- a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java +++ b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java @@ -1,6 +1,5 @@ package ai.reveng; -import ai.reveng.model.FunctionsDetailResponse; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisResultsLoaded; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; @@ -9,7 +8,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.*; import ai.reveng.toolkit.ghidra.plugins.AnalysisManagementPlugin; import ghidra.program.database.ProgramBuilder; -import ghidra.program.model.data.TypeDef; import ghidra.program.model.data.Undefined; import ghidra.program.model.symbol.SourceType; import ghidra.util.task.TaskMonitor; @@ -35,7 +33,7 @@ public void testInfoLoading() throws Exception { var tool = env.getTool(); addMockedService(tool, new UnimplementedAPI() { @Override - public List getFunctionInfo(BinaryID binaryID) { + public List getFunctionInfo(AnalysisID analysisID) { return List.of( new FunctionInfo(new FunctionID(1), "portal_name_demangled", "portal_name_mangled", 0x4000L, 0x100) ); @@ -92,7 +90,6 @@ public FunctionDetails getFunctionDetails(FunctionID id) { 0x4000L, 0x100L, new AnalysisID(1), - new BinaryID(1), "binary_name", new BinaryHash("dummyhash"), "portal_name_demangled" @@ -122,9 +119,9 @@ public FunctionDetails getFunctionDetails(FunctionID id) { waitForSwing(); - var id = new GhidraRevengService.ProgramWithBinaryID(program, new BinaryID(1), new AnalysisID(1)); + var id = new GhidraRevengService.ProgramWithID(program, new AnalysisID(1)); var service = defaultTool.getService(GhidraRevengService.class); - service.addBinaryIDtoProgramOptions(program, id.binaryID()); + service.addAnalysisIDtoProgramOptions(program, id.analysisID()); // Register a listener for the results loaded event, to verify that has been fired later AtomicBoolean receivedResultsLoadedEvent = new AtomicBoolean(false); diff --git a/src/test/java/ai/reveng/TestAnalysisLogComponent.java b/src/test/java/ai/reveng/TestAnalysisLogComponent.java index 633d517..57938c3 100644 --- a/src/test/java/ai/reveng/TestAnalysisLogComponent.java +++ b/src/test/java/ai/reveng/TestAnalysisLogComponent.java @@ -20,13 +20,12 @@ public class TestAnalysisLogComponent extends RevEngMockableHeadedIntegrationTest { - private GhidraRevengService.ProgramWithBinaryID getPlaceHolderID() throws Exception{ + private GhidraRevengService.ProgramWithID getPlaceHolderID() throws Exception{ var builder = new ghidra.program.database.ProgramBuilder("mock", ProgramBuilder._8051, this); // Add an example function var program = builder.getProgram(); - return new GhidraRevengService.ProgramWithBinaryID( + return new GhidraRevengService.ProgramWithID( program, - new BinaryID(1), new AnalysisID(1) ); } @@ -92,7 +91,7 @@ public String getAnalysisLogs(AnalysisID analysisID) { } @Override - public List getFunctionInfo(BinaryID binaryID) { + public List getFunctionInfo(AnalysisID analysisID) { return List.of(); } } diff --git a/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java b/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java new file mode 100644 index 0000000..7ee26c4 --- /dev/null +++ b/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java @@ -0,0 +1,64 @@ +package ai.reveng; + +import ai.reveng.invoker.ApiException; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; +import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; +import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; +import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; +import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; +import ghidra.program.database.ProgramBuilder; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Optional; + +public class TestUpgradeFromBinaryID extends RevEngMockableHeadedIntegrationTest { + + /// Tests the logic for handling a program that has only a binary ID stored in its properties + @Test + public void test() throws Exception { + var builder = new ProgramBuilder("upgrade-test", ProgramBuilder._8051, this); + var tId = builder.getProgram().startTransaction("Set Binary ID"); + builder.getProgram() + .getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY) + .setLong(ReaiPluginPackage.OPTION_KEY_BINID, 1); + builder.getProgram().endTransaction(tId, true); + addMockedService(env.getTool(), new UnimplementedAPI() { + @Override + public AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { + if (binaryID.value() == 1) { + return new AnalysisID(42); + } + return null; + } + + @Override + public AnalysisStatus status(BinaryID binID) throws ApiException { + if (binID.value() == 1) { + return AnalysisStatus.Complete; + } + return AnalysisStatus.Error; + } + }); + var program = builder.getProgram(); + + env.open(program); + + var service = env.getTool().getService(GhidraRevengService.class); + Optional analysisID = service.getKnownProgram(program); + Assert.assertTrue(analysisID.isPresent()); + Assert.assertEquals(42, analysisID.get().analysisID().id()); + // Verify that after opening, the program has the Analysis ID set and the Binary ID removed + Assert.assertEquals(-1, program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(ReaiPluginPackage.OPTION_KEY_BINID, -1)); + Assert.assertEquals(42, program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(ReaiPluginPackage.OPTION_KEY_ANALYSIS_ID, -1)); + + + + + + + + + } +} diff --git a/src/test/java/ai/reveng/UnstripTest.java b/src/test/java/ai/reveng/UnstripTest.java index caa47fa..bc6820c 100644 --- a/src/test/java/ai/reveng/UnstripTest.java +++ b/src/test/java/ai/reveng/UnstripTest.java @@ -1,5 +1,6 @@ package ai.reveng; +import ai.reveng.invoker.ApiException; import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; @@ -37,13 +38,8 @@ public AutoUnstripResponse autoUnstrip(AnalysisID analysisID) { } @Override - public BinaryID analyse(AnalysisOptionsBuilder binHash) { - return new BinaryID(1); - } - - @Override - public AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { - return new AnalysisID(binaryID.value()); + public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + return new AnalysisID(1); } @Override @@ -52,7 +48,7 @@ public AnalysisStatus status(AnalysisID analysisID) { } @Override - public List getFunctionInfo(BinaryID binaryID) { + public List getFunctionInfo(AnalysisID analysisID) { return List.of(new FunctionInfo(new FunctionID(1), "default_function_info_name", "default_function_info_name_mangled",0x1000L, 10)); } }); @@ -117,13 +113,8 @@ public AutoUnstripResponse autoUnstrip(AnalysisID analysisID) { } @Override - public BinaryID analyse(AnalysisOptionsBuilder binHash) { - return new BinaryID(1); - } - - @Override - public AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { - return new AnalysisID(binaryID.value()); + public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + return new AnalysisID(1); } @Override @@ -132,7 +123,7 @@ public AnalysisStatus status(AnalysisID analysisID) { } @Override - public List getFunctionInfo(BinaryID binaryID) { + public List getFunctionInfo(AnalysisID analysisID) { // the function info will return a name, but it will _not_ be the unstripped name return List.of(new FunctionInfo(new FunctionID(1), "default_function_info_name", "default_function_info_name_mangled",0x1000L, 10)); } From c078ae3a44a1258f13c6a356147fdfdda7cee544 Mon Sep 17 00:00:00 2001 From: Florian Magin Date: Wed, 26 Nov 2025 21:21:03 +0100 Subject: [PATCH 07/10] Work around Ghidra 11.2.X weirdness in SignatureSource --- .../ghidra/core/services/api/GhidraRevengService.java | 1 + .../java/ai/reveng/PortalAnalysisIntegrationTest.java | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java index 84d9a7c..05b9513 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java @@ -413,6 +413,7 @@ public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMo functionSignatureMessageOpt.get(), SourceType.ANALYSIS ).applyTo(analysedProgram.program(), monitor); + // For unclear reasons the signature source is not set by the command in Ghidra 11.2.x and lower if (success) { ghidraRenamedFunctions++; } else { diff --git a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java index fd0da28..4b4e1ac 100644 --- a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java +++ b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java @@ -7,6 +7,8 @@ import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.*; import ai.reveng.toolkit.ghidra.plugins.AnalysisManagementPlugin; +import ghidra.framework.Application; +import ghidra.framework.ApplicationVersion; import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; import ghidra.program.model.symbol.SourceType; @@ -147,9 +149,15 @@ public FunctionDetails getFunctionDetails(FunctionID id) { // Check that the function names have been updated to the one returned by the portal assertEquals("portal_name_demangled", exampleFunc.getName()); - assertEquals(SourceType.ANALYSIS, exampleFunc.getSignatureSource()); var signature = exampleFunc.getSignature(true); assertEquals("int portal_name_demangled(char * named_param_1)", signature.getPrototypeString()); + // For unclear reasons the signature source is not set by the command in Ghidra 11.2.x + // So we only test this for Ghidra 11.3 and above + ApplicationVersion version = new ApplicationVersion(Application.getApplicationVersion()); + if (version.compareTo(new ApplicationVersion("11.3")) > 0) { + assertEquals(SourceType.ANALYSIS, exampleFunc.getSignatureSource()); + } + // Check the function ID has been stored in the program options From 5c9ebc39a5966569b19e2b428c4a1ad060fb1b9b Mon Sep 17 00:00:00 2001 From: Florian Magin Date: Fri, 28 Nov 2025 12:13:08 +0100 Subject: [PATCH 08/10] Make some methods private that shouldn't be used outside of GhidraRevengService --- .../services/api/GhidraRevengService.java | 13 +++--------- .../reveng/PortalAnalysisIntegrationTest.java | 20 ++++++++++++++++--- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java index 05b9513..7520162 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java @@ -109,14 +109,7 @@ public AnalysedProgram registerFinishedAnalysisForProgram(ProgramWithID programW return analysedProgram; } - @Deprecated - public ProgramWithID addBinaryIDtoProgramOptions(Program program, BinaryID binID){ - var analysisID = api.getAnalysisIDfromBinaryID(binID); - return addAnalysisIDtoProgramOptions(program, analysisID); - } - - /// This method is only for mocking purposes - public ProgramWithID addAnalysisIDtoProgramOptions(Program program, AnalysisID analysisID){ + private ProgramWithID addAnalysisIDtoProgramOptions(Program program, AnalysisID analysisID){ var transactionId = program.startTransaction("Associate Binary ID with Program"); program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY) .setLong(OPTION_KEY_ANALYSIS_ID, analysisID.id()); @@ -475,7 +468,7 @@ public BiMap getFunctionMap(Program program){ /** * Get the Ghidra Function for a given FunctionInfo if there is one */ - public Optional getFunctionFor(FunctionInfo functionInfo, Program program){ + private Optional getFunctionFor(FunctionInfo functionInfo, Program program){ // These addresses used to be relative, but are now absolute again var defaultAddressSpace = program.getAddressFactory().getDefaultAddressSpace(); var funcAddress = defaultAddressSpace.getAddress(functionInfo.functionVirtualAddress()); @@ -672,7 +665,7 @@ public Optional getAnalysedProgram(Program program) { return Optional.empty(); } - public Optional getFunctionSignatureArtifact(ProgramWithID program, FunctionID functionID) { + public Optional getFunctionSignatureArtifact(AnalysedProgram program, FunctionID functionID) { return api.getFunctionDataTypes(program.analysisID(), functionID).flatMap(FunctionDataTypeStatus::data_types); } diff --git a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java index 4b4e1ac..1adb2f6 100644 --- a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java +++ b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java @@ -1,7 +1,9 @@ package ai.reveng; +import ai.reveng.invoker.ApiException; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisResultsLoaded; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; +import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; @@ -97,6 +99,11 @@ public FunctionDetails getFunctionDetails(FunctionID id) { "portal_name_demangled" ); } + + @Override + public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + return new AnalysisID(1); + } }); var builder = new ProgramBuilder("mock", ProgramBuilder._8051, this); // Add an example function @@ -121,9 +128,12 @@ public FunctionDetails getFunctionDetails(FunctionID id) { waitForSwing(); - var id = new GhidraRevengService.ProgramWithID(program, new AnalysisID(1)); var service = defaultTool.getService(GhidraRevengService.class); - service.addAnalysisIDtoProgramOptions(program, id.analysisID()); + // We start an analysis to get an Analysis ID associated with the program + var id = service.startAnalysis(program, null); + + assert service.getKnownProgram(program).isPresent(); + assert service.getAnalysedProgram(program).isEmpty(); // Register a listener for the results loaded event, to verify that has been fired later AtomicBoolean receivedResultsLoadedEvent = new AtomicBoolean(false); @@ -131,7 +141,7 @@ public FunctionDetails getFunctionDetails(FunctionID id) { receivedResultsLoadedEvent.set(true); }); - // Simulate the analysis status change event + // Simulate the analysis status change event being triggered when the analysis is complete // We have to run this without waiting, otherwise the test case doesn't continue until the dialog is closed runSwing( () -> defaultTool.firePluginEvent( @@ -146,6 +156,10 @@ public FunctionDetails getFunctionDetails(FunctionID id) { waitForSwing(); // Check that we received the results loaded event, i.e. other plugins would have been notified assertTrue(receivedResultsLoadedEvent.get()); + + // Check that an analysed program is now known + assert service.getAnalysedProgram(program).isPresent(); + // Check that the function names have been updated to the one returned by the portal assertEquals("portal_name_demangled", exampleFunc.getName()); From a70d5c7c32ec5af1ce2229c5bc0a065b7193a594 Mon Sep 17 00:00:00 2001 From: Florian Magin Date: Fri, 5 Dec 2025 17:53:46 +0100 Subject: [PATCH 09/10] A lot more cleanup and refactoring, to include datatypes when applying matches --- build.gradle | 2 +- .../binarysimilarity/cmds/ApplyMatchCmd.java | 76 +++-- .../cmds/ComputeTypeInfoTask.java | 11 +- .../aidecompiler/AIDecompilationdWindow.java | 4 +- .../ui/autounstrip/AutoUnstripDialog.java | 144 ++------ .../DataSetControlPanelComponent.java | 6 +- .../AbstractFunctionMatchingDialog.java | 169 ++++------ .../BinaryLevelFunctionMatchingDialog.java | 57 +--- .../FunctionLevelFunctionMatchingDialog.java | 46 +-- .../RecentAnalysesTableModel.java | 5 +- .../recentanalyses/RecentAnalysisDialog.java | 6 +- .../services/api/AnalysisOptionsBuilder.java | 5 +- .../services/api/GhidraRevengService.java | 318 ++++++++++++------ .../services/api/TypedApiImplementation.java | 88 +++-- .../core/services/api/TypedApiInterface.java | 67 +++- .../services/api/mocks/UnimplementedAPI.java | 6 - .../core/services/api/types/AnalysisID.java | 7 - .../services/api/types/AnalysisResult.java | 9 +- .../api/types/AutoUnstripResponse.java | 73 ---- .../core/services/api/types/BinaryHash.java | 20 -- .../core/services/api/types/Collection.java | 9 +- .../core/services/api/types/CollectionID.java | 4 - .../core/services/api/types/DataTypeList.java | 6 +- .../api/types/FunctionDataTypeStatus.java | 8 +- .../services/api/types/FunctionDetails.java | 13 +- .../core/services/api/types/FunctionID.java | 10 - .../core/services/api/types/FunctionInfo.java | 5 +- .../services/api/types/FunctionMatch.java | 50 +-- .../services/api/types/FunctionNameScore.java | 5 +- .../api/types/GhidraFunctionMatch.java | 18 +- .../GhidraFunctionMatchWithSignature.java | 64 +--- .../api/types/LegacyAnalysisResult.java | 16 +- .../services/api/types/LegacyCollection.java | 5 +- .../core/services/api/types/SearchFilter.java | 13 - .../api/types/binsync/FunctionArtifact.java | 1 + .../binsync/FunctionDataTypeMessage.java | 6 +- .../types/binsync/FunctionDependencies.java | 40 ++- .../services/api/types/binsync/Struct.java | 13 + .../api/types/binsync/StructMember.java | 11 + .../api/types/binsync/TypePathAndName.java | 26 +- .../services/api/types/binsync/Typedef.java | 9 + .../devplugin/RevEngMetadataProvider.java | 8 +- .../plugins/BinarySimilarityPlugin.java | 11 +- .../toolkit/ghidra/plugins/DevPlugin.java | 8 +- .../java/ConvertBinSyncArtifactTests.java | 46 +-- .../reveng/PortalAnalysisIntegrationTest.java | 14 +- .../ai/reveng/TestAnalysisLogComponent.java | 3 +- .../java/ai/reveng/TestMockableService.java | 3 - .../ai/reveng/TestUpgradeFromBinaryID.java | 2 +- src/test/java/ai/reveng/UnstripTest.java | 69 ++-- .../api/AnalysisOptionsBuilderTest.java | 15 +- 51 files changed, 798 insertions(+), 832 deletions(-) delete mode 100644 src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisID.java delete mode 100644 src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AutoUnstripResponse.java delete mode 100644 src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryHash.java delete mode 100644 src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/CollectionID.java delete mode 100644 src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionID.java delete mode 100644 src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/SearchFilter.java diff --git a/build.gradle b/build.gradle index 6382770..5f92d5d 100644 --- a/build.gradle +++ b/build.gradle @@ -65,7 +65,7 @@ dependencies { implementation 'org.json:json:20250107' implementation "com.google.guava:guava:33.2.0-jre" implementation group: 'com.fifesoft', name: 'rsyntaxtextarea', version: '3.5.2' - implementation "ai.reveng:sdk:2.42.0" + implementation "ai.reveng:sdk:2.52.1" testImplementation('junit:junit:4.13.1') testRuntimeOnly("org.junit.vintage:junit-vintage-engine:5.8.2") diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ApplyMatchCmd.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ApplyMatchCmd.java index 23468cc..ce46173 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ApplyMatchCmd.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ApplyMatchCmd.java @@ -5,14 +5,10 @@ import ghidra.app.cmd.function.ApplyFunctionSignatureCmd; import ghidra.app.cmd.label.RenameLabelCmd; import ghidra.framework.cmd.Command; -import ghidra.framework.model.DomainObject; -import ghidra.program.model.data.DataTypeDependencyException; -import ghidra.program.model.data.FunctionDefinitionDataType; import ghidra.program.model.listing.CircularDependencyException; import ghidra.program.model.listing.Program; import ghidra.program.model.symbol.Namespace; import ghidra.program.model.symbol.SourceType; -import ghidra.util.Msg; import ghidra.util.exception.DuplicateNameException; import ghidra.util.exception.InvalidInputException; import org.jetbrains.annotations.NotNull; @@ -21,31 +17,61 @@ import static ai.reveng.toolkit.ghidra.plugins.BinarySimilarityPlugin.REVENG_AI_NAMESPACE; -// We can't use Command because that breaks compatibility with Ghidra 11.0 -public class ApplyMatchCmd implements Command { +/// The central command to apply a function match to a {@link Program} +/// It centralizes product design decisions about how to apply a match, like moving it to a namespace, +/// renaming it, applying the signature, etc. +/// It will apply the function signature if available, otherwise it will just rename the function +/// There are various considerations when ap +public class ApplyMatchCmd implements Command { - private final Program program; + private final GhidraRevengService.AnalysedProgram analyzedProgram; private final GhidraFunctionMatchWithSignature match; @Nullable private final GhidraRevengService service; + private final Boolean includeBinaryNameInNameSpace; public ApplyMatchCmd( @Nullable GhidraRevengService service, - @NotNull GhidraFunctionMatchWithSignature match) { + @NotNull GhidraRevengService.AnalysedProgram program, + @NotNull GhidraFunctionMatchWithSignature match, + Boolean includeBinaryNameInNameSpace + + ) { super(); - this.program = match.function().getProgram(); + this.analyzedProgram = program; this.match = match; this.service = service; + this.includeBinaryNameInNameSpace = includeBinaryNameInNameSpace; + } + + private boolean shouldApplyMatch() { + var func = match.function(); + return func != null && + // Do not override user-defined function names + func.getSymbol().getSource() != SourceType.USER_DEFINED && + // Exclude thunks and external functions + !func.isThunk() && + !func.isExternal() && + // Only accept valid names (no spaces) + !match.functionMatch().nearest_neighbor_mangled_function_name().contains(" ") && + !match.functionMatch().nearest_neighbor_function_name().contains(" ") + // Only rename if the function ID is known (boundaries matched) + && analyzedProgram.getIDForFunction(func).map(id -> id.functionID() != match.functionMatch().origin_function_id()).orElse(false); } + @Override - public boolean applyTo(DomainObject obj) { + public boolean applyTo(Program obj) { // Check that this is the same program - if (obj != this.program) { + if (obj != this.analyzedProgram.program()) { throw new IllegalArgumentException("This command can only be applied to the same program as the one provided in the constructor"); } - var libraryNamespace = getLibraryNameSpaceForName(match.functionMatch().nearest_neighbor_binary_name()); + if (!shouldApplyMatch()) { + return false; + } + + var nameSpace = includeBinaryNameInNameSpace ? getLibraryNameSpaceForName(match.functionMatch().nearest_neighbor_binary_name()): getRevEngAINameSpace(); var function = match.function(); try { - function.setParentNamespace(libraryNamespace); + function.setParentNamespace(nameSpace); } catch (DuplicateNameException e) { throw new RuntimeException(e); } catch (InvalidInputException e) { @@ -54,43 +80,36 @@ public boolean applyTo(DomainObject obj) { throw new RuntimeException(e); } - FunctionDefinitionDataType signature = null; - if (match.signature().isPresent()) { - try { - signature = GhidraRevengService.getFunctionSignature(match.signature().get()); - } catch (DataTypeDependencyException e) { - Msg.showError(this, null,"Failed to create function signature", - "Failed to create signature for match function with type %s" - .formatted(match.signature().get().func_types().getSignature()), - e); - } - } + this.analyzedProgram.setMangledNameForFunction(function, match.functionMatch().nearest_neighbor_mangled_function_name()); + var signature = match.signature(); if (signature != null) { var cmd = new ApplyFunctionSignatureCmd(function.getEntryPoint(), signature, SourceType.USER_DEFINED); - cmd.applyTo(program); + cmd.applyTo(analyzedProgram.program()); } else { var renameCmd = new RenameLabelCmd(match.function().getSymbol(), match.functionMatch().name(), SourceType.USER_DEFINED); - renameCmd.applyTo(program); + renameCmd.applyTo(analyzedProgram.program()); } // If we have a service then push the name. If not then it was explicitly not provided, i.e. the caller // is responsible for pushing the names in batch if (service != null) { - service.getApi().renameFunction(match.functionMatch().origin_function_id(), match.functionMatch().name()); + service.getApi().renameFunction(match.functionMatch().origin_function_id(), match.functionMatch().nearest_neighbor_function_name()); } - return false; + return true; } public void applyWithTransaction() { + var program = this.analyzedProgram.program(); var tID = program.startTransaction("RevEng.AI: Apply Match"); var status = applyTo(program); program.endTransaction(tID, status); } private Namespace getRevEngAINameSpace() { + var program = this.analyzedProgram.program(); Namespace revengMatchNamespace = null; try { revengMatchNamespace = program.getSymbolTable().getOrCreateNameSpace( @@ -105,6 +124,7 @@ private Namespace getRevEngAINameSpace() { } private Namespace getLibraryNameSpaceForName(String name) { + var program = this.analyzedProgram.program(); Namespace libraryNamespace = null; try { libraryNamespace = program.getSymbolTable().getOrCreateNameSpace( diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ComputeTypeInfoTask.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ComputeTypeInfoTask.java index d8f4acf..63cfb80 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ComputeTypeInfoTask.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ComputeTypeInfoTask.java @@ -1,6 +1,7 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.cmds; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ghidra.util.exception.CancelledException; import ghidra.util.task.Task; @@ -10,9 +11,7 @@ import java.time.Duration; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import static java.util.stream.Collectors.groupingBy; @@ -23,12 +22,12 @@ * */ public class ComputeTypeInfoTask extends Task { - private final List functions; + private final List functions; private final GhidraRevengService service; private final DataTypeAvailableCallback callback; public ComputeTypeInfoTask(GhidraRevengService service, - List functions, + List functions, @Nullable DataTypeAvailableCallback callback) { super("Computing Type Info", true, true, false); this.service = service; @@ -49,7 +48,7 @@ public void run(TaskMonitor monitor) throws CancelledException { service.getApi().generateFunctionDataTypes(analysisID, functions.stream().map(FunctionDetails::functionId).toList()); }); - Set missing = new HashSet<>(functions); + Set missing = new HashSet<>(functions); while (!missing.isEmpty()) { try { @@ -76,6 +75,6 @@ public void run(TaskMonitor monitor) throws CancelledException { } public interface DataTypeAvailableCallback { - void dataTypeAvailable(FunctionID functionID, FunctionDataTypeStatus dataTypeStatus); + void dataTypeAvailable(TypedApiInterface.FunctionID functionID, FunctionDataTypeStatus dataTypeStatus); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java index 3957a38..d28ba7e 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java @@ -2,7 +2,7 @@ import ai.reveng.invoker.ApiException; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ai.reveng.toolkit.ghidra.core.services.api.types.AIDecompilationStatus; @@ -262,7 +262,7 @@ public void run(TaskMonitor monitor) throws CancelledException { } - private void waitForDecomp(FunctionID id, TaskMonitor monitor) throws CancelledException { + private void waitForDecomp(TypedApiInterface.FunctionID id, TaskMonitor monitor) throws CancelledException { var logger = tool.getService(ReaiLoggingService.class); var api = service.getApi(); AIDecompilationStatus lastDecompStatus = null; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java index ccf4260..791c750 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java @@ -1,34 +1,21 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.ui.autounstrip; +import ai.reveng.model.AutoUnstripResponse; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.dialog.RevEngDialogComponentProvider; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; -import ai.reveng.toolkit.ghidra.core.services.api.types.AutoUnstripResponse; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.framework.plugintool.PluginTool; -import ghidra.program.model.address.Address; -import ghidra.program.model.listing.Function; -import ghidra.program.model.listing.Program; -import ghidra.program.model.symbol.SourceType; -import ghidra.util.exception.DuplicateNameException; -import ghidra.util.exception.InvalidInputException; import ghidra.util.task.TaskMonitorComponent; import javax.swing.*; import javax.swing.table.DefaultTableModel; import java.awt.*; -import java.util.ArrayList; -import java.util.List; import java.util.Objects; -import static ai.reveng.toolkit.ghidra.plugins.BinarySimilarityPlugin.REVENG_AI_NAMESPACE; - public class AutoUnstripDialog extends RevEngDialogComponentProvider { - private final AnalysisID analysisID; + private final GhidraRevengService.AnalysedProgram analysedProgram; private final GhidraRevengService revengService; - private final Program program; - private AutoUnstripResponse autoUnstripResponse; // UI components private JLabel statusLabel; @@ -39,23 +26,16 @@ public class AutoUnstripDialog extends RevEngDialogComponentProvider { private final TaskMonitorComponent taskMonitorComponent; private JTable resultsTable; private JScrollPane resultsScrollPane; - private final List renameResults; // Polling configuration private static final int POLL_INTERVAL_MS = 2000; // Poll every 2 seconds - // Inner class to hold rename results - private record RenameResult(String virtualAddress, String originalName, String newName) { - } - - public AutoUnstripDialog(PluginTool tool, GhidraRevengService.ProgramWithID analysisID) { + public AutoUnstripDialog(PluginTool tool, GhidraRevengService.AnalysedProgram analysedProgram) { super(ReaiPluginPackage.WINDOW_PREFIX + "Auto Unstrip", true); - this.analysisID = analysisID.analysisID(); - this.program = analysisID.program(); + this.analysedProgram = analysedProgram; this.revengService = tool.getService(GhidraRevengService.class); this.taskMonitorComponent = new TaskMonitorComponent(false, true); - this.renameResults = new ArrayList<>(); // Initialize UI addDismissButton(); @@ -85,14 +65,18 @@ private void startAutoUnstrip() { private void pollAutoUnstripStatus() { SwingUtilities.invokeLater(() -> { try { - autoUnstripResponse = revengService.getApi().autoUnstrip(analysisID); - updateUI(); + var autoUnstripResponse = revengService.autoUnstrip(analysedProgram).autoUnstripResponse(); + updateUI(autoUnstripResponse); // Check if we're done - if (autoUnstripResponse.progress() >= 100 || Objects.equals(autoUnstripResponse.status(), "COMPLETED")) { + if (autoUnstripResponse.getProgress() >= 100 || Objects.equals(autoUnstripResponse.getStatus(), "COMPLETED")) { stopPolling(); + // Pull function information (names and types) from the server, instead of dealing with matches + // in the auto unstrip response + var changes = revengService.pullFunctionInfoFromAnalysis(analysedProgram, taskMonitorComponent); taskMonitorComponent.setVisible(false); - importFunctionNames(autoUnstripResponse); + + SwingUtilities.invokeLater(() -> updateResultsTable(changes)); } } catch (Exception e) { handleError("Failed to poll auto unstrip status: " + e.getMessage()); @@ -101,96 +85,36 @@ private void pollAutoUnstripStatus() { }); } - private void importFunctionNames(AutoUnstripResponse autoUnstripResponse) { - var functionMgr = program.getFunctionManager(); - - // Retrieve the mangled names map once outside the transaction - var mangledNameMapOpt = revengService.getFunctionMangledNamesMap(program); - - // Retrieve the function ID map once outside the transaction - var functionMap = revengService.getFunctionMap(program); - - program.withTransaction("Apply Auto-Unstrip Function Names", () -> { - try { - var revengMatchNamespace = program.getSymbolTable().getOrCreateNameSpace( - program.getGlobalNamespace(), - REVENG_AI_NAMESPACE, - SourceType.ANALYSIS - ); - - autoUnstripResponse.matches().forEach(match -> { - Address addr = program.getAddressFactory().getDefaultAddressSpace().getAddress(match.function_vaddr()); - Function func = functionMgr.getFunctionAt(addr); - - var revEngMangledName = match.suggested_name(); - var revEngDemangledName = match.suggested_demangled_name(); - var functionID = functionMap.get(new FunctionID(match.function_id().value())); - - if ( - func != null && - // Do not override user-defined function names - func.getSymbol().getSource() != SourceType.USER_DEFINED && - // Exclude thunks and external functions - !func.isThunk() && - !func.isExternal() && - // Only accept valid names (no spaces) - !revEngMangledName.contains(" ") && - !revEngDemangledName.contains(" ") - // Only rename if the function ID is known (boundaries matched) - && functionID != null - ) { - try { - // Capture original name before renaming - String originalName = func.getName(); - - func.setName(revEngDemangledName, SourceType.ANALYSIS); - func.setParentNamespace(revengMatchNamespace); - - // Update the mangled name map with the RevEng.AI mangled name - mangledNameMapOpt.ifPresent(mangledNameMap -> { - try { - mangledNameMap.add(func.getEntryPoint(), revEngMangledName); - } catch (Exception e) { - handleError("Failed to update mangled name map for function at " + addr + ": " + e.getMessage()); - } - }); - - // Add to rename results - renameResults.add(new RenameResult(String.format("%08x", match.function_vaddr()), originalName, revEngDemangledName)); - } catch (Exception e) { - handleError("Failed to rename function at " + addr + ": " + e.getMessage()); - } - } - }); - } catch (DuplicateNameException | InvalidInputException e) { - throw new RuntimeException(e); - } - } - ); - - // Show results table after import is complete - SwingUtilities.invokeLater(this::updateResultsTable); - } - - private void updateUI() { + private void updateUI(AutoUnstripResponse autoUnstripResponse) { if (autoUnstripResponse == null) return; // Update progress bar - taskMonitorComponent.setProgress(autoUnstripResponse.progress()); - taskMonitorComponent.setMessage(autoUnstripResponse.progress() + "%"); + taskMonitorComponent.setProgress(autoUnstripResponse.getProgress()); + taskMonitorComponent.setMessage(autoUnstripResponse.getProgress() + "%"); // Update status - statusLabel.setText("Status: " + getFriendlyStatusMessage(autoUnstripResponse.status())); + statusLabel.setText("Status: " + getFriendlyStatusMessage(autoUnstripResponse.getStatus())); // Handle error message - dynamically add/remove error panel - if (autoUnstripResponse.error_message() != null && !autoUnstripResponse.error_message().isEmpty()) { - showError(autoUnstripResponse.error_message()); + if (autoUnstripResponse.getErrorMessage() != null && !autoUnstripResponse.getErrorMessage().isEmpty()) { + showError(autoUnstripResponse.getErrorMessage()); } else { hideError(); } + final var map = analysedProgram.getFunctionMap(); + var currentRenameResults = autoUnstripResponse.getMatches().stream().map(match -> { + // This functionID is for the local/origin function and not the matched/neighbour function! + var functionId = new TypedApiInterface.FunctionID(match.getFunctionId()); + var function = map.get(functionId); + return new GhidraRevengService.RenameResult( + function, + function.getName(), + match.getSuggestedName() + ); + }).toList(); // Update results table - updateResultsTable(); + updateResultsTable(currentRenameResults); } /** @@ -211,10 +135,10 @@ private String getFriendlyStatusMessage(String apiStatus) { }; } - private void updateResultsTable() { + private void updateResultsTable(java.util.List renameResults) { DefaultTableModel model = new DefaultTableModel(new Object[]{"Virtual Address", "Original Name", "New Name"}, 0); - for (RenameResult result : renameResults) { - model.addRow(new Object[]{result.virtualAddress, result.originalName, result.newName}); + for (GhidraRevengService.RenameResult result : renameResults) { + model.addRow(new Object[]{result.virtualAddress(), result.originalName(), result.newName()}); } resultsTable.setModel(model); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/collectiondialog/DataSetControlPanelComponent.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/collectiondialog/DataSetControlPanelComponent.java index 9d9ea4c..44cd88b 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/collectiondialog/DataSetControlPanelComponent.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/collectiondialog/DataSetControlPanelComponent.java @@ -1,8 +1,8 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.ui.collectiondialog; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; import docking.action.builder.ActionBuilder; import generic.theme.GIcon; import ghidra.framework.plugintool.ComponentProviderAdapter; @@ -22,7 +22,7 @@ *

*

*

- * Currently, collections are "flat", and effectively a set of {@link ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID}
+ * Currently, collections are "flat", and effectively a set of {@link TypedApiInterface.AnalysisID}
* In the future they may support hierarchies/nesting
*

* This dialog allows sharing the collection choice logic between all components that need to select a collection, @@ -154,7 +154,7 @@ public void run(TaskMonitor monitor) throws CancelledException { // Get all rows that are already selected to be included var selectedBinaries = binaryTableModel.getModelData().stream().filter(BinaryRowObject::include).toList(); // - Set selectedSet = selectedBinaries.stream().map(row -> row.analysisResult().analysisID() ).collect(Collectors.toSet()); + Set selectedSet = selectedBinaries.stream().map(row -> row.analysisResult().analysisID() ).collect(Collectors.toSet()); binaryTableModel.clearData(); // var searchTerm = analysisSearchTextBox.getText(); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java index 7014ad4..0bedc95 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java @@ -1,12 +1,17 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.ui.functionmatching; import ai.reveng.model.*; +import ai.reveng.toolkit.ghidra.binarysimilarity.cmds.ApplyMatchCmd; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.dialog.RevEngDialogComponentProvider; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.components.CollectionSelectionPanel; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.components.BinarySelectionPanel; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.components.SelectableItem; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatch; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatchWithSignature; +import com.google.common.collect.BiMap; import ghidra.program.model.listing.Function; import ghidra.program.model.symbol.SourceType; import ghidra.util.exception.DuplicateNameException; @@ -49,31 +54,32 @@ public abstract class AbstractFunctionMatchingDialog extends RevEngDialogCompone // Data protected Basic analysisBasicInfo; - protected FunctionMatchingBatchResponse functionMatchingResponse; - protected final List functionMatchResults; - protected final List filteredFunctionMatchResults; + protected FunctionMatchingResponse functionMatchingResponse; + protected final List functionMatchResults; + protected final List filteredFunctionMatchResults; // Polling configuration protected static final int POLL_INTERVAL_MS = 2000; // Poll every 2 seconds - // Inner class to hold function match results - protected record FunctionMatchResult( - String virtualAddress, - String functionName, - String bestMatchName, - String bestMatchMangledName, - String similarity, - String confidence, - String matchedHash, - String binary, - Long matcherFunctionId - ) { - // Constructor for function-level dialog (without virtual address and function name) - public FunctionMatchResult(String bestMatchName, String bestMatchMangledName, String similarity, - String confidence, String matchedHash, String binary, Long matcherFunctionId) { - this("", "", bestMatchName, bestMatchMangledName, similarity, confidence, matchedHash, binary, matcherFunctionId); - } - } +// /// Inner class to hold function match results +// /// @deprecated {@link ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch} +// public record FunctionMatchResult( +// String virtualAddress, +// String functionName, +// String bestMatchName, +// String bestMatchMangledName, +// String similarity, +// String confidence, +// String matchedHash, +// String binary, +// Long matcherFunctionId +// ) { +// // Constructor for function-level dialog (without virtual address and function name) +// public FunctionMatchResult(String bestMatchName, String bestMatchMangledName, String similarity, +// String confidence, String matchedHash, String binary, Long matcherFunctionId) { +// this("", "", bestMatchName, bestMatchMangledName, similarity, confidence, matchedHash, binary, matcherFunctionId); +// } +// } protected AbstractFunctionMatchingDialog(String title, Boolean isModal, GhidraRevengService revengService, GhidraRevengService.AnalysedProgram analyzedProgram) { @@ -122,28 +128,38 @@ protected void startFunctionMatching() { protected abstract void pollFunctionMatchingStatus(); - protected void processFunctionMatchingResults(FunctionMatchingBatchResponse response) { + protected void processFunctionMatchingResults(FunctionMatchingResponse response) { functionMatchResults.clear(); - - var functionMap = revengService.getFunctionMap(analyzedProgram.program()); + List matches = new ArrayList<>(); + final BiMap functionMap = analyzedProgram.getFunctionMap(); response.getMatches().forEach(matchResult -> { + // Retrieve the local function name + var funcId = new TypedApiInterface.FunctionID(matchResult.getFunctionId()); + Function localFunction = functionMap.get(funcId); + if (localFunction == null) { + // If we can't find the local function, skip this match (boundaries do not match the remote ones) + return; + } // Process each matched function in this result matchResult.getMatchedFunctions().forEach(match -> { - // Retrieve the local function name - Function localFunction = functionMap.get(new FunctionID(matchResult.getFunctionId())); - - if (localFunction == null) { - // If we can't find the local function, skip this match (boundaries do not match the remote ones) - return; - } - // Create function match result using the abstract method - FunctionMatchResult result = createFunctionMatchResult(localFunction, match, matchResult.getFunctionId()); - functionMatchResults.add(result); + FunctionMatch result = FunctionMatch.fromMatchedFunctionAPIType(match, funcId); + GhidraFunctionMatch ghidraResult = new GhidraFunctionMatch(localFunction, result); +// FunctionMatch result = createFunctionMatchResult(localFunction, match, funcId); + matches.add(ghidraResult); }); }); + // Get the type info for all matches + var ghidraResultsWithSignatures = revengService.getSignatures(matches); + + // For all matches we got, create a GhidraFunctionMatchWithSignature object (signature can be null!) + for (GhidraFunctionMatch match : matches) { + var sig = ghidraResultsWithSignatures.get(match); + functionMatchResults.add(new GhidraFunctionMatchWithSignature(match.function(), match.functionMatch(), sig)); + } + // Apply any existing function filter after getting results onFunctionFilterChanged(); @@ -151,7 +167,6 @@ protected void processFunctionMatchingResults(FunctionMatchingBatchResponse resp SwingUtilities.invokeLater(this::updateResultsTable); } - protected abstract FunctionMatchResult createFunctionMatchResult(Function localFunction, MatchedFunction match, Long matcherFunctionId); protected void updateUI() { if (functionMatchingResponse == null) return; @@ -181,7 +196,7 @@ protected void updateUI() { protected void updateResultsTable() { // Determine which results to show based on whether we have an active filter String filterText = functionFilterField != null ? functionFilterField.getText().trim() : ""; - List resultsToShow; + List resultsToShow; if (filterText.isEmpty()) { // No filter text, show all results @@ -192,7 +207,7 @@ protected void updateResultsTable() { } DefaultTableModel model = new DefaultTableModel(getTableColumnNames(), 0); - for (FunctionMatchResult result : resultsToShow) { + for (GhidraFunctionMatchWithSignature result : resultsToShow) { model.addRow(getTableRowData(result)); } resultsTable.setModel(model); @@ -228,7 +243,7 @@ protected void updateResultsTable() { } protected abstract String[] getTableColumnNames(); - protected abstract Object[] getTableRowData(FunctionMatchResult result); + protected abstract Object[] getTableRowData(GhidraFunctionMatchWithSignature result); protected abstract String getTableTitle(); protected abstract int getTableSelectionMode(); protected abstract void configureTableColumns(); @@ -710,7 +725,7 @@ protected void onFunctionFilterChanged() { updateResultsTable(); } - protected abstract boolean matchesFilter(FunctionMatchResult result); + protected abstract boolean matchesFilter(GhidraFunctionMatchWithSignature result); // Utility methods public int getThreshold() { @@ -770,10 +785,10 @@ protected void onRenameSelectedButtonClicked() { hideError(); } - List resultsToShow = filteredFunctionMatchResults.isEmpty() ? + List resultsToShow = filteredFunctionMatchResults.isEmpty() ? functionMatchResults : filteredFunctionMatchResults; - List selectedMatches = new ArrayList<>(); + List selectedMatches = new ArrayList<>(); for (int row : selectedRows) { if (row < resultsToShow.size()) { selectedMatches.add(resultsToShow.get(row)); @@ -784,76 +799,26 @@ protected void onRenameSelectedButtonClicked() { importFunctionNames(selectedMatches); } - protected void batchRenameFunctions(List functionMatches) { - var matches = functionMatches.stream() - .map(result -> { - var func = new FunctionRenameMap(); - func.setFunctionId(result.matcherFunctionId()); - func.setNewName(result.bestMatchName()); - func.setNewMangledName(result.bestMatchMangledName()); - return func; - }) - .toList(); - - var functionsListRename = new FunctionsListRename(); - functionsListRename.setFunctions(matches); - + protected void batchRenameFunctions(List functionMatches) { try { - revengService.batchRenameFunctions(functionsListRename); + revengService.batchRenamingGhidraMatchesWithSignatures(functionMatches); } catch (Exception e) { showError("Failed to rename functions: " + e.getMessage()); } } - protected void importFunctionNames(List functionMatches) { + protected void importFunctionNames(List functionMatches) { var program = analyzedProgram.program(); - var mangledNameMapOpt = revengService.getFunctionMangledNamesMap(program); - var functionMap = revengService.getFunctionMap(program); program.withTransaction("Apply Function Matching Renames", () -> { - try { - var revengMatchNamespace = program.getSymbolTable().getOrCreateNameSpace( - program.getGlobalNamespace(), - REVENG_AI_NAMESPACE, - SourceType.ANALYSIS - ); - - functionMatches.forEach(match -> { - var funcID = new FunctionID(match.matcherFunctionId()); - Function func = functionMap.get(funcID); - - var revEngMangledName = match.bestMatchMangledName(); - var revEngDemangledName = match.bestMatchName(); - - if (func != null && - func.getSymbol().getSource() != SourceType.USER_DEFINED && - !func.isThunk() && - !func.isExternal() && - !revEngMangledName.contains(" ") && - !revEngDemangledName.contains(" ") - ) { - try { - func.setName(revEngDemangledName, SourceType.ANALYSIS); - func.setParentNamespace(revengMatchNamespace); - - mangledNameMapOpt.ifPresent(mangledNameMap -> { - try { - mangledNameMap.add(func.getEntryPoint(), revEngMangledName); - } catch (Exception e) { - handleError("Failed to update mangled name map for function at " + func.getEntryPoint() + ": " + e.getMessage()); - } - }); - - } catch (Exception e) { - handleError("Failed to rename function at " + func.getEntryPoint() + ": " + e.getMessage()); - } + for (GhidraFunctionMatchWithSignature match : functionMatches) { + // We don't pass a service, because we don't need to update individual functions on the portal + // batchRenameFunctions already does that for us + var cmd = new ApplyMatchCmd(null, analyzedProgram, match, false); + cmd.applyTo(program); } - }); - } catch (DuplicateNameException | InvalidInputException e) { - throw new RuntimeException(e); - } - }); - + } + ); SwingUtilities.invokeLater(this::updateResultsTable); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/BinaryLevelFunctionMatchingDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/BinaryLevelFunctionMatchingDialog.java index def4897..2db1e48 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/BinaryLevelFunctionMatchingDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/BinaryLevelFunctionMatchingDialog.java @@ -2,6 +2,9 @@ import ai.reveng.model.*; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatch; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatchWithSignature; import ghidra.framework.plugintool.PluginTool; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.program.model.listing.Function; @@ -71,47 +74,21 @@ protected void pollFunctionMatchingStatus() { }); } - @Override - protected FunctionMatchResult createFunctionMatchResult(Function localFunction, MatchedFunction match, Long matcherFunctionId) { - String virtualAddress = String.format("%08x", match.getFunctionVaddr()); - String functionName = localFunction.getName(); - String bestMatchName = match.getFunctionName(); - String bestMatchMangledName = match.getMangledName(); - String similarity = match.getSimilarity() != null ? - String.format("%.2f%%", match.getSimilarity().doubleValue()) : "N/A"; - String confidence = match.getConfidence() != null ? - String.format("%.2f%%", match.getConfidence().doubleValue()) : "N/A"; - String matchedHash = match.getSha256Hash(); - String binary = match.getBinaryName(); - - return new FunctionMatchResult( - virtualAddress, - functionName, - bestMatchName, - bestMatchMangledName, - similarity, - confidence, - matchedHash, - binary, - matcherFunctionId - ); - } - @Override protected String[] getTableColumnNames() { return new String[]{"Virtual Address", "Function Name", "Matched Function", "Similarity", "Confidence", "Matched Hash", "Matched Binary"}; } @Override - protected Object[] getTableRowData(FunctionMatchResult result) { + protected Object[] getTableRowData(GhidraFunctionMatchWithSignature result) { return new Object[]{ - result.virtualAddress(), - result.functionName(), - result.bestMatchName(), - result.similarity(), - result.confidence(), - result.matchedHash(), - result.binary() + result.function().getEntryPoint().toString(), + result.function().getName(), + result.functionMatch().nearest_neighbor_function_name(), + result.functionMatch().similarity(), + result.functionMatch().confidence(), + result.functionMatch().nearest_neighbor_sha_256_hash().sha256(), + result.functionMatch().nearest_neighbor_binary_name() }; } @@ -175,12 +152,12 @@ protected String getDialogDescription() { } @Override - protected boolean matchesFilter(FunctionMatchResult result) { + protected boolean matchesFilter(GhidraFunctionMatchWithSignature result) { String filterText = functionFilterField.getText().trim().toLowerCase(); - return result.virtualAddress().toLowerCase().contains(filterText) || - result.functionName().toLowerCase().contains(filterText) || - result.bestMatchName().toLowerCase().contains(filterText) || - result.matchedHash().toLowerCase().contains(filterText) || - result.binary().toLowerCase().contains(filterText); + return result.function().getEntryPoint().toString().toLowerCase().contains(filterText) || + result.function().getName().toLowerCase().contains(filterText) || + result.functionMatch().nearest_neighbor_function_name().toLowerCase().contains(filterText) || + result.functionMatch().nearest_neighbor_sha_256_hash().sha256().toLowerCase().contains(filterText) || + result.functionMatch().nearest_neighbor_binary_name().toLowerCase().contains(filterText); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/FunctionLevelFunctionMatchingDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/FunctionLevelFunctionMatchingDialog.java index 469f874..614bb96 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/FunctionLevelFunctionMatchingDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/FunctionLevelFunctionMatchingDialog.java @@ -2,6 +2,10 @@ import ai.reveng.model.*; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatch; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatchWithSignature; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.listing.Function; @@ -87,41 +91,19 @@ protected void pollFunctionMatchingStatus() { }); } - @Override - protected FunctionMatchResult createFunctionMatchResult(Function localFunction, MatchedFunction match, Long matcherFunctionId) { - String bestMatchName = match.getFunctionName(); - String bestMatchMangledName = match.getMangledName(); - String similarity = match.getSimilarity() != null ? - String.format("%.2f%%", match.getSimilarity().doubleValue()) : "N/A"; - String confidence = match.getConfidence() != null ? - String.format("%.2f%%", match.getConfidence().doubleValue()) : "N/A"; - String matchedHash = match.getSha256Hash(); - String binary = match.getBinaryName(); - - return new FunctionMatchResult( - bestMatchName, - bestMatchMangledName, - similarity, - confidence, - matchedHash, - binary, - matcherFunctionId - ); - } - @Override protected String[] getTableColumnNames() { return new String[]{"Matched Function", "Similarity", "Confidence", "Matched Hash", "Matched Binary"}; } @Override - protected Object[] getTableRowData(FunctionMatchResult result) { + protected Object[] getTableRowData(GhidraFunctionMatchWithSignature result) { return new Object[]{ - result.bestMatchName(), - result.similarity(), - result.confidence(), - result.matchedHash(), - result.binary() + result.functionMatch().nearest_neighbor_function_name(), + result.functionMatch().similarity(), + result.functionMatch().confidence(), + result.functionMatch().nearest_neighbor_sha_256_hash().sha256(), + result.functionMatch().nearest_neighbor_binary_name() }; } @@ -169,11 +151,11 @@ protected String getDialogDescription() { } @Override - protected boolean matchesFilter(FunctionMatchResult result) { + protected boolean matchesFilter(GhidraFunctionMatchWithSignature result) { String filterText = functionFilterField.getText().trim().toLowerCase(); - return result.bestMatchName().toLowerCase().contains(filterText) || - result.matchedHash().toLowerCase().contains(filterText) || - result.binary().toLowerCase().contains(filterText); + return result.functionMatch().nearest_neighbor_function_name().toLowerCase().contains(filterText) || + result.functionMatch().nearest_neighbor_sha_256_hash().sha256().toLowerCase().contains(filterText) || + result.functionMatch().nearest_neighbor_binary_name().toLowerCase().contains(filterText); } @Override diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysesTableModel.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysesTableModel.java index 18fba58..c3452d9 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysesTableModel.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysesTableModel.java @@ -1,6 +1,7 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.ui.recentanalyses; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.core.services.function.export.ExportFunctionBoundariesService; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; @@ -16,10 +17,10 @@ import ghidra.util.task.TaskMonitor; public class RecentAnalysesTableModel extends ThreadedTableModelStub { - private final BinaryHash hash; + private final TypedApiInterface.BinaryHash hash; private final Address imageBase; - public RecentAnalysesTableModel(PluginTool tool, BinaryHash hash, Address imageBase) { + public RecentAnalysesTableModel(PluginTool tool, TypedApiInterface.BinaryHash hash, Address imageBase) { super("Recent Analyses Table Model", tool); this.hash = hash; this.imageBase = imageBase; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java index c3a5e9b..d034aeb 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java @@ -3,8 +3,8 @@ import ai.reveng.toolkit.ghidra.binarysimilarity.ui.dialog.RevEngDialogComponentProvider; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.LegacyAnalysisResult; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryHash; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.listing.Program; @@ -18,7 +18,7 @@ /** - * Shows a dialog with a table of {@link LegacyAnalysisResult} for a given {@link BinaryHash}, + * Shows a dialog with a table of {@link LegacyAnalysisResult} for a given {@link TypedApiInterface.BinaryHash}, * and fires an event when the user picks an analysis */ public class RecentAnalysisDialog extends RevEngDialogComponentProvider { @@ -34,7 +34,7 @@ public RecentAnalysisDialog(PluginTool tool, Program program) { this.program = program; this.ghidraRevengService = tool.getService(GhidraRevengService.class); - var hash = new BinaryHash(program.getExecutableSHA256()); + var hash = new TypedApiInterface.BinaryHash(program.getExecutableSHA256()); recentAnalysesTableModel = new RecentAnalysesTableModel(tool, hash, this.program.getImageBase()); recentAnalysesTable = new GhidraFilterTable<>(recentAnalysesTableModel); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilder.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilder.java index 6bba37c..ad720a9 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilder.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilder.java @@ -1,7 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api; import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisScope; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryHash; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionBoundary; import ghidra.program.model.listing.Program; import ghidra.util.Msg; @@ -35,7 +34,7 @@ public AnalysisOptionsBuilder functionBoundaries(long base, List statusCache = new HashMap<>(); + private final Map statusCache = new HashMap<>(); private TypedApiInterface api; private ApiInfo apiInfo; private final List collections = new ArrayList<>(); private final List analysisIDFilter = new ArrayList<>(); + /// dedicated functions on the GhidraRevengService should be used instead to enforce assumptions via + /// type level guarantees +// @Deprecated public TypedApiInterface getApi() { return api; } @@ -96,7 +101,7 @@ public URI getServer() { return this.apiInfo.hostURI(); } - public AnalysedProgram registerFinishedAnalysisForProgram(ProgramWithID programWithID, TaskMonitor monitor) { + public AnalysedProgram registerFinishedAnalysisForProgram(ProgramWithID programWithID, TaskMonitor monitor) throws CancelledException { var status = status(programWithID); if (!status.equals(AnalysisStatus.Complete)){ throw new IllegalStateException("Analysis %s is not complete yet, current status: %s" @@ -106,10 +111,11 @@ public AnalysedProgram registerFinishedAnalysisForProgram(ProgramWithID programW var analysedProgram = associateFunctionInfo(programWithID); pullFunctionInfoFromAnalysis(analysedProgram, monitor); + monitor.checkCancelled(); return analysedProgram; } - private ProgramWithID addAnalysisIDtoProgramOptions(Program program, AnalysisID analysisID){ + private ProgramWithID addAnalysisIDtoProgramOptions(Program program, TypedApiInterface.AnalysisID analysisID){ var transactionId = program.startTransaction("Associate Binary ID with Program"); program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY) .setLong(OPTION_KEY_ANALYSIS_ID, analysisID.id()); @@ -129,7 +135,8 @@ public Optional getBinaryIDFor(Program program) { return getBinaryIDfromOptions(program); } - private Optional getAnalysisIDFor(Program program){ + @SuppressWarnings("deprecation") // Using deprecated method to support legacy BinaryID + private Optional getAnalysisIDFor(Program program){ var optAnalysisID = getAnalysisIDFromOptions(program); if (optAnalysisID.isPresent()){ return optAnalysisID; @@ -148,7 +155,7 @@ private Optional getAnalysisIDFor(Program program){ } return Optional.empty(); } - private Optional getAnalysisIDFromOptions( + private Optional getAnalysisIDFromOptions( Program program ) { long bid = program.getOptions( @@ -157,7 +164,7 @@ private Optional getAnalysisIDFromOptions( if (bid == ReaiPluginPackage.INVALID_ANALYSIS_ID) { return Optional.empty(); } - var analysisID = new AnalysisID((int) bid); + var analysisID = new TypedApiInterface.AnalysisID((int) bid); if (!statusCache.containsKey(analysisID)) { // Check that it's really valid in the context of the currently configured API try { @@ -293,6 +300,11 @@ private AnalysedProgram associateFunctionInfo(ProgramWithID knownProgram) { } + public record RenameResult(Function func, String originalName, String newName) { + public String virtualAddress() { + return func.getEntryPoint().toString(); + } + } /// Pull the server side information about the functions from a remote Analysis and update the local {@link Program} /// based on it /// This currently includes: @@ -302,18 +314,24 @@ private AnalysedProgram associateFunctionInfo(ProgramWithID knownProgram) { /// It assumes that the initial load already happened, i.e. the functions have an associated FunctionID already. /// The initial association happens in {@link #associateFunctionInfo(ProgramWithID)} /// - public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMonitor monitor) { + public List pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMonitor monitor) { var transactionId = analysedProgram.program().startTransaction("RevEng.AI: Pull Function Info from Analysis"); - StringPropertyMap mangledNameMap = analysedProgram.program() - .getUsrPropertyManager() - .getStringPropertyMap(REAI_FUNCTION_MANGLED_MAP); + List renameResults = new ArrayList<>(); + int failedRenames = 0; - int ghidraRenamedFunctions; - ghidraRenamedFunctions = 0; + Map signatureMap = api.listFunctionDataTypesForAnalysis(analysedProgram.analysisID).getItems() + .stream() + .filter(item -> item.getStatus().equals("completed")) + .filter(item -> item.getDataTypes().getFuncTypes() != null) + .collect( + Collectors.toMap( + item -> new TypedApiInterface.FunctionID(item.getFunctionId()), + fdtStatus -> fdtStatus + ) + ); - int failedRenames = 0; for (Function function : analysedProgram.program().getFunctionManager().getFunctions(true)) { if (monitor.isCancelled()) { continue; @@ -336,7 +354,6 @@ public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMo // Extract the mangled name from Ghidra var revEngMangledName = details.mangledFunctionName(); - // TODO: This is currently just a placeholder until the server provides demangled names at this endpoint! var revEngDemangledName = details.demangledName(); // Skip invalid function mangled names @@ -345,19 +362,16 @@ public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMo continue; } - + var sig = Optional.ofNullable(signatureMap.get(fID.get().functionID)); // Get the type information on the server side - Optional functionSignatureMessageOpt = api.getFunctionDataTypes( - analysedProgram.analysisID(), - fID.get().functionID() - ) + Optional functionSignatureMessageOpt = sig // Try getting the data types if they are available - .flatMap(FunctionDataTypeStatus::data_types) // If they are available, try converting them to a Ghidra signature // If the conversion fails, act like there is no signature available + .flatMap (item -> Optional.ofNullable(item.getDataTypes())) .flatMap((functionDataTypeMessage -> { try { - return Optional.of(getFunctionSignature(functionDataTypeMessage)); + return getFunctionSignature(functionDataTypeMessage); } catch (DataTypeDependencyException e) { // Something went wrong loading the data type dependencies // just skip applying the signature and treat it like none being available @@ -367,7 +381,7 @@ public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMo })); - mangledNameMap.add(function.getEntryPoint(), revEngMangledName); + analysedProgram.setMangledNameForFunction(function, revEngMangledName); /// Source types: /// DEFAULT: placeholder name automatically assigned by Ghidra when it doesn’t know the real name. @@ -388,7 +402,11 @@ public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMo var success = new SetFunctionNameCmd(function.getEntryPoint(), revEngDemangledName, SourceType.ANALYSIS) .applyTo(analysedProgram.program()); if (success) { - ghidraRenamedFunctions++; + renameResults.add(new RenameResult( + function, + ghidraMangledName, + revEngDemangledName + )); } else { failedRenames++; Msg.error(this, "Failed to rename function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); @@ -408,7 +426,11 @@ public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMo ).applyTo(analysedProgram.program(), monitor); // For unclear reasons the signature source is not set by the command in Ghidra 11.2.x and lower if (success) { - ghidraRenamedFunctions++; + renameResults.add(new RenameResult( + function, + ghidraMangledName, + revEngDemangledName + )); } else { Msg.error(this, "Failed to apply signature to function %s".formatted(function.getName())); failedRenames++; @@ -420,13 +442,14 @@ public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMo } // Done iterating over all functions. If nothing changed, discard the transaction, to keep undo history clean - analysedProgram.program().endTransaction(transactionId, ghidraRenamedFunctions > 0); + analysedProgram.program().endTransaction(transactionId, !renameResults.isEmpty() && !monitor.isCancelled()); if (failedRenames > 0){ Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Function Update Summary", ("Failed to update %d functions from RevEng.AI. Please check the error log for details.").formatted( failedRenames )); } + return renameResults; } /** @@ -438,33 +461,11 @@ public void pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMo * @deprecated Use {@link AnalysedProgram#getIDForFunction(Function)} instead. It forces the caller to prove that they know that the {@link Program} is indeed known on the server and associated by having to provide a {@link AnalysedProgram} instance. */ @Deprecated - public Optional getFunctionIDFor(Function function){ + public Optional getFunctionIDFor(Function function){ return getAnalysedProgram(function.getProgram()) .flatMap(knownProgram -> knownProgram.getIDForFunction(function).map(fidWithStatus -> fidWithStatus.functionID)); } - public Optional getFunctionMangledNamesMap(Program program) { - return Optional.ofNullable(program.getUsrPropertyManager().getStringPropertyMap(REAI_FUNCTION_MANGLED_MAP)); - } - - public BiMap getFunctionMap(Program program){ - var propMap = program.getUsrPropertyManager().getLongPropertyMap(REAI_FUNCTION_PROP_MAP); - BiMap functionMap = HashBiMap.create(); - propMap.getPropertyIterator().forEachRemaining( - addr -> { - var func = program.getFunctionManager().getFunctionAt(addr); - - try { - functionMap.put(new FunctionID(propMap.getLong(addr)), func); - } catch (NoValueException e) { - // This should never happen, because we're iterating over the keys - throw new RuntimeException(e); - } - } - ); - return functionMap; - } - /** * Get the Ghidra Function for a given FunctionInfo if there is one */ @@ -478,7 +479,7 @@ private Optional getFunctionFor(FunctionInfo functionInfo, Program pro } @Deprecated - public List searchForHash(BinaryHash hash){ + public List searchForHash(TypedApiInterface.BinaryHash hash){ return api.search(hash); } @@ -487,6 +488,7 @@ public void removeProgramAssociation(Program program){ program.getUsrPropertyManager().removePropertyMap(REAI_FUNCTION_PROP_MAP); program.getUsrPropertyManager().removePropertyMap(REAI_FUNCTION_MANGLED_MAP); var reaiOptions = program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY); + //noinspection deprecation reaiOptions.setLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID); reaiOptions.setLong(OPTION_KEY_ANALYSIS_ID, ReaiPluginPackage.INVALID_ANALYSIS_ID); // Clear the entire cache. Getting the correct ID is not worth the effort in terms of edge cases to handle @@ -517,12 +519,12 @@ public static List exportFunctionBoundaries(Program program){ return result; } - private BinaryHash hashOfProgram(Program program) { + private TypedApiInterface.BinaryHash hashOfProgram(Program program) { // TODO: we break the guarantee that a BinaryHash implies that a file of this hash has already been uploaded - return new BinaryHash(program.getExecutableSHA256()); + return new TypedApiInterface.BinaryHash(program.getExecutableSHA256()); } - public BinaryHash upload(Program program) { + public TypedApiInterface.BinaryHash upload(Program program) { // TODO: Check if the program is already uploaded on the server // But this requires a dedicated API to do cleanly @@ -552,7 +554,7 @@ public BinaryHash upload(Program program) { } } - public BinaryHash upload(Path path) { + public TypedApiInterface.BinaryHash upload(Path path) { try { return api.upload(path); } catch (FileNotFoundException | ApiException e) { @@ -570,7 +572,7 @@ public AnalysisStatus pollStatus(BinaryID bid) { } /// Use this method if you just have an AnalysisID and it is not clear yet if it can be accessed - public AnalysisStatus pollStatus(AnalysisID id) throws ApiException { + public AnalysisStatus pollStatus(TypedApiInterface.AnalysisID id) throws ApiException { return api.status(id); } @@ -633,7 +635,7 @@ public String decompileFunctionViaAI(FunctionWithID functionWithID, TaskMonitor /// This method analyses a program by uploading it (if necessary), triggering an analysis, and _blocking_ /// until the analysis is complete. This is for scripts and tests, and must not be used on the UI thread - /// It does not upload the program, this must be done beforehand, and the hash must be associated via {@link AnalysisOptionsBuilder#hash(BinaryHash)} + /// It does not upload the program, this must be done beforehand, and the hash must be associated via {@link AnalysisOptionsBuilder#hash(TypedApiInterface.BinaryHash)} public AnalysedProgram analyse(Program program, AnalysisOptionsBuilder analysisOptionsBuilder, TaskMonitor monitor) throws CancelledException, ApiException { // Check if we are on the swing thread var programWithBinaryID = startAnalysis(program, analysisOptionsBuilder); @@ -665,58 +667,58 @@ public Optional getAnalysedProgram(Program program) { return Optional.empty(); } - public Optional getFunctionSignatureArtifact(AnalysedProgram program, FunctionID functionID) { - return api.getFunctionDataTypes(program.analysisID(), functionID).flatMap(FunctionDataTypeStatus::data_types); - } - /** - * Create a {@link FunctionDefinitionDataType} from a @{@link FunctionDataTypeMessage} in isolation + * Create a {@link FunctionDefinitionDataType} from a @{@link FunctionInfoOutput} in isolation * - * All the required dependency types should be stored in the DataTypeManager that is associated with this + * All the required dependency types will be stored in the DataTypeManager that is associated with this * FunctionDefinitionDataType * * @param functionDataTypeMessage The message containing the function signature, received from the API * @return Self-contained signature for the function */ - public static FunctionDefinitionDataType getFunctionSignature(FunctionDataTypeMessage functionDataTypeMessage) throws DataTypeDependencyException { - - // TODO: Do we need the program or data type manager? - // Or can we just create a new one with all the necessary types and then they get merged? + public static Optional getFunctionSignature(FunctionInfoOutput functionDataTypeMessage) throws DataTypeDependencyException { // Create Data Type Manager with all dependencies - var dtm = loadDependencyDataTypes(functionDataTypeMessage.func_deps()); + var d = FunctionDependencies.fromOpenAPI(functionDataTypeMessage.getFuncDeps()); + var dtm = loadDependencyDataTypes(d); - FunctionDefinitionDataType f = new FunctionDefinitionDataType(functionDataTypeMessage.functionName(), dtm); + if (functionDataTypeMessage.getFuncTypes() == null){ + return Optional.empty(); + } + var funcName = functionDataTypeMessage.getFuncTypes().getName(); + FunctionDefinitionDataType f = new FunctionDefinitionDataType(funcName, dtm); try { - f.setName(functionDataTypeMessage.functionName()); + f.setName(funcName); } catch (InvalidNameException e) { throw new RuntimeException(e); } - ParameterDefinitionImpl[] args = Arrays.stream(functionDataTypeMessage.func_types().header().args()).map( + ParameterDefinitionImpl[] args = functionDataTypeMessage.getFuncTypes().getHeader().getArgs().values().stream().map( arg -> { DataType ghidraType = null; try { - ghidraType = loadDataType(dtm, arg.type(), functionDataTypeMessage.func_deps()); + var scopedName = TypePathAndName.fromString(arg.getType()); + ghidraType = loadDataType(dtm, scopedName); } catch (DataTypeDependencyException e) { Msg.error(GhidraRevengService.class, - "Couldn't find type '%s' for param of %s".formatted(arg.type(), functionDataTypeMessage.functionName()) + ("" + + "Couldn't find type '%s' for param of %s").formatted(arg.getType(), funcName) ); - ghidraType = Undefined.getUndefinedDataType(arg.size()); + ghidraType = Undefined.getUndefinedDataType(arg.getSize()); } // Add the type to the DataTypeManager - return new ParameterDefinitionImpl(arg.name(), ghidraType, null); + return new ParameterDefinitionImpl(arg.getName(), ghidraType, null); }).toArray(ParameterDefinitionImpl[]::new); f.setArguments(args); DataType returnType = null; - returnType = loadDataType(dtm, functionDataTypeMessage.func_types().header().type(), functionDataTypeMessage.func_deps()); + returnType = loadDataType(dtm, TypePathAndName.fromString(functionDataTypeMessage.getFuncTypes().getHeader().getType())); f.setReturnType(returnType); - return f; + return Optional.of(f); } public static DataTypeManager loadDependencyDataTypes(FunctionDependencies dependencies){ @@ -761,7 +763,8 @@ public static DataTypeManager loadDependencyDataTypes(FunctionDependencies depen var path = TypePathAndName.fromString(typeDef.name()); DataType type; try { - type = dataTypeParser.parse(typeDef.type()); + var scopedType = TypePathAndName.fromString(typeDef.type()); + type = dataTypeParser.parse(scopedType.name()); } catch (InvalidDataTypeException e) { // The type wasn't available in the DataTypeManager yet, try again later typeDefsToAdd.add(typeDef); @@ -770,7 +773,7 @@ public static DataTypeManager loadDependencyDataTypes(FunctionDependencies depen } catch (CancelledException e) { throw new RuntimeException(e); } - TypedefDataType typedefDataType = new TypedefDataType(new CategoryPath(CategoryPath.ROOT, path.path()), path.name(), type, null); + TypedefDataType typedefDataType = new TypedefDataType(path.toCategoryPath(), path.name(), type, null); dtm.addDataType(typedefDataType, DataTypeConflictHandler.REPLACE_EMPTY_STRUCTS_OR_RENAME_AND_ADD_HANDLER); } @@ -785,7 +788,7 @@ public static DataTypeManager loadDependencyDataTypes(FunctionDependencies depen binSyncStructMember -> { DataType fieldType = null; try { - fieldType = loadDataType(dtm, binSyncStructMember.type(), dependencies); + fieldType = loadDataType(dtm, TypePathAndName.fromString(binSyncStructMember.type())); } catch (DataTypeDependencyException e) { Msg.error( GhidraRevengService.class, @@ -813,7 +816,7 @@ public static DataTypeManager loadDependencyDataTypes(FunctionDependencies depen return dtm; } - private static DataType loadDataType(DataTypeManager dtm, String name, FunctionDependencies dependencies) throws DataTypeDependencyException { + private static DataType loadDataType(DataTypeManager dtm, TypePathAndName type) throws DataTypeDependencyException { DataTypeParser dataTypeParser = new DataTypeParser( dtm, null, @@ -821,17 +824,17 @@ private static DataType loadDataType(DataTypeManager dtm, String name, FunctionD DataTypeParser.AllowedDataTypes.ALL); DataType dataType; try { - dataType = dataTypeParser.parse(name); + dataType = dataTypeParser.parse(type.name()); } catch (InvalidDataTypeException e) { // The type wasn't available in the DataTypeManager, so we have to find it in the dependencies - throw new DataTypeDependencyException("Data type not found in DataTypeManager: %s".formatted(name), e); + throw new DataTypeDependencyException("Data type not found in DataTypeManager: %s".formatted(type), e); } catch (CancelledException e) { throw new RuntimeException(e); } return dataType; } - public String getAnalysisLog(AnalysisID analysisID) { + public String getAnalysisLog(TypedApiInterface.AnalysisID analysisID) { return api.getAnalysisLogs(analysisID); } @@ -850,7 +853,7 @@ public BoxPlot getNameScoreForMatch(GhidraFunctionMatch functionMatch) { public static final String PORTAL_FUNCTIONS = "function/"; - public void openFunctionInPortal(FunctionID functionID) { + public void openFunctionInPortal(TypedApiInterface.FunctionID functionID) { openPortal(PORTAL_FUNCTIONS, String.valueOf(functionID.value())); } @@ -861,7 +864,7 @@ public void openCollectionInPortal(Collection collection) { public void openPortalFor(Collection c){ openCollectionInPortal(c); } - public void openPortalFor(FunctionID f){ + public void openPortalFor(TypedApiInterface.FunctionID f){ openFunctionInPortal(f); } @@ -873,7 +876,7 @@ public void openPortalFor(ProgramWithID programWithID) { openPortalFor(programWithID.analysisID()); } - public void openPortalFor(AnalysisID analysisID) { + public void openPortalFor(TypedApiInterface.AnalysisID analysisID) { openPortal("analyses", String.valueOf(analysisID.id())); } @@ -989,7 +992,7 @@ public Map getNameScores(java.util.Collection r = api.getNameScores(values.stream().map(GhidraFunctionMatch::functionMatch).toList(), false); // Collect to a Map from the FunctionID to the actual score - Map plots = r.stream().collect(Collectors.toMap(FunctionNameScore::functionID, FunctionNameScore::score)); + Map plots = r.stream().collect(Collectors.toMap(FunctionNameScore::functionID, FunctionNameScore::score)); return values.stream().collect(Collectors.toMap( match -> match, match -> plots.get(match.functionMatch().origin_function_id()) @@ -1001,23 +1004,43 @@ public Map getNameScores(java.util.Collection getSignatures(java.util.Collection values) { + public Map getSignatures(java.util.Collection values) { + - DataTypeList dataTypesList = this.api.getFunctionDataTypes(values.stream().map(GhidraFunctionMatch::nearest_neighbor_id).toList()); - Map signatureMap = Arrays.stream(dataTypesList.dataTypes()) - .filter(FunctionDataTypeStatus::completed) - .filter(status -> status.data_types().isPresent()) + // Get all data type info for the neighbour functions + var dataTypesList = this.api.listFunctionDataTypesForFunctions( + values.stream().map(GhidraFunctionMatch::nearest_neighbor_id).toList() + ); + // Create a map from FunctionID to FunctionInfoOutput for easy lookup, only for completed signatures + Map signatureMap = dataTypesList.getItems().stream() + // Only keep completed signatures + .filter(FunctionDataTypesListItem::getCompleted) + // Double check that there is a data type available + .filter(functionDataTypesListItem -> functionDataTypesListItem.getDataTypes() != null) .collect(Collectors.toMap( - FunctionDataTypeStatus::functionID, - status -> status.data_types().get() + item -> new TypedApiInterface.FunctionID(item.getFunctionId()), + FunctionDataTypesListItem::getDataTypes )); - return values.stream() + Map matchMap = values.stream() .filter(match -> signatureMap.containsKey(match.functionMatch().nearest_neighbor_id())) .collect(Collectors.toMap( match -> match, match -> signatureMap.get(match.functionMatch().nearest_neighbor_id()) )); + + // Now parse all signatures + Map result = new HashMap<>(); + for (var entry : matchMap.entrySet()){ + try { + var funcDefOpt = getFunctionSignature(entry.getValue()); + funcDefOpt.ifPresent(funcDef -> result.put(entry.getKey(), funcDef)); + } catch (DataTypeDependencyException e) { + Msg.error(this, "Could not parse signature for function %s".formatted(entry.getKey().functionMatch()), e); + } + } + return result; + } public CompletableFuture> searchCollectionsWithIds(String query, String modelName) { @@ -1070,28 +1093,62 @@ public CompletableFuture> searchBinariesWithIds(String quer }); } - public Basic getBasicDetailsForAnalysis(AnalysisID analysisID) throws ApiException { + public Basic getBasicDetailsForAnalysis(TypedApiInterface.AnalysisID analysisID) throws ApiException { return api.getAnalysisBasicInfo(analysisID); } - public FunctionMatchingBatchResponse getFunctionMatchingForAnalysis(AnalysisID analysisID, AnalysisFunctionMatchingRequest request) throws ApiException { + public FunctionMatchingResponse getFunctionMatchingForAnalysis(TypedApiInterface.AnalysisID analysisID, AnalysisFunctionMatchingRequest request) throws ApiException { return api.analysisFunctionMatching(analysisID, request); } - public FunctionMatchingBatchResponse getFunctionMatchingForFunction(FunctionMatchingRequest request) throws ApiException { + public FunctionMatchingResponse getFunctionMatchingForFunction(FunctionMatchingRequest request) throws ApiException { return api.functionFunctionMatching(request); } - public void batchRenameFunctions(FunctionsListRename functionsList) throws ApiException { - api.batchRenameFunctions(functionsList); + public void batchRenamingGhidraMatchesWithSignatures(List functionsList) throws ApiException { + // Pushing types to the portal is not supported yet, so we just extract the function matches and call the other method + batchRenameMatches(functionsList.stream() + .map(GhidraFunctionMatchWithSignature::functionMatch) + .toList()); + + } + + public void batchRenameGhidraMatches(List functionsList) throws ApiException { + var matches = functionsList.stream() + .map(GhidraFunctionMatch::functionMatch) + .toList(); + batchRenameMatches(matches); } + public void batchRenameMatches(List functionsList) throws ApiException { + var matches = functionsList.stream() + .map(result -> { + var func = new FunctionRenameMap(); + func.setFunctionId(result.origin_function_id().value()); + func.setNewName(result.nearest_neighbor_function_name()); + func.setNewMangledName(result.nearest_neighbor_mangled_function_name()); + return func; + }) + .toList(); + + var functionsListRename = new FunctionsListRename(); + functionsListRename.setFunctions(matches); + + api.batchRenameFunctions(functionsListRename); + } + + + public TypedApiInterface.TypedAutoUnstripResponse autoUnstrip(AnalysedProgram program) throws ApiException { + return api.autoUnstrip(program.analysisID); + } + + /// Old Helper Datatype that encapsulates a Ghidra program with a binary ID and analysis ID /// This only guarantees that the program has an associated analysis, but not that the analysis is finished /// The id of this should also be stored in the program options, but this is currently not enforced yet to allow easier testing public record ProgramWithID( Program program, - AnalysisID analysisID + TypedApiInterface.AnalysisID analysisID ){} @@ -1105,14 +1162,14 @@ public record ProgramWithID( public static class AnalysedProgram { private final Program program; - private final AnalysisID analysisID; + private final TypedApiInterface.AnalysisID analysisID; /// The constructor is private to enforce that only the GhidraRevengService class /// can create instances of this class, ensuring the guarantees hold private AnalysedProgram( Program program, - AnalysisID analysisID + TypedApiInterface.AnalysisID analysisID ) { this.program = program; this.analysisID = analysisID; @@ -1123,7 +1180,7 @@ public Program program() { return program; } - public AnalysisID analysisID() { + public TypedApiInterface.AnalysisID analysisID() { return analysisID; } @@ -1137,6 +1194,10 @@ private LongPropertyMap getFunctionIDPropertyMap(AnalysedProgram program){ /// Only returns Optional.Empty if the function is not known on the server (e.g. because it's a thunk) public Optional getIDForFunction(Function function) { + if (function == null) { + Msg.error(AnalysedProgram.class, "Function provided to getIDForFunction is null"); + return Optional.empty(); + } if (function.getProgram() != this.program){ throw new IllegalArgumentException("Function %s does not belong to program %s".formatted(function, this.program.getName())); } @@ -1144,17 +1205,64 @@ public Optional getIDForFunction(Function function) { var rawId = functionIDMap.get(function.getEntryPoint()); return Optional .ofNullable(rawId) - .map(FunctionID::new) + .map(TypedApiInterface.FunctionID::new) .map( functionID -> new FunctionWithID(function, functionID) ); } + public BiMap getFunctionMap(){ + var propMap = getFunctionIDPropertyMap(this); + + BiMap functionMap = HashBiMap.create(); + propMap.getPropertyIterator().forEachRemaining( + addr -> { + var func = program.getFunctionManager().getFunctionAt(addr); + + try { + functionMap.put(new TypedApiInterface.FunctionID(propMap.getLong(addr)), func); + } catch (NoValueException e) { + // This should never happen, because we're iterating over the keys + throw new RuntimeException(e); + } + } + ); + return functionMap; + } + + + public Optional getFunctionForID(TypedApiInterface.FunctionID functionID) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + public void setMangledNameForFunction(Function function, String mangledName) { + if (function.getProgram() != this.program){ + throw new IllegalArgumentException("Function %s does not belong to program %s".formatted(function, this.program.getName())); + } + StringPropertyMap mangledNameMap = this.program.getUsrPropertyManager().getStringPropertyMap(REAI_FUNCTION_MANGLED_MAP); + if (mangledNameMap == null){ + throw new IllegalStateException("Mangled name property map not found for supposedly known program %s".formatted(this.program.getName())); + } + mangledNameMap.add(function.getEntryPoint(), mangledName); + } + + public String getMangledNameForFunction(Function function) { + if (function.getProgram() != this.program){ + throw new IllegalArgumentException("Function %s does not belong to program %s".formatted(function, this.program.getName())); + } + StringPropertyMap mangledNameMap = this.program.getUsrPropertyManager().getStringPropertyMap(REAI_FUNCTION_MANGLED_MAP); + if (mangledNameMap == null){ + throw new IllegalStateException("Mangled name property map not found for supposedly known program %s".formatted(this.program.getName())); + } + return mangledNameMap.getString(function.getEntryPoint()); + } + + } /// Holding this object serves as the proof that a Function has an associated FunctionID public static record FunctionWithID( Function function, - FunctionID functionID + TypedApiInterface.FunctionID functionID ) {} } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java index f48ff91..b8fc0ae 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java @@ -3,8 +3,8 @@ import ai.reveng.api.*; import ai.reveng.model.*; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.services.api.types.AutoUnstripResponse; import ai.reveng.toolkit.ghidra.core.services.api.types.Collection; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; import ai.reveng.toolkit.ghidra.core.services.api.types.LegacyCollection; import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIAuthenticationException; import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIConflictException; @@ -39,15 +39,12 @@ import static ai.reveng.toolkit.ghidra.core.services.api.Utils.mapJSONArray; import static java.net.http.HttpClient.Version.HTTP_1_1; -/** - * The main implementation of the RevEng HTTP API - * Design notes: - * - every method should correspond to a single API endpoint - * - every method should simply execute the request and return the response - * - i.e. no smart checks relying on other API calls to check if e.g. a binary has already been uploaded - * - * - */ +/// The main implementation of the RevEng HTTP API +/// It partially relies on the old manual implementation, but should be migrated to the OpenAPI generated client over time +/// Design notes: +/// - every method should correspond to a single API endpoint +/// - every method should simply execute the request and return the response +/// - i.e. no smart checks relying on other API calls to check if e.g. a binary has already been uploaded public class TypedApiImplementation implements TypedApiInterface { private final HttpClient httpClient; private final String baseUrl; @@ -60,6 +57,7 @@ public class TypedApiImplementation implements TypedApiInterface { private final FunctionsCoreApi functionsCoreApi; private final FunctionsRenamingHistoryApi functionsRenamingHistoryApi; private final FunctionsAiDecompilationApi functionsAiDecompilationApi; + private final FunctionsDataTypesApi functionsDataTypesApi; // Cache for binary ID to analysis ID mappings @Deprecated @@ -102,6 +100,7 @@ public TypedApiImplementation(String baseUrl, String apiKey) { this.functionsCoreApi = new FunctionsCoreApi(apiClient); this.functionsRenamingHistoryApi = new FunctionsRenamingHistoryApi(apiClient); this.functionsAiDecompilationApi = new FunctionsAiDecompilationApi(apiClient); + this.functionsDataTypesApi = new FunctionsDataTypesApi(apiClient); this.baseUrl = baseUrl + "/"; this.httpClient = HttpClient.newBuilder() @@ -199,7 +198,7 @@ private JSONObject sendRequest(HttpRequest request) throws APIAuthenticationExce @Override public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { var analysisRequest = options.toAnalysisCreateRequest(); - var result = this.analysisCoreApi.createAnalysis(analysisRequest); + var result = this.analysisCoreApi.createAnalysis(analysisRequest, null); return new AnalysisID(result.getData().getAnalysisId()); } @@ -389,7 +388,6 @@ public DataTypeList generateFunctionDataTypes(AnalysisID analysisID, List functionIDS) { String queryString = functionIDS.stream().map( f -> "function_ids=" + f.value() ).reduce((a, b) -> a + "&" + b).orElseThrow(); - var request = requestBuilderForEndpoint("functions", "data_types?", queryString) .GET() .header("Content-Type", "application/json" ) @@ -398,6 +396,33 @@ public DataTypeList getFunctionDataTypes(List functionIDS) { return DataTypeList.fromJson(response.getJsonData()); } + public FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID id, List ids) { + try { + List functionIds = null; + if (ids == null) { + functionIds = null; + } else { + functionIds = ids.stream().map(FunctionID::value).map(Long::intValue).toList(); + } + var r = functionsDataTypesApi.listFunctionDataTypesForAnalysis(id.id(), functionIds); + var data = r.getData(); + return data; + } catch (ApiException e) { + throw new RuntimeException(e); + } + } + + @Override + public FunctionDataTypesList listFunctionDataTypesForFunctions(List functionIDs) { + try { + var r = functionsDataTypesApi.listFunctionDataTypesForFunctions(functionIDs.stream().map(FunctionID::value).map(Long::intValue).toList()); + var data = r.getData(); + return data; + } catch (ApiException e) { + throw new RuntimeException(e); + } + } + @Override public Optional getFunctionDataTypes(AnalysisID analysisID, FunctionID functionID) { // https://api.reveng.ai/v2/analyses/{analysis_id}/info/functions/{function_id}/data_types @@ -543,32 +568,21 @@ public FunctionDetails getFunctionDetails(FunctionID id) { } @Override - public AutoUnstripResponse autoUnstrip(AnalysisID analysisID) { - JSONObject params = new JSONObject(); - params.put("apply", true); - params.put("min_similarity", 90); // 90% - params.put("confidence_threshold", 90); // 90% - params.put("min_group_size", 1); // At least 1 function in the group - - var request = requestBuilderForEndpoint("analyses", String.valueOf(analysisID.id()), "functions", "auto-unstrip") - .POST(HttpRequest.BodyPublishers.ofString(params.toString())) - .header("Content-Type", "application/json" ) - .build(); - - return AutoUnstripResponse.fromJSONObject(sendRequest(request)); + public TypedAutoUnstripResponse autoUnstrip(AnalysisID analysisID) { + try { + return new TypedAutoUnstripResponse(functionsCoreApi.autoUnstrip(analysisID.id(), new AutoUnstripRequest())); + } catch (ApiException e) { + throw new RuntimeException(e); + } } @Override - public AutoUnstripResponse aiUnstrip(AnalysisID analysisID) { - JSONObject params = new JSONObject(); - params.put("apply", true); - - var request = requestBuilderForEndpoint("analyses", String.valueOf(analysisID.id()), "functions", "ai-unstrip") - .POST(HttpRequest.BodyPublishers.ofString(params.toString())) - .header("Content-Type", "application/json" ) - .build(); - - return AutoUnstripResponse.fromJSONObject(sendRequest(request)); + public TypedAutoUnstripResponse aiUnstrip(AnalysisID analysisID) { + try { + return new TypedAutoUnstripResponse(functionsCoreApi.aiUnstrip(analysisID.id(), new AiUnstripRequest())); + } catch (ApiException e) { + throw new RuntimeException(e); + } } @Override @@ -612,12 +626,12 @@ public ai.reveng.model.Basic getAnalysisBasicInfo(AnalysisID analysisID) throws } @Override - public FunctionMatchingBatchResponse analysisFunctionMatching(AnalysisID analysisID, AnalysisFunctionMatchingRequest request) throws ApiException { + public FunctionMatchingResponse analysisFunctionMatching(AnalysisID analysisID, AnalysisFunctionMatchingRequest request) throws ApiException { return this.functionsCoreApi.analysisFunctionMatching(analysisID.id(), request); } @Override - public FunctionMatchingBatchResponse functionFunctionMatching(FunctionMatchingRequest request) throws ApiException { + public FunctionMatchingResponse functionFunctionMatching(FunctionMatchingRequest request) throws ApiException { return this.functionsCoreApi.batchFunctionMatching(request); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java index 12ea84f..e7b1250 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java @@ -6,8 +6,9 @@ import java.util.Optional; import ai.reveng.model.*; +import ai.reveng.model.AutoUnstripResponse; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.services.api.types.AutoUnstripResponse; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.InvalidAPIInfoException; import javax.annotation.Nullable; @@ -21,15 +22,26 @@ * * It aims to stick close to the API functions themselves. * E.g. if a feature is implemented via two API calls, it should be implemented as two methods here. - * + * "Typed" refers to using special types for IDs like {@link AnalysisID} and {@link FunctionID}, rather than raw integers or strings. * Wrapping this feature into one conceptual method should then happen inside the {@link ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService} - * + * This exists as an interface so tests can mock it out more easily. * */ public interface TypedApiInterface { - // Analysis -// @Deprecated -// BinaryID legacyAnalyse(AnalysisOptionsBuilder binHash) throws ApiException; + /// Data type to represent the RevEng.AI API concept of a function ID + record FunctionID(long value){} + + /// This is a special box type for an analysis ID + /// It enforces that the integer is specifically an analysis ID, + /// and it implies that the user has (at least read) access to this ID + record AnalysisID(int id) {} + // TODO: could add a box type for an analysis that the user has _write_ access to + + record CollectionID(int id) {} + + /// Data type for all reveng API responses or parameters that are a binary hash (as returned by the upload method) + /// The existence of a BinaryHash implies that there is a binary with this hash on the server + record BinaryHash(String sha256) {} default AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { throw new UnsupportedOperationException("analyse not implemented yet"); @@ -66,6 +78,19 @@ default BinaryHash upload(Path binPath) throws FileNotFoundException, ai.reveng. throw new UnsupportedOperationException("upload not implemented yet"); } + + /** + * Special filters for the collection search endpoint + * https://api.reveng.ai/v2/docs#tag/Collections/operation/list_collections_v2_collections_get + */ + enum SearchFilter { + official_only, + user_only, + team_only, + public_only, + hide_empty + } + default List searchCollections(String searchTerm, @Nullable List filter, int limit, @@ -98,6 +123,17 @@ default Optional getFunctionDataTypes(AnalysisID analysi throw new UnsupportedOperationException("getFunctionDataTypes not implemented yet"); } + default FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID) { + return listFunctionDataTypesForAnalysis(analysisID, null); + } + + default FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID, @Nullable List ids) { + throw new UnsupportedOperationException("listFunctionDataTypesForAnalysis not implemented yet"); + } + + default FunctionDataTypesList listFunctionDataTypesForFunctions(List functionIDs) { + throw new UnsupportedOperationException("listFunctionDataTypesForFunctions not implemented yet"); + } @Deprecated default AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { @@ -134,11 +170,22 @@ default FunctionDetails getFunctionDetails(FunctionID id) { throw new UnsupportedOperationException("getFunctionInfo not implemented yet"); } - default AutoUnstripResponse autoUnstrip(AnalysisID analysisID) { + /// + /// Typed Box Placeholder for {@link AutoUnstripResponse} + record TypedAutoUnstripResponse( + AutoUnstripResponse autoUnstripResponse + ) { + } + + /// {@link MatchedFunctionSuggestion} + record TypedAutoUnstripMatch(MatchedFunctionSuggestion suggestedFunction) { } + + + default TypedAutoUnstripResponse autoUnstrip(AnalysisID analysisID) { throw new UnsupportedOperationException("autoUnstrip not implemented yet"); } - default AutoUnstripResponse aiUnstrip(AnalysisID analysisID) { + default TypedAutoUnstripResponse aiUnstrip(AnalysisID analysisID) { throw new UnsupportedOperationException("aiUnstrip not implemented yet"); } @@ -158,11 +205,11 @@ default ai.reveng.model.Basic getAnalysisBasicInfo(AnalysisID analysisID) throws throw new UnsupportedOperationException("getAnalysisBasicInfo not implemented yet"); } - default FunctionMatchingBatchResponse analysisFunctionMatching(AnalysisID analysisID, AnalysisFunctionMatchingRequest request) throws ApiException { + default FunctionMatchingResponse analysisFunctionMatching(AnalysisID analysisID, AnalysisFunctionMatchingRequest request) throws ApiException { throw new UnsupportedOperationException("analysisFunctionMatching not implemented yet"); } - default FunctionMatchingBatchResponse functionFunctionMatching(FunctionMatchingRequest request) throws ApiException { + default FunctionMatchingResponse functionFunctionMatching(FunctionMatchingRequest request) throws ApiException { throw new UnsupportedOperationException("functionFunctionMatching not implemented yet"); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java index b31821e..804b314 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java @@ -1,20 +1,14 @@ package ai.reveng.toolkit.ghidra.core.services.api.mocks; -import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; -import ai.reveng.toolkit.ghidra.core.services.api.ModelName; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.InvalidAPIInfoException; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.HexFormat; -import java.util.List; -import java.util.Map; import java.util.Objects; public class UnimplementedAPI implements TypedApiInterface { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisID.java deleted file mode 100644 index 2cdacdc..0000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisID.java +++ /dev/null @@ -1,7 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -/// This is a special box type for an analysis ID -/// It enforces that the integer is specifically an analysis ID, -/// and it implies that the user has access to this ID -public record AnalysisID(int id) { -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java index a0e58ed..639c490 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java @@ -1,20 +1,17 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; -import ai.reveng.invoker.ApiException; -import ai.reveng.model.BaseResponseBasic; import ai.reveng.model.Basic; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; -import org.json.JSONObject; /// This is a remnant of an older class that contained the analysis result data directly. /// Now it's a wrapper around the generated Basic class with some shim methods for convenience. public record AnalysisResult( - AnalysisID analysisID, + TypedApiInterface.AnalysisID analysisID, Basic base_response_basic ) { - public BinaryHash sha_256_hash() { - return new BinaryHash(base_response_basic().getSha256Hash()); + public TypedApiInterface.BinaryHash sha_256_hash() { + return new TypedApiInterface.BinaryHash(base_response_basic().getSha256Hash()); } public String binary_name() { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AutoUnstripResponse.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AutoUnstripResponse.java deleted file mode 100644 index 410c140..0000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AutoUnstripResponse.java +++ /dev/null @@ -1,73 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - - -import org.json.JSONArray; -import org.json.JSONObject; -import java.util.List; -import java.util.ArrayList; - -/** - * Structured response for the auto unstrip endpoint - { - "progress": 0, - "status": "string", - "total_time": 0, - "matches": [ - { - "function_id": 0, - "function_vaddr": 0, - "suggested_name": "string" - } - ], - "applied": true, - "error_message": "string" - } - * - */ -public record AutoUnstripResponse( - int progress, - String status, - int total_time, - List matches, - boolean applied, - String error_message -) { - - /** - * Represents a single match in the auto unstrip response - */ - public record Match( - FunctionID function_id, - long function_vaddr, - String suggested_name, - String suggested_demangled_name - ) { - public static Match fromJSONObject(JSONObject json) { - return new Match( - new FunctionID(json.getInt("function_id")), - json.getLong("function_vaddr"), - json.getString("suggested_name"), - json.getString("suggested_demangled_name") - ); - } - } - - public static AutoUnstripResponse fromJSONObject(JSONObject json) { - List matches = new ArrayList<>(); - if (!json.isNull("matches")) { - JSONArray matchesArray = json.getJSONArray("matches"); - for (int i = 0; i < matchesArray.length(); i++) { - matches.add(Match.fromJSONObject(matchesArray.getJSONObject(i))); - } - } - - return new AutoUnstripResponse( - json.getInt("progress"), - json.getString("status"), - json.getInt("total_time"), - matches, - json.getBoolean("applied"), - !json.isNull("error_message") ? json.getString("error_message") : null - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryHash.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryHash.java deleted file mode 100644 index 43aaeec..0000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryHash.java +++ /dev/null @@ -1,20 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -import org.json.JSONObject; - -/* - * Data type for all reveng API responses or parameters that are a binary hash (as returned by the upload method) - * The existence of a BinaryHash implies that there is a binary with this hash on the server! - * - * This could later be enforced via package private methods - * - */ -public record BinaryHash(String sha256) { - public static BinaryHash fromJsonString(String json) { - return new BinaryHash(new JSONObject(json).getString("sha_256_hash")); - } - public static BinaryHash fromJSONObject(JSONObject json) { - return new BinaryHash(json.getString("sha_256_hash")); - } - -} \ No newline at end of file diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/Collection.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/Collection.java index 5655356..4225621 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/Collection.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/Collection.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; import java.util.List; @@ -18,7 +19,7 @@ * @param tags */ public record Collection( - CollectionID collectionID, + TypedApiInterface.CollectionID collectionID, String collectionName, String description, Integer modelID, @@ -26,11 +27,11 @@ public record Collection( String collectionScope, String creationDate, List tags, - List binaries + List binaries ) { public static Collection fromJSONObject(JSONObject json){ return new Collection( - new CollectionID(json.getInt("collection_id")), + new TypedApiInterface.CollectionID(json.getInt("collection_id")), json.getString("collection_name"), json.getString("description"), json.getInt("model_id"), @@ -38,7 +39,7 @@ public static Collection fromJSONObject(JSONObject json){ json.getString("collection_scope"), json.getString("created_at"), json.has("tags") ? json.getJSONArray("tags").toList().stream().map(Object::toString).toList() : null, - json.has("binaries") ? json.getJSONArray("binaries").toList().stream().map( rawID -> new AnalysisID((Integer) rawID)).toList() : null + json.has("binaries") ? json.getJSONArray("binaries").toList().stream().map( rawID -> new TypedApiInterface.AnalysisID((Integer) rawID)).toList() : null ); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/CollectionID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/CollectionID.java deleted file mode 100644 index 583b556..0000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/CollectionID.java +++ /dev/null @@ -1,4 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -public record CollectionID(int id) { -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/DataTypeList.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/DataTypeList.java index 207a656..a2efa25 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/DataTypeList.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/DataTypeList.java @@ -1,10 +1,14 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.model.FunctionDataTypesList; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; /** * Example in data_types_batch_response.json + * @deprecated Use OpenAPI {@link FunctionDataTypesList} */ +@Deprecated public record DataTypeList( int totalCount, int totalDataTypesCount, @@ -22,7 +26,7 @@ public static DataTypeList fromJson(JSONObject json) { return new DataTypeList(totalCount, totalDataTypesCount, dataTypes); } - public FunctionDataTypeStatus statusForFunction(FunctionID functionID) { + public FunctionDataTypeStatus statusForFunction(TypedApiInterface.FunctionID functionID) { for (FunctionDataTypeStatus status : dataTypes) { if (status.functionID().equals(functionID)) { return status; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDataTypeStatus.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDataTypeStatus.java index eb5c9a3..e2ca65c 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDataTypeStatus.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDataTypeStatus.java @@ -1,5 +1,7 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.model.FunctionDataTypes; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.FunctionDataTypeMessage; import org.json.JSONObject; @@ -44,14 +46,16 @@ * }, * "status": "completed" * } + * @deprecated Use {@link FunctionDataTypes} instead */ +@Deprecated public record FunctionDataTypeStatus( boolean completed, Optional data_types, // JSONObject data_types, String status, @Nullable Integer dataTypesVersion, - @Nullable FunctionID functionID + @Nullable TypedApiInterface.FunctionID functionID ) { public static FunctionDataTypeStatus fromJson(JSONObject json) { @@ -67,7 +71,7 @@ public static FunctionDataTypeStatus fromJson(JSONObject json) { !json.isNull("data_types") ? Optional.of(FunctionDataTypeMessage.fromJsonObject(json.getJSONObject("data_types"))) : Optional.empty(), json.getString("status"), dataTypesVersion, - json.has("function_id") ? new FunctionID(json.getInt("function_id")) : null + json.has("function_id") ? new TypedApiInterface.FunctionID(json.getInt("function_id")) : null ); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java index 4f1c825..b1c7414 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java @@ -1,31 +1,32 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; import ai.reveng.model.FunctionsDetailResponse; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; /** * Record representing detailed function information from the RevEng.AI API */ public record FunctionDetails( - FunctionID functionId, + TypedApiInterface.FunctionID functionId, String mangledFunctionName, Long functionVaddr, Long functionSize, - AnalysisID analysisId, + TypedApiInterface.AnalysisID analysisId, String binaryName, - BinaryHash sha256Hash, + TypedApiInterface.BinaryHash sha256Hash, String demangledName ) { public static FunctionDetails fromServerResponse(FunctionsDetailResponse response) { return new FunctionDetails( - new FunctionID(response.getFunctionId()), + new TypedApiInterface.FunctionID(response.getFunctionId()), response.getFunctionNameMangled(), response.getFunctionVaddr(), response.getFunctionSize().longValue(), - new AnalysisID(response.getAnalysisId()), + new TypedApiInterface.AnalysisID(response.getAnalysisId()), response.getBinaryName(), - new BinaryHash(response.getSha256Hash()), + new TypedApiInterface.BinaryHash(response.getSha256Hash()), response.getFunctionName() ); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionID.java deleted file mode 100644 index 6a03f7c..0000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionID.java +++ /dev/null @@ -1,10 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -/** - * Data type to represent the RevEng.AI API concept of a function ID - */ -public record FunctionID( - long value - -) { -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionInfo.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionInfo.java index a8c28ee..8817019 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionInfo.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionInfo.java @@ -1,9 +1,10 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; public record FunctionInfo( - FunctionID functionID, + TypedApiInterface.FunctionID functionID, String functionName, String functionMangledName, // This is an absolute address @@ -12,7 +13,7 @@ public record FunctionInfo( ) { public static FunctionInfo fromJSONObject(JSONObject json) { return new FunctionInfo( - new FunctionID(json.getInt("function_id")), + new TypedApiInterface.FunctionID(json.getInt("function_id")), json.getString("function_name"), json.getString("function_mangled_name"), json.getLong("function_vaddr"), diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionMatch.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionMatch.java index 66894fe..1d14351 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionMatch.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionMatch.java @@ -1,36 +1,42 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.model.MatchedFunction; +import ai.reveng.toolkit.ghidra.binarysimilarity.ui.functionmatching.AbstractFunctionMatchingDialog; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; -/** - * @param origin_function_id - * @param nearest_neighbor_id - * @param nearest_neighbor_function_name - * @param nearest_neighbor_binary_name - * @param nearest_neighbor_sha_256_hash - * @param nearest_neighbor_debug - * @param similarity - */ +import javax.annotation.Nullable; +import java.math.BigDecimal; + +/// The Typed Class to represent a FunctionMatch returned by the RevEng.AI API +/// At the very least it needs to contain the origin_function_id and nearest_neighbor_id +/// Other fields may be null depending on the context in which the FunctionMatch was returned +/// They can be derived from the ids via the API if needed +/// For representing the combination of a local Ghidra Function and its FunctionMatch, use {@link GhidraFunctionMatch} public record FunctionMatch( - FunctionID origin_function_id, - FunctionID nearest_neighbor_id, + TypedApiInterface.FunctionID origin_function_id, + TypedApiInterface.FunctionID nearest_neighbor_id, String nearest_neighbor_function_name, + String nearest_neighbor_mangled_function_name, String nearest_neighbor_binary_name, - BinaryHash nearest_neighbor_sha_256_hash, + TypedApiInterface.BinaryHash nearest_neighbor_sha_256_hash, Boolean nearest_neighbor_debug, - double similarity + BigDecimal similarity, + BigDecimal confidence + ) { - public static FunctionMatch fromJSONObject(JSONObject json) { + + public static FunctionMatch fromMatchedFunctionAPIType(MatchedFunction matchedFunction, TypedApiInterface.FunctionID originFunctionID) { return new FunctionMatch( - new FunctionID(json.getInt("origin_function_id")), - new FunctionID(json.getInt("nearest_neighbor_id")), - json.getString("nearest_neighbor_function_name"), - json.getString("nearest_neighbor_binary_name"), - new BinaryHash(json.getString("nearest_neighbor_sha_256_hash")), - json.has("nearest_neighbor_debug") ? json.getBoolean("nearest_neighbor_debug") : null, - // This is called confidence for legacy reasons, but it is actually the similarity - json.getDouble("confidence") + originFunctionID, + new TypedApiInterface.FunctionID(matchedFunction.getFunctionId()), + matchedFunction.getFunctionName(), + matchedFunction.getMangledName(), + matchedFunction.getBinaryName(), + new TypedApiInterface.BinaryHash(matchedFunction.getSha256Hash()), + matchedFunction.getDebug(), + matchedFunction.getSimilarity(), + matchedFunction.getConfidence() ); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionNameScore.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionNameScore.java index 298c770..837a528 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionNameScore.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionNameScore.java @@ -1,15 +1,16 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; public record FunctionNameScore( - FunctionID functionID, + TypedApiInterface.FunctionID functionID, BoxPlot score ) { public static FunctionNameScore fromJSONObject(JSONObject jsonObject) { var boxplotJson = jsonObject.getJSONObject("box_plot"); return new FunctionNameScore( - new FunctionID(jsonObject.getInt("function_id")), + new TypedApiInterface.FunctionID(jsonObject.getInt("function_id")), BoxPlot.fromJSONObject(boxplotJson) ); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatch.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatch.java index e9fc5ac..8dfb3fc 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatch.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatch.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ghidra.program.model.listing.Function; /** @@ -11,20 +12,9 @@ public record GhidraFunctionMatch( Function function, FunctionMatch functionMatch ) { - - public String nearest_neighbor_function_name() { - return functionMatch.nearest_neighbor_function_name(); - } - - public String nearest_neighbor_binary_name() { - return functionMatch.nearest_neighbor_binary_name(); - } - - public FunctionID nearest_neighbor_id() { + // The following methods are just convenience methods to access the FunctionMatch fields + // This simplifies using this class in a stream with method references + public TypedApiInterface.FunctionID nearest_neighbor_id() { return functionMatch.nearest_neighbor_id(); } - public double similarity() { - return functionMatch.similarity(); - } - } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatchWithSignature.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatchWithSignature.java index aed0691..e31f7a9 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatchWithSignature.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatchWithSignature.java @@ -1,69 +1,21 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.FunctionDataTypeMessage; +import ghidra.program.model.data.FunctionDefinitionDataType; import ghidra.program.model.listing.Function; +import javax.annotation.Nullable; import java.util.Optional; /** - * The signature is not final because it can be computed on demand - * * @param function The local function that we searched for matches for * @param functionMatch A match that was found and returned by the RevEng.AI server * @param signature The optional signature of the function match */ -public class GhidraFunctionMatchWithSignature { - private final Function function; - private final FunctionMatch functionMatch; - private Optional signature; - private final Optional nameScore; - - public GhidraFunctionMatchWithSignature( - Function function, - FunctionMatch functionMatch, - Optional signature, - Optional nameScore) { - if (function == null) { - throw new IllegalArgumentException("Function cannot be null"); - } - if (functionMatch == null) { - throw new IllegalArgumentException("FunctionMatch cannot be null"); - } - if (nameScore == null) { - throw new IllegalArgumentException("NameScore cannot be null, use Optional.empty() instead"); - } - this.function = function; - this.functionMatch = functionMatch; - this.signature = signature; - this.nameScore = nameScore; - } - - public GhidraFunctionMatchWithSignature(GhidraFunctionMatch functionMatch, FunctionDataTypeMessage signature, BoxPlot nameScore) { - this(functionMatch.function(), functionMatch.functionMatch(), Optional.ofNullable(signature), Optional.ofNullable(nameScore)); - } - - public Function function() { - return function; - } - - public FunctionMatch functionMatch() { - return functionMatch; - } - - public Optional signature() { - return signature; - } - - public void setSignature(Optional signature) { - this.signature = signature; - } - - public Optional nameScore() { - return nameScore; - } - - public FunctionID nearest_neighbor_id() { - return functionMatch.nearest_neighbor_id(); - } -} \ No newline at end of file +public record GhidraFunctionMatchWithSignature( + Function function, + FunctionMatch functionMatch, + @Nullable FunctionDefinitionDataType signature +){} \ No newline at end of file diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java index bd1118f..568c0eb 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java @@ -1,35 +1,35 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; -/* -{"analyses":[ -{"analysis_scope":"PRIVATE","binary_id":27665,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:57:18 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27664,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:55:28 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27663,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:53:33 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27662,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:32:34 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27661,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:24:44 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27660,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:23:54 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27633,"binary_name":"true","creation":"Thu, 18 Apr 2024 17:34:42 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27632,"binary_name":"true","creation":"Thu, 18 Apr 2024 17:33:36 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27631,"binary_name":"ls","creation":"Thu, 18 Apr 2024 17:17:48 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"2e7ef3da2b295c77820a0782b00c9d607cd48d5c8f7458a76b0921bec20a30ae","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27630,"binary_name":"ls","creation":"Thu, 18 Apr 2024 17:16:48 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"2e7ef3da2b295c77820a0782b00c9d607cd48d5c8f7458a76b0921bec20a30ae","status":"Error"}]} - */ +/// {"analyses":[ +/// {"analysis_scope":"PRIVATE","binary_id":27665,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:57:18 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27664,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:55:28 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27663,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:53:33 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27662,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:32:34 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27661,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:24:44 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27660,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:23:54 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27633,"binary_name":"true","creation":"Thu, 18 Apr 2024 17:34:42 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27632,"binary_name":"true","creation":"Thu, 18 Apr 2024 17:33:36 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27631,"binary_name":"ls","creation":"Thu, 18 Apr 2024 17:17:48 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"2e7ef3da2b295c77820a0782b00c9d607cd48d5c8f7458a76b0921bec20a30ae","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27630,"binary_name":"ls","creation":"Thu, 18 Apr 2024 17:16:48 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"2e7ef3da2b295c77820a0782b00c9d607cd48d5c8f7458a76b0921bec20a30ae","status":"Error"}]} +/// @deprecated Use {@link AnalysisResult)} @Deprecated public record LegacyAnalysisResult( - AnalysisID analysis_id, + TypedApiInterface.AnalysisID analysis_id, @Deprecated BinaryID binary_id, String binary_name, String creation, int model_id, String model_name, - BinaryHash sha_256_hash, + TypedApiInterface.BinaryHash sha_256_hash, AnalysisStatus status, long base_address, String function_boundaries_hash ) { public static LegacyAnalysisResult fromJSONObject(JSONObject json) { return new LegacyAnalysisResult( - new AnalysisID(json.getInt("analysis_id")), + new TypedApiInterface.AnalysisID(json.getInt("analysis_id")), new BinaryID(json.getInt("binary_id")), json.getString("binary_name"), json.getString("creation"), json.getInt("model_id"), json.getString("model_name"), - new BinaryHash(json.getString("sha_256_hash")), + new TypedApiInterface.BinaryHash(json.getString("sha_256_hash")), AnalysisStatus.valueOf(json.getString("status")), json.getLong("base_address"), json.getString("function_boundaries_hash") diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyCollection.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyCollection.java index 20dad75..c2c0ea8 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyCollection.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyCollection.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; import java.util.List; @@ -18,7 +19,7 @@ * @param tags */ public record LegacyCollection( - CollectionID collectionID, + TypedApiInterface.CollectionID collectionID, String collectionScope, String collectionName, String owner, @@ -29,7 +30,7 @@ public record LegacyCollection( ) { public static LegacyCollection fromJSONObject(JSONObject json){ return new LegacyCollection( - new CollectionID(json.getInt("collection_id")), + new TypedApiInterface.CollectionID(json.getInt("collection_id")), json.getString("collection_scope"), json.getString("collection_name"), json.getString("collection_owner"), diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/SearchFilter.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/SearchFilter.java deleted file mode 100644 index a74a97c..0000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/SearchFilter.java +++ /dev/null @@ -1,13 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -/** - * Special filters for the collection search endpoint - * https://api.reveng.ai/v2/docs#tag/Collections/operation/list_collections_v2_collections_get - */ -public enum SearchFilter { - official_only, - user_only, - team_only, - public_only, - hide_empty -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArtifact.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArtifact.java index 6cb1e6d..6b10509 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArtifact.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArtifact.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +import ai.reveng.model.FunctionTypeOutput; import org.json.JSONObject; import java.util.ArrayList; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDataTypeMessage.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDataTypeMessage.java index 5a84ab7..2225ef0 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDataTypeMessage.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDataTypeMessage.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +import ai.reveng.model.FunctionInfoOutput; import org.json.JSONObject; /** @@ -227,14 +228,17 @@ * * This object isn't part of the BinSync types * The func_deps members are either typedefs or structures + * + * @deprecated see {@link FunctionInfoOutput} */ +@Deprecated public record FunctionDataTypeMessage( FunctionArtifact func_types, FunctionDependencies func_deps ) { public static FunctionDataTypeMessage fromJsonObject(JSONObject dataTypes) { return new FunctionDataTypeMessage( - FunctionArtifact.fromJsonObject(dataTypes.getJSONObject("func_types")), + FunctionArtifact.fromJsonObject(dataTypes. getJSONObject("func_types")), FunctionDependencies.fromJsonObject(dataTypes.getJSONArray("func_deps")) ); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDependencies.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDependencies.java index 3e751b8..4346766 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDependencies.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDependencies.java @@ -1,16 +1,18 @@ package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +import ai.reveng.model.*; +import ghidra.util.Msg; import org.json.JSONArray; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Optional; /** * Describes the dependencies of a function. - * Has no corresponding artifact in BinSync, in the RevEng.AI response it's just a list of + * Has no corresponding artifact in BinSync or the RevEng.AI OpenAPI spec. In the RevEng.AI response it's just a list of * type artifacts - * * For ease of use we split it into its own record, and split the type of artifacts into their own * arrays. This needs to happen for deserialization anyway */ @@ -18,6 +20,40 @@ public record FunctionDependencies( Typedef[] typedefs, Struct[] structs ) { + + public static FunctionDependencies fromOpenAPI(List deps) { + if (deps.isEmpty()) { + return null; + } + var typedefs = new ArrayList(); + var structs = new ArrayList(); + + for (var dep : deps) { + var instance = dep.getActualInstance(); + switch (instance) { + case TypeDefinition typedef -> typedefs.add(Typedef.fromOpenAPI(typedef)); + case Structure struct -> structs.add(Struct.fromOpenAPI(struct)); + case Enumeration enumeration -> { + // We don't handle enums for now + } + case GlobalVariable globalVariable -> { + // We don't handle global variables for now + } + default ->{ + Msg.error(FunctionDependencies.class, "Unexpected type dependency: " + instance); + } + + + } + } + + return new FunctionDependencies( + typedefs.toArray(new Typedef[0]), + structs.toArray(new Struct[0]) + ); + + } + public static FunctionDependencies fromJsonObject(JSONArray funcDeps) { // We need to distinguish the object type somehow if (funcDeps.isEmpty()) { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Struct.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Struct.java index db4ff4b..7610114 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Struct.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Struct.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +import ai.reveng.model.Structure; import org.json.JSONObject; /** @@ -35,4 +36,16 @@ public static boolean matches(JSONObject jsonObject) { } + public static Struct fromOpenAPI(Structure struct) { + StructMember[] members = struct.getMembers().values().stream() + .map(StructMember::fromOpenAPI) + .toList().toArray(new StructMember[0]); + + return new Struct( + struct.getLastChange(), + struct.getName(), + struct.getSize(), + members + ); + } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StructMember.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StructMember.java index faa8ab7..d89905b 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StructMember.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StructMember.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +import ai.reveng.model.StructureMember; import org.json.JSONObject; /** @@ -22,4 +23,14 @@ public static StructMember fromJsonObject(JSONObject jsonObject) { jsonObject.getInt("size") ); } + + public static StructMember fromOpenAPI(StructureMember structureMember) { + return new StructMember( + structureMember.getLastChange(), + structureMember.getName(), + structureMember.getOffset(), + structureMember.getType(), + structureMember.getSize() + ); + } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/TypePathAndName.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/TypePathAndName.java index 9b55330..699c302 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/TypePathAndName.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/TypePathAndName.java @@ -7,12 +7,28 @@ public record TypePathAndName( String[] path ) { + + + /// based on `ArtifactLifter.parse_scoped_type` from binsync + /// Takes strings like: + /// + /// - "uint32_t" + /// - "stdint::uint32_t" + /// - "DWARF::stdio.h::off_t" + /// @param str + /// @return public static TypePathAndName fromString(String str){ - String[] parts = str.split("/"); - String name = parts[parts.length - 1]; - String[] path = new String[parts.length - 1]; - System.arraycopy(parts, 0, path, 0, parts.length - 1); - return new TypePathAndName(name, path); + // split into path and name on "::" + if (str.contains("::")) { + String[] pathPlusType = str.split("::"); + var baseType = pathPlusType[pathPlusType.length - 1]; + + String[] parts = new String[pathPlusType.length - 1]; + System.arraycopy(pathPlusType, 0, parts, 0, pathPlusType.length - 1); + return new TypePathAndName(baseType, parts); + } else { + return new TypePathAndName(str, new String[0]); + } } public CategoryPath toCategoryPath(){ diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Typedef.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Typedef.java index 172a61e..57671e3 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Typedef.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Typedef.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +import ai.reveng.model.TypeDefinition; import org.json.JSONObject; import java.util.Set; @@ -26,4 +27,12 @@ public static boolean matches(JSONObject obj) { return obj.keySet().equals(Set.of("last_change", "name", "type")); // return obj.has("type") && obj.has("name"); } + + public static Typedef fromOpenAPI(TypeDefinition typedef) { + return new Typedef( + typedef.getLastChange(), + typedef.getName(), + typedef.getType() + ); + } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java b/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java index 86f6e77..f5b0ff5 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java @@ -1,10 +1,8 @@ package ai.reveng.toolkit.ghidra.devplugin; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionID; import ghidra.framework.plugintool.ComponentProviderAdapter; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.listing.Function; @@ -17,8 +15,8 @@ public class RevEngMetadataProvider extends ComponentProviderAdapter { private final JPanel panel; private final JTextField serverField; - AnalysisID analysisID; - FunctionID functionID; + TypedApiInterface.AnalysisID analysisID; + TypedApiInterface.FunctionID functionID; Function function; JTextField binaryIDField; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java index 145583a..8ee9a1c 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java @@ -22,6 +22,7 @@ import ai.reveng.toolkit.ghidra.binarysimilarity.ui.functionmatching.FunctionLevelFunctionMatchingDialog; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisResultsLoaded; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.core.services.function.export.ExportFunctionBoundariesService; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; @@ -132,13 +133,13 @@ private void setupActions() { "Analysis must have completed before running auto unstrip"); return; } - var knownProgram = apiService.getKnownProgram(program); - if (knownProgram.isEmpty()){ + var analysedProgram = apiService.getAnalysedProgram(program); + if (analysedProgram.isEmpty()){ Msg.info(this, "Program has no saved binary ID"); return; } - var autoUnstrip = new AutoUnstripDialog(tool, knownProgram.get()); + var autoUnstrip = new AutoUnstripDialog(tool, analysedProgram.get()); tool.showDialog(autoUnstrip); }) @@ -259,7 +260,7 @@ private void setupActions() { public void readDataState(SaveState saveState) { int[] rawCollectionIDs = saveState.getInts("collectionIDs", new int[0]); var restoredCollections = Arrays.stream(rawCollectionIDs) - .mapToObj(CollectionID::new) + .mapToObj(TypedApiInterface.CollectionID::new) .map(cID -> apiService.getApi().getCollectionInfo(cID)) .toList(); apiService.setActiveCollections(restoredCollections); @@ -267,7 +268,7 @@ public void readDataState(SaveState saveState) { @Override public void writeDataState(SaveState saveState) { - int[] collectionIDs = apiService.getActiveCollections().stream().map(Collection::collectionID).mapToInt(CollectionID::id).toArray(); + int[] collectionIDs = apiService.getActiveCollections().stream().map(Collection::collectionID).mapToInt(TypedApiInterface.CollectionID::id).toArray(); saveState.putInts("collectionIDs", collectionIDs); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/DevPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/DevPlugin.java index 918a5b1..7c37534 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/DevPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/DevPlugin.java @@ -1,7 +1,7 @@ package ai.reveng.toolkit.ghidra.plugins; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIConflictException; import ai.reveng.toolkit.ghidra.devplugin.RevEngMetadataProvider; import docking.action.builder.ActionBuilder; @@ -50,8 +50,8 @@ public DevPlugin(PluginTool tool) { .onAction(e -> { GhidraRevengService reAIService = tool.getService(GhidraRevengService.class); var api = reAIService.getApi(); - AnalysisID analysisID = reAIService.getAnalysedProgram(currentProgram).orElseThrow().analysisID(); - var functionMap = reAIService.getFunctionMap(currentProgram); + var analysedProgram = reAIService.getAnalysedProgram(currentProgram).orElseThrow(); + var functionMap = analysedProgram.getFunctionMap(); var task = new Task("Generate Signatures", true, true, true) { @Override public void run(TaskMonitor monitor) { @@ -60,7 +60,7 @@ public void run(TaskMonitor monitor) { (fID, function) -> { try { monitor.checkCancelled(); - api.generateFunctionDataTypes(analysisID, List.of(fID)); + api.generateFunctionDataTypes(analysedProgram.analysisID(), List.of(fID)); monitor.incrementProgress(1); } catch (APIConflictException e) { // Already requested diff --git a/src/test/java/ConvertBinSyncArtifactTests.java b/src/test/java/ConvertBinSyncArtifactTests.java index 2a622c5..392bfbc 100644 --- a/src/test/java/ConvertBinSyncArtifactTests.java +++ b/src/test/java/ConvertBinSyncArtifactTests.java @@ -1,5 +1,8 @@ +import ai.reveng.model.FunctionDataTypes; +import ai.reveng.model.FunctionInfoOutput; import ai.reveng.toolkit.ghidra.binarysimilarity.cmds.ComputeTypeInfoTask; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.V2Response; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; @@ -14,21 +17,23 @@ import org.junit.Ignore; import org.junit.Test; +import java.io.IOException; import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; public class ConvertBinSyncArtifactTests extends AbstractRevEngIntegrationTest { - AnalysisID analysisID = new AnalysisID(1337); + TypedApiInterface.AnalysisID analysisID = new TypedApiInterface.AnalysisID(1337); @Test - public void testSimpleGhidraSignatureGeneration() throws DataTypeDependencyException { + public void testSimpleGhidraSignatureGeneration() throws DataTypeDependencyException, IOException { V2Response mockResponse = getMockResponseFromFile("main_fdupes_77846709.json"); - FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var signature = GhidraRevengService.getFunctionSignature(functionDataTypeStatus.data_types().get()); +// FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); + var funcInfo = FunctionDataTypes.fromJson(mockResponse.getJsonData().toString()); + var signature = GhidraRevengService.getFunctionSignature(funcInfo.getDataTypes()).orElseThrow(); assert signature.getName().equals("main"); assert signature.getReturnType().getName().equals("int"); @@ -66,11 +71,11 @@ public void testDependencyToDtm() { } @Test - public void testComplexGhidraSignatureGeneration() throws DataTypeDependencyException { + public void testComplexGhidraSignatureGeneration() throws DataTypeDependencyException, IOException { var mockResponse = getMockResponseFromFile("confirmmatch_fdupes_77846700.json"); - FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var signature = GhidraRevengService.getFunctionSignature(functionDataTypeStatus.data_types().get()); + var funcInfo = FunctionInfoOutput.fromJson(mockResponse.getJsonData().toString()); + var signature = GhidraRevengService.getFunctionSignature(funcInfo).orElseThrow(); assert signature.getName().equals("confirmmatch"); Msg.info(this, signature); @@ -78,22 +83,24 @@ public void testComplexGhidraSignatureGeneration() throws DataTypeDependencyExce @Test - public void testComplexGhidraSignatureGeneration2() throws DataTypeDependencyException { + public void testComplexGhidraSignatureGeneration2() throws DataTypeDependencyException, IOException { var mockResponse = getMockResponseFromFile("summarizematches_fdupes.json"); FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var signature = GhidraRevengService.getFunctionSignature(functionDataTypeStatus.data_types().get()); + var funcInfo = FunctionInfoOutput.fromJson(mockResponse.getJsonData().toString()); + var signature = GhidraRevengService.getFunctionSignature(funcInfo).orElseThrow(); assert signature.getName().equals("summarizematches"); Msg.info(this, signature); } @Test - public void testComplexGhidraSignatureGeneration3() throws DataTypeDependencyException { + public void testComplexGhidraSignatureGeneration3() throws DataTypeDependencyException, IOException { var mockResponse = getMockResponseFromFile("md5_process_fdupes.json"); FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var signature = GhidraRevengService.getFunctionSignature(functionDataTypeStatus.data_types().get()); + var funcInfo = FunctionInfoOutput.fromJson(mockResponse.getJsonData().toString()); + var signature = GhidraRevengService.getFunctionSignature(funcInfo).orElseThrow(); var dtm = signature.getDataTypeManager(); assert signature.getName().equals("md5_process"); @@ -110,11 +117,11 @@ public void testComplexGhidraSignatureGeneration3() throws DataTypeDependencyExc */ @Ignore("Ignored until it can properly distinguish an infinite loop and an exception") @Test - public void testNoLoopForBrokenDeps() throws DataTypeDependencyException { + public void testNoLoopForBrokenDeps() throws DataTypeDependencyException, IOException { var mockResponse = getMockResponseFromFile("errormsg.json"); - FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var signature = GhidraRevengService.getFunctionSignature(functionDataTypeStatus.data_types().get()); + var funcInfo = FunctionInfoOutput.fromJson(mockResponse.getJsonData().toString()); + var signature = GhidraRevengService.getFunctionSignature(funcInfo).orElseThrow(); assert signature.getName().equals("md5_process"); Msg.info(this, signature); @@ -126,10 +133,11 @@ public void testNoLoopForBrokenDeps() throws DataTypeDependencyException { * The function is `registerpair` from `fdupes`: registerpair */ @Test - public void testFunctionPointerArgument() throws DataTypeDependencyException { + public void testFunctionPointerArgument() throws DataTypeDependencyException, IOException { var mockResponse = getMockResponseFromFile("complex_pointer.json"); - FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var signature = GhidraRevengService.getFunctionSignature(functionDataTypeStatus.data_types().get()); + var signature = GhidraRevengService.getFunctionSignature( + FunctionInfoOutput.fromJson(mockResponse.getJsonData().toString()) + ); } @Test @@ -146,7 +154,7 @@ public void testBatchResponse() { var mockResponse = getMockResponseFromFile("data_types_batch_response.json"); DataTypeList batchResponse = DataTypeList.fromJson(mockResponse.getJsonData()); - var r1 = batchResponse.statusForFunction(new FunctionID(266294328)); + var r1 = batchResponse.statusForFunction(new TypedApiInterface.FunctionID(266294328)); assert r1.data_types().orElseThrow().functionName().equals("sort_pairs_by_mtime"); } @@ -155,7 +163,7 @@ public void testDataTypeGenerationTask() throws CancelledException { var mockApi = new TypeGenerationMock(); var task = new ComputeTypeInfoTask( new GhidraRevengService(mockApi), - IntStream.range(0, 5).boxed().map(FunctionID::new).collect(Collectors.toList()), null + IntStream.range(0, 5).boxed().map(TypedApiInterface.FunctionID::new).collect(Collectors.toList()), null ); task.run(TaskMonitor.DUMMY); diff --git a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java index 1adb2f6..26c9d4b 100644 --- a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java +++ b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java @@ -5,6 +5,7 @@ import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.*; @@ -159,6 +160,7 @@ public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { // Check that an analysed program is now known assert service.getAnalysedProgram(program).isPresent(); + var analyzedProgram = service.getAnalysedProgram(program).get(); // Check that the function names have been updated to the one returned by the portal assertEquals("portal_name_demangled", exampleFunc.getName()); @@ -175,19 +177,15 @@ public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { // Check the function ID has been stored in the program options - var funcIDMap = service.getFunctionMap(program); - - var storedFunc = funcIDMap.get(new FunctionID(1)); + var funcIDMap = analyzedProgram.getFunctionMap(); + var storedFunc = funcIDMap.get(new TypedApiInterface.FunctionID(1)); Assert.assertNotNull(storedFunc); assertEquals("portal_name_demangled", storedFunc.getName()); // Check the function mangled name has been stored - var mangledNamesMap = service.getFunctionMangledNamesMap(program); - - assertTrue(mangledNamesMap.isPresent()); - assertEquals("portal_name_mangled", mangledNamesMap.get().getString(exampleFunc.getEntryPoint())); - +// assertEquals("portal_name_mangled", mangledNamesMap.get().getString(exampleFunc.getEntryPoint())); + assertEquals("portal_name_mangled", analyzedProgram.getMangledNameForFunction(exampleFunc)); // TODO: What else should happen when the analysis is finished? } } diff --git a/src/test/java/ai/reveng/TestAnalysisLogComponent.java b/src/test/java/ai/reveng/TestAnalysisLogComponent.java index 57938c3..51eb196 100644 --- a/src/test/java/ai/reveng/TestAnalysisLogComponent.java +++ b/src/test/java/ai/reveng/TestAnalysisLogComponent.java @@ -2,6 +2,7 @@ import ai.reveng.toolkit.ghidra.binarysimilarity.ui.misc.AnalysisLogComponent; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.plugins.AnalysisManagementPlugin; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; @@ -26,7 +27,7 @@ private GhidraRevengService.ProgramWithID getPlaceHolderID() throws Exception{ var program = builder.getProgram(); return new GhidraRevengService.ProgramWithID( program, - new AnalysisID(1) + new TypedApiInterface.AnalysisID(1) ); } diff --git a/src/test/java/ai/reveng/TestMockableService.java b/src/test/java/ai/reveng/TestMockableService.java index 92616f8..f1014d3 100644 --- a/src/test/java/ai/reveng/TestMockableService.java +++ b/src/test/java/ai/reveng/TestMockableService.java @@ -2,9 +2,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryHash; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; import org.junit.Test; diff --git a/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java b/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java index 7ee26c4..7294458 100644 --- a/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java +++ b/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java @@ -3,7 +3,6 @@ import ai.reveng.invoker.ApiException; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; @@ -13,6 +12,7 @@ import java.util.Optional; +@SuppressWarnings("deprecation") // This test ensures upgrade from deprecated Binary ID usage public class TestUpgradeFromBinaryID extends RevEngMockableHeadedIntegrationTest { /// Tests the logic for handling a program that has only a binary ID stored in its properties diff --git a/src/test/java/ai/reveng/UnstripTest.java b/src/test/java/ai/reveng/UnstripTest.java index bc6820c..d5c946e 100644 --- a/src/test/java/ai/reveng/UnstripTest.java +++ b/src/test/java/ai/reveng/UnstripTest.java @@ -1,9 +1,12 @@ package ai.reveng; import ai.reveng.invoker.ApiException; +import ai.reveng.model.AutoUnstripResponse; +import ai.reveng.model.MatchedFunctionSuggestion; import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; -import ai.reveng.toolkit.ghidra.core.services.api.types.*; +import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; import ai.reveng.toolkit.ghidra.plugins.BinarySimilarityPlugin; import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; @@ -26,15 +29,22 @@ public void testFinishedUnstrip() throws Exception { @Override - public AutoUnstripResponse autoUnstrip(AnalysisID analysisID) { - return new AutoUnstripResponse( - 100, - "STATUS", - 0, - List.of(new AutoUnstripResponse.Match(new FunctionID(1), 0x1000, "unstripped_function_name", "unstripped_function_name_demangled" ) ), - false, - null - ); + public TypedAutoUnstripResponse autoUnstrip(AnalysisID analysisID) { + var r = new AutoUnstripResponse() + .progress(100) + .status("STATUS") + .applied(false) + .totalTime(0) + .matches(List.of( + new MatchedFunctionSuggestion() + .functionId(1L) + .functionVaddr(0x1000L) + .suggestedName("unstripped_function_name") + .suggestedDemangledName("unstripped_function_name_demangled") + ) + ); + + return new TypedAutoUnstripResponse(r); } @Override @@ -77,14 +87,13 @@ public List getFunctionInfo(AnalysisID analysisID) { public void testProgressingUnstrip() throws Exception { var tool = env.getTool(); Iterator responses = List.of( - new AutoUnstripResponse( - 0, - "QUEUED", - 0, - List.of(), - false, - null - ), + new AutoUnstripResponse() + .progress(0) + .status("QUEUED") + .applied(false) + .totalTime(0) + .matches(List.of()) + , // The poll interval is not configurable, so we can only test two responses before hitting a Ghidra test timeout // new AutoUnstripResponse( // 50, @@ -94,22 +103,26 @@ public void testProgressingUnstrip() throws Exception { // false, // null // ), - new AutoUnstripResponse( - 100, - "COMPLETED", - 0, - List.of(new AutoUnstripResponse.Match(new FunctionID(1), 0x1000, "unstripped_function_name", "unstripped_function_name_demangled") ), - false, - null - ) + new AutoUnstripResponse() + .progress(100) + .status("COMPLETED") + .totalTime(1) + .matches(List.of( + new MatchedFunctionSuggestion() + .functionId(1L) + .functionVaddr(0x1000L) + .suggestedName("unstripped_function_name") + .suggestedDemangledName("unstripped_function_name_demangled") + )) + .applied(false) ).iterator(); var service = addMockedService(tool, new UnimplementedAPI() { @Override - public AutoUnstripResponse autoUnstrip(AnalysisID analysisID) { - return responses.next(); + public TypedAutoUnstripResponse autoUnstrip(AnalysisID analysisID) { + return new TypedAutoUnstripResponse(responses.next()); } @Override diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilderTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilderTest.java index a3cc9ce..eb465e9 100644 --- a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilderTest.java +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilderTest.java @@ -2,7 +2,6 @@ import ai.reveng.model.AnalysisCreateRequest; import ai.reveng.model.Tag; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryHash; import org.junit.Test; import java.util.Arrays; @@ -16,7 +15,7 @@ public class AnalysisOptionsBuilderTest { public void testToAnalysisCreateRequest_WithNoTags() { // Create a builder with no tags AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash("a".repeat(64))) + .hash(new TypedApiInterface.BinaryHash("a".repeat(64))) .fileName("test.bin"); AnalysisCreateRequest request = builder.toAnalysisCreateRequest(); @@ -31,7 +30,7 @@ public void testToAnalysisCreateRequest_WithNoTags() { public void testToAnalysisCreateRequest_WithValidTags() { // Create a builder with valid tags AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash("b".repeat(64))) + .hash(new TypedApiInterface.BinaryHash("b".repeat(64))) .fileName("test.bin") .addTag("malware") .addTag("suspicious"); @@ -54,7 +53,7 @@ public void testToAnalysisCreateRequest_WithValidTags() { public void testToAnalysisCreateRequest_WithEmptyStringTag() { // Create a builder with an empty string tag AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash("c".repeat(64))) + .hash(new TypedApiInterface.BinaryHash("c".repeat(64))) .fileName("test.bin") .addTag("") .addTag(" ") // whitespace only @@ -72,7 +71,7 @@ public void testToAnalysisCreateRequest_WithEmptyStringTag() { public void testToAnalysisCreateRequest_WithMultipleTags() { // Create a builder with multiple tags using addTags method AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash("d".repeat(64))) + .hash(new TypedApiInterface.BinaryHash("d".repeat(64))) .fileName("test.bin") .addTags(Arrays.asList("tag1", "tag2", "tag3")); @@ -87,7 +86,7 @@ public void testToAnalysisCreateRequest_WithMultipleTags() { public void testToAnalysisCreateRequest_WithOnlyEmptyTags() { // Create a builder with only empty/whitespace tags AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash("e".repeat(64))) + .hash(new TypedApiInterface.BinaryHash("e".repeat(64))) .fileName("test.bin") .addTag("") .addTag(" ") @@ -108,7 +107,7 @@ public void testToAnalysisCreateRequest_BasicFields() { String hash = "f".repeat(64); AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash(hash)) + .hash(new TypedApiInterface.BinaryHash(hash)) .fileName(filename); AnalysisCreateRequest request = builder.toAnalysisCreateRequest(); @@ -121,7 +120,7 @@ public void testToAnalysisCreateRequest_BasicFields() { public void testGetTags() { // Test the getTags method AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash("a".repeat(64))) + .hash(new TypedApiInterface.BinaryHash("a".repeat(64))) .fileName("test.bin"); // Initially should be empty From 97d4e9de79b0d7624d4c5c0ab693925522867ea9 Mon Sep 17 00:00:00 2001 From: Florian Magin Date: Sun, 7 Dec 2025 19:17:55 +0100 Subject: [PATCH 10/10] Fix test cases and optimize name/type pulling function (only one API call for all func names) --- .../services/api/GhidraRevengService.java | 39 ++++++- .../services/api/mocks/UnimplementedAPI.java | 10 ++ .../java/ConvertBinSyncArtifactTests.java | 1 + src/test/java/HelperTests.java | 2 +- .../reveng/PortalAnalysisIntegrationTest.java | 101 +++++++++++------- src/test/java/ai/reveng/UnstripTest.java | 33 ++++-- 6 files changed, 138 insertions(+), 48 deletions(-) diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java index 3915e7c..5611b4e 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/GhidraRevengService.java @@ -22,6 +22,7 @@ import ghidra.program.model.address.Address; import ghidra.program.model.data.*; import ghidra.program.model.data.Structure; +import ghidra.program.model.listing.CircularDependencyException; import ghidra.program.model.listing.Function; import ghidra.program.model.listing.FunctionSignature; import ghidra.program.model.listing.Program; @@ -33,6 +34,7 @@ import ghidra.util.data.DataTypeParser; import ghidra.util.exception.CancelledException; import ghidra.util.exception.DuplicateNameException; +import ghidra.util.exception.InvalidInputException; import ghidra.util.exception.NoValueException; import ghidra.util.task.TaskMonitor; import org.jetbrains.annotations.NotNull; @@ -50,6 +52,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; +import static ai.reveng.toolkit.ghidra.plugins.BinarySimilarityPlugin.REVENG_AI_NAMESPACE; import static ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage.OPTION_KEY_ANALYSIS_ID; @@ -123,6 +126,19 @@ private ProgramWithID addAnalysisIDtoProgramOptions(Program program, TypedApiInt return new ProgramWithID(program, analysisID); } + private Namespace getRevEngAINameSpace(Program program) { + Namespace revengMatchNamespace = null; + try { + revengMatchNamespace = program.getSymbolTable().getOrCreateNameSpace( + program.getGlobalNamespace(), + REVENG_AI_NAMESPACE, + SourceType.ANALYSIS + ); + } catch (DuplicateNameException | InvalidInputException e) { + throw new RuntimeException(e); + } + return revengMatchNamespace; + } /** * Tries to find a BinaryID for a given program * If the program already has a BinaryID associated with it, it will return that @@ -321,6 +337,17 @@ public List pullFunctionInfoFromAnalysis(AnalysedProgram analysedP int failedRenames = 0; + var revEngNamespace = getRevEngAINameSpace(analysedProgram.program()); + + Map functionInfoMap = api.getFunctionInfo(analysedProgram.analysisID()).stream() + .collect( + Collectors.toMap( + FunctionInfo::functionID, + fi -> fi + ) + ); + + Map signatureMap = api.listFunctionDataTypesForAnalysis(analysedProgram.analysisID).getItems() .stream() .filter(item -> item.getStatus().equals("completed")) @@ -350,11 +377,12 @@ public List pullFunctionInfoFromAnalysis(AnalysedProgram analysedP } // Get the current name on the server side - FunctionDetails details = api.getFunctionDetails(fID.get().functionID); +// FunctionDetails details = api.getFunctionDetails(fID.get().functionID); + FunctionInfo details = functionInfoMap.get(fID.get().functionID); // Extract the mangled name from Ghidra - var revEngMangledName = details.mangledFunctionName(); - var revEngDemangledName = details.demangledName(); + var revEngMangledName = details.functionMangledName(); + var revEngDemangledName = details.functionName(); // Skip invalid function mangled names if (revEngMangledName.contains(" ") || revEngDemangledName.contains(" ")) { @@ -399,6 +427,11 @@ public List pullFunctionInfoFromAnalysis(AnalysedProgram analysedP // but just to be safe and make that assumption explicit we check it explicitly if (!function.getSymbol().getName(false).equals(revEngDemangledName)) { Msg.info(this, "Renaming function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); + try { + function.setParentNamespace(revEngNamespace); + } catch (DuplicateNameException | InvalidInputException | CircularDependencyException e) { + throw new RuntimeException(e); + } var success = new SetFunctionNameCmd(function.getEntryPoint(), revEngDemangledName, SourceType.ANALYSIS) .applyTo(analysedProgram.program()); if (success) { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java index 804b314..6a8c339 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java @@ -1,7 +1,9 @@ package ai.reveng.toolkit.ghidra.core.services.api.mocks; +import ai.reveng.model.FunctionDataTypesList; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; +import org.jetbrains.annotations.Nullable; import java.io.IOException; import java.nio.file.Files; @@ -9,6 +11,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.HexFormat; +import java.util.List; import java.util.Objects; public class UnimplementedAPI implements TypedApiInterface { @@ -50,4 +53,11 @@ public BinaryHash upload(Path binPath) { } } + + /// This gets called when registering the initial mock analysis + /// it just pretends that there is no type info available + @Override + public FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID, @Nullable List ids) { + return new FunctionDataTypesList(); + } } diff --git a/src/test/java/ConvertBinSyncArtifactTests.java b/src/test/java/ConvertBinSyncArtifactTests.java index 392bfbc..511fef1 100644 --- a/src/test/java/ConvertBinSyncArtifactTests.java +++ b/src/test/java/ConvertBinSyncArtifactTests.java @@ -22,6 +22,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +@Ignore("Integration tests that rely on mock data from files") public class ConvertBinSyncArtifactTests extends AbstractRevEngIntegrationTest { TypedApiInterface.AnalysisID analysisID = new TypedApiInterface.AnalysisID(1337); diff --git a/src/test/java/HelperTests.java b/src/test/java/HelperTests.java index ddd3fde..17895b9 100644 --- a/src/test/java/HelperTests.java +++ b/src/test/java/HelperTests.java @@ -6,7 +6,7 @@ public class HelperTests { @Test public void testPathSplitting(){ - var path = TypePathAndName.fromString("a/b/c"); + var path = TypePathAndName.fromString("a::b::c"); assert path.name().equals("c"); assert path.path().length == 2; assert path.path()[0].equals("a"); diff --git a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java index 26c9d4b..6e22f6d 100644 --- a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java +++ b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java @@ -1,6 +1,10 @@ package ai.reveng; import ai.reveng.invoker.ApiException; +import ai.reveng.model.FunctionDataTypesList; +import ai.reveng.model.FunctionDataTypesListItem; +import ai.reveng.model.FunctionInfoOutput; +import ai.reveng.model.FunctionTypeOutput; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisResultsLoaded; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; @@ -16,9 +20,11 @@ import ghidra.program.model.data.Undefined; import ghidra.program.model.symbol.SourceType; import ghidra.util.task.TaskMonitor; +import org.jetbrains.annotations.Nullable; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; import java.util.List; import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; @@ -45,41 +51,64 @@ public List getFunctionInfo(AnalysisID analysisID) { } @Override - public Optional getFunctionDataTypes(AnalysisID analysisID, FunctionID functionID) { - var ds = new FunctionDataTypeStatus( - true, - Optional.of( - new FunctionDataTypeMessage( - new FunctionArtifact( - 0x4000L, - 0x100, - new FunctionHeader( - null, - "portal_name_demangled", - 0x4000L, - "int", - new FunctionArgument[]{ - new FunctionArgument(0, 8, - null, - "named_param_1", - "char *" - ), - } - - ), - new StackVariable[]{} - ), - new FunctionDependencies( - new Typedef[]{}, - new Struct[]{} - ) - ) - ), - "completed", - null, - functionID - ); - return Optional.of(ds); + public FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID, @Nullable List ids) { + + try { + var list = FunctionDataTypesList.fromJson( + """ + { + "total_count": 1, + "total_data_types_count": 1, + "items": [ + { + "completed": true, + "status": "completed", + "data_types": { + "func_types": { + "addr": 1052960, + "size": 22, + "header": { + "name": "portal_name_demangled", + "addr": 1052960, + "type": "int", + "args": { + "0x0": { + "offset": 0, + "name": "ctx", + "type": "ossl_typ.h::EVP_PKEY_CTX *", + "size": 1 + } + } + }, + "name": "portal_name_demangled", + "type": "int", + "artifact_type": "Function" + }, + "func_deps": [ + { + "name": "evp_pkey_ctx_st", + "size": 0, + "members": {}, + "artifact_type": "Struct" + }, + { + "name": "EVP_PKEY_CTX", + "type": "ossl_typ.h::evp_pkey_ctx_st", + "artifact_type": "Typedef" + } + ] + }, + "function_id": 1 + } + ] + } + """ + ); + return list; + + } catch (IOException e) { + throw new RuntimeException(e); + } } @Override @@ -166,7 +195,7 @@ public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { assertEquals("portal_name_demangled", exampleFunc.getName()); var signature = exampleFunc.getSignature(true); - assertEquals("int portal_name_demangled(char * named_param_1)", signature.getPrototypeString()); + assertEquals("int portal_name_demangled(EVP_PKEY_CTX * ctx)", signature.getPrototypeString()); // For unclear reasons the signature source is not set by the command in Ghidra 11.2.x // So we only test this for Ghidra 11.3 and above ApplicationVersion version = new ApplicationVersion(Application.getApplicationVersion()); diff --git a/src/test/java/ai/reveng/UnstripTest.java b/src/test/java/ai/reveng/UnstripTest.java index d5c946e..b48f8ec 100644 --- a/src/test/java/ai/reveng/UnstripTest.java +++ b/src/test/java/ai/reveng/UnstripTest.java @@ -2,6 +2,7 @@ import ai.reveng.invoker.ApiException; import ai.reveng.model.AutoUnstripResponse; +import ai.reveng.model.FunctionDataTypesList; import ai.reveng.model.MatchedFunctionSuggestion; import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; @@ -11,6 +12,7 @@ import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; import ghidra.util.task.TaskMonitor; +import org.jetbrains.annotations.Nullable; import org.junit.Test; import java.util.Iterator; @@ -27,12 +29,12 @@ public void testFinishedUnstrip() throws Exception { var tool = env.getTool(); var service = addMockedService(tool, new UnimplementedAPI() { - + boolean autoUnstripCalled = false; @Override public TypedAutoUnstripResponse autoUnstrip(AnalysisID analysisID) { var r = new AutoUnstripResponse() .progress(100) - .status("STATUS") + .status("COMPLETED") .applied(false) .totalTime(0) .matches(List.of( @@ -43,7 +45,7 @@ public TypedAutoUnstripResponse autoUnstrip(AnalysisID analysisID) { .suggestedDemangledName("unstripped_function_name_demangled") ) ); - + autoUnstripCalled = true; return new TypedAutoUnstripResponse(r); } @@ -59,14 +61,19 @@ public AnalysisStatus status(AnalysisID analysisID) { @Override public List getFunctionInfo(AnalysisID analysisID) { - return List.of(new FunctionInfo(new FunctionID(1), "default_function_info_name", "default_function_info_name_mangled",0x1000L, 10)); + if (!autoUnstripCalled) { + return List.of(new FunctionInfo(new FunctionID(1), "FUN_0x1000", "FUN_0x1000",0x1000L, 10)); + } + else { + return List.of(new FunctionInfo(new FunctionID(1), "unstripped_function_name_demangled", "unstripped_function_name_mangled",0x1000L, 10)); + } } }); addPlugin(tool, BinarySimilarityPlugin.class); var builder = new ProgramBuilder("mock", ProgramBuilder._8051, this); var func = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); - + builder.createMemory("test", "01000", 0x100); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); env.showTool(programWithID.program()); @@ -137,8 +144,18 @@ public AnalysisStatus status(AnalysisID analysisID) { @Override public List getFunctionInfo(AnalysisID analysisID) { - // the function info will return a name, but it will _not_ be the unstripped name - return List.of(new FunctionInfo(new FunctionID(1), "default_function_info_name", "default_function_info_name_mangled",0x1000L, 10)); + // the function info will return a name, but it will _not_ be the unstripped name by default + if (responses.hasNext()) { + return List.of(new FunctionInfo(new FunctionID(1), "FUN_0x1000", "FUN_0x1000",0x1000L, 10)); + } else { + return List.of(new FunctionInfo(new FunctionID(1), "unstripped_function_name_demangled", "unstripped_function_name_mangled",0x1000L, 10)); + } + } + + @Override + public FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID) { + var list = new FunctionDataTypesList(); + return list; } }); @@ -146,7 +163,7 @@ public List getFunctionInfo(AnalysisID analysisID) { var builder = new ProgramBuilder("mock", ProgramBuilder._8051, this); // We provide no function name, so ghidra will assign the default "FUN_1000" name var func = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); - + builder.createMemory("test", "01000", 0x100); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); env.showTool(programWithID.program());