diff --git a/build.gradle b/build.gradle index ffd2f714..5f92d5db 100644 --- a/build.gradle +++ b/build.gradle @@ -65,7 +65,7 @@ dependencies { implementation 'org.json:json:20250107' implementation "com.google.guava:guava:33.2.0-jre" implementation group: 'com.fifesoft', name: 'rsyntaxtextarea', version: '3.5.2' - implementation "ai.reveng:sdk:2.37.4" + implementation "ai.reveng:sdk:2.52.1" testImplementation('junit:junit:4.13.1') testRuntimeOnly("org.junit.vintage:junit-vintage-engine:5.8.2") diff --git a/ghidra_scripts/RevEngExamplePostScript.java b/ghidra_scripts/RevEngExamplePostScript.java index 600db865..dbdfe7e4 100644 --- a/ghidra_scripts/RevEngExamplePostScript.java +++ b/ghidra_scripts/RevEngExamplePostScript.java @@ -1,6 +1,7 @@ import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.types.ApiInfo; +import ghidra.app.plugin.core.analysis.AutoAnalysisManager; import ghidra.app.script.GhidraScript; public class RevEngExamplePostScript extends GhidraScript { @@ -12,9 +13,6 @@ protected void run() throws Exception { ghidraRevengService.upload(currentProgram); AnalysisOptionsBuilder options = AnalysisOptionsBuilder.forProgram(currentProgram); - var binID = ghidraRevengService.analyse(currentProgram, options, monitor); - - // Wait for analysis to finish - ghidraRevengService.waitForFinishedAnalysis(monitor, binID, null, null); + var analyzedProgram = ghidraRevengService.analyse(currentProgram, options, monitor); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ApplyMatchCmd.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ApplyMatchCmd.java index 23468cc8..ce461736 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ApplyMatchCmd.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ApplyMatchCmd.java @@ -5,14 +5,10 @@ import ghidra.app.cmd.function.ApplyFunctionSignatureCmd; import ghidra.app.cmd.label.RenameLabelCmd; import ghidra.framework.cmd.Command; -import ghidra.framework.model.DomainObject; -import ghidra.program.model.data.DataTypeDependencyException; -import ghidra.program.model.data.FunctionDefinitionDataType; import ghidra.program.model.listing.CircularDependencyException; import ghidra.program.model.listing.Program; import ghidra.program.model.symbol.Namespace; import ghidra.program.model.symbol.SourceType; -import ghidra.util.Msg; import ghidra.util.exception.DuplicateNameException; import ghidra.util.exception.InvalidInputException; import org.jetbrains.annotations.NotNull; @@ -21,31 +17,61 @@ import static ai.reveng.toolkit.ghidra.plugins.BinarySimilarityPlugin.REVENG_AI_NAMESPACE; -// We can't use Command because that breaks compatibility with Ghidra 11.0 -public class ApplyMatchCmd implements Command { +/// The central command to apply a function match to a {@link Program} +/// It centralizes product design decisions about how to apply a match, like moving it to a namespace, +/// renaming it, applying the signature, etc. +/// It will apply the function signature if available, otherwise it will just rename the function +/// There are various considerations when ap +public class ApplyMatchCmd implements Command { - private final Program program; + private final GhidraRevengService.AnalysedProgram analyzedProgram; private final GhidraFunctionMatchWithSignature match; @Nullable private final GhidraRevengService service; + private final Boolean includeBinaryNameInNameSpace; public ApplyMatchCmd( @Nullable GhidraRevengService service, - @NotNull GhidraFunctionMatchWithSignature match) { + @NotNull GhidraRevengService.AnalysedProgram program, + @NotNull GhidraFunctionMatchWithSignature match, + Boolean includeBinaryNameInNameSpace + + ) { super(); - this.program = match.function().getProgram(); + this.analyzedProgram = program; this.match = match; this.service = service; + this.includeBinaryNameInNameSpace = includeBinaryNameInNameSpace; + } + + private boolean shouldApplyMatch() { + var func = match.function(); + return func != null && + // Do not override user-defined function names + func.getSymbol().getSource() != SourceType.USER_DEFINED && + // Exclude thunks and external functions + !func.isThunk() && + !func.isExternal() && + // Only accept valid names (no spaces) + !match.functionMatch().nearest_neighbor_mangled_function_name().contains(" ") && + !match.functionMatch().nearest_neighbor_function_name().contains(" ") + // Only rename if the function ID is known (boundaries matched) + && analyzedProgram.getIDForFunction(func).map(id -> id.functionID() != match.functionMatch().origin_function_id()).orElse(false); } + @Override - public boolean applyTo(DomainObject obj) { + public boolean applyTo(Program obj) { // Check that this is the same program - if (obj != this.program) { + if (obj != this.analyzedProgram.program()) { throw new IllegalArgumentException("This command can only be applied to the same program as the one provided in the constructor"); } - var libraryNamespace = getLibraryNameSpaceForName(match.functionMatch().nearest_neighbor_binary_name()); + if (!shouldApplyMatch()) { + return false; + } + + var nameSpace = includeBinaryNameInNameSpace ? getLibraryNameSpaceForName(match.functionMatch().nearest_neighbor_binary_name()): getRevEngAINameSpace(); var function = match.function(); try { - function.setParentNamespace(libraryNamespace); + function.setParentNamespace(nameSpace); } catch (DuplicateNameException e) { throw new RuntimeException(e); } catch (InvalidInputException e) { @@ -54,43 +80,36 @@ public boolean applyTo(DomainObject obj) { throw new RuntimeException(e); } - FunctionDefinitionDataType signature = null; - if (match.signature().isPresent()) { - try { - signature = GhidraRevengService.getFunctionSignature(match.signature().get()); - } catch (DataTypeDependencyException e) { - Msg.showError(this, null,"Failed to create function signature", - "Failed to create signature for match function with type %s" - .formatted(match.signature().get().func_types().getSignature()), - e); - } - } + this.analyzedProgram.setMangledNameForFunction(function, match.functionMatch().nearest_neighbor_mangled_function_name()); + var signature = match.signature(); if (signature != null) { var cmd = new ApplyFunctionSignatureCmd(function.getEntryPoint(), signature, SourceType.USER_DEFINED); - cmd.applyTo(program); + cmd.applyTo(analyzedProgram.program()); } else { var renameCmd = new RenameLabelCmd(match.function().getSymbol(), match.functionMatch().name(), SourceType.USER_DEFINED); - renameCmd.applyTo(program); + renameCmd.applyTo(analyzedProgram.program()); } // If we have a service then push the name. If not then it was explicitly not provided, i.e. the caller // is responsible for pushing the names in batch if (service != null) { - service.getApi().renameFunction(match.functionMatch().origin_function_id(), match.functionMatch().name()); + service.getApi().renameFunction(match.functionMatch().origin_function_id(), match.functionMatch().nearest_neighbor_function_name()); } - return false; + return true; } public void applyWithTransaction() { + var program = this.analyzedProgram.program(); var tID = program.startTransaction("RevEng.AI: Apply Match"); var status = applyTo(program); program.endTransaction(tID, status); } private Namespace getRevEngAINameSpace() { + var program = this.analyzedProgram.program(); Namespace revengMatchNamespace = null; try { revengMatchNamespace = program.getSymbolTable().getOrCreateNameSpace( @@ -105,6 +124,7 @@ private Namespace getRevEngAINameSpace() { } private Namespace getLibraryNameSpaceForName(String name) { + var program = this.analyzedProgram.program(); Namespace libraryNamespace = null; try { libraryNamespace = program.getSymbolTable().getOrCreateNameSpace( diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ComputeTypeInfoTask.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ComputeTypeInfoTask.java index d8f4acfc..63cfb80c 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ComputeTypeInfoTask.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/cmds/ComputeTypeInfoTask.java @@ -1,6 +1,7 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.cmds; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ghidra.util.exception.CancelledException; import ghidra.util.task.Task; @@ -10,9 +11,7 @@ import java.time.Duration; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; import static java.util.stream.Collectors.groupingBy; @@ -23,12 +22,12 @@ * */ public class ComputeTypeInfoTask extends Task { - private final List functions; + private final List functions; private final GhidraRevengService service; private final DataTypeAvailableCallback callback; public ComputeTypeInfoTask(GhidraRevengService service, - List functions, + List functions, @Nullable DataTypeAvailableCallback callback) { super("Computing Type Info", true, true, false); this.service = service; @@ -49,7 +48,7 @@ public void run(TaskMonitor monitor) throws CancelledException { service.getApi().generateFunctionDataTypes(analysisID, functions.stream().map(FunctionDetails::functionId).toList()); }); - Set missing = new HashSet<>(functions); + Set missing = new HashSet<>(functions); while (!missing.isEmpty()) { try { @@ -76,6 +75,6 @@ public void run(TaskMonitor monitor) throws CancelledException { } public interface DataTypeAvailableCallback { - void dataTypeAvailable(FunctionID functionID, FunctionDataTypeStatus dataTypeStatus); + void dataTypeAvailable(TypedApiInterface.FunctionID functionID, FunctionDataTypeStatus dataTypeStatus); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java index 07f4519a..d28ba7e2 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/aidecompiler/AIDecompilationdWindow.java @@ -2,7 +2,7 @@ import ai.reveng.invoker.ApiException; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ai.reveng.toolkit.ghidra.core.services.api.types.AIDecompilationStatus; @@ -53,10 +53,15 @@ public AIDecompilationdWindow(PluginTool tool, String owner) { public void actionPerformed(ActionContext context) { if (function != null) { var service = tool.getService(GhidraRevengService.class); - var fID = service.getFunctionIDFor(function); + var analyzedProgram = service.getAnalysedProgram(function.getProgram()); + if (analyzedProgram.isEmpty()) { + Msg.error(this, "Failed to send positive feedback: Program is not known to RevEng.AI"); + return; + } + var fID = analyzedProgram.get().getIDForFunction(function); fID.ifPresent(id -> { try { - service.getApi().aiDecompRating(id, "POSITIVE", ""); + service.getApi().aiDecompRating(id.functionID(), "POSITIVE", ""); } catch (ApiException e) { // Fail silently because this is not a critical feature Msg.error(this, "Failed to send positive feedback for function %s: %s".formatted(function.getName(), e.getMessage())); @@ -89,10 +94,15 @@ public void actionPerformed(ActionContext context) { if (!dialog.isCanceled()) { if (function != null) { var service = tool.getService(GhidraRevengService.class); - var fID = service.getFunctionIDFor(function); + var programWithID = service.getAnalysedProgram(function.getProgram()); + if (programWithID.isEmpty()) { + Msg.error(this, "Failed to send negative feedback: Program is not known to RevEng.AI"); + return; + } + var fID = programWithID.get().getIDForFunction(function); fID.ifPresent(id -> { try { - service.getApi().aiDecompRating(id, "NEGATIVE", dialog.getValue()); + service.getApi().aiDecompRating(id.functionID(), "NEGATIVE", dialog.getValue()); } catch (ApiException e) { // Fail silently because this is not a critical feature Msg.error(this, "Failed to send negative feedback for function %s: %s".formatted(function.getName(), e.getMessage())); @@ -164,11 +174,11 @@ private void clear() { descriptionArea.setText(""); } - public void refresh(Function function) { + public void refresh(GhidraRevengService.FunctionWithID function) { // Check if we know this function already - var cachedStatus = cache.get(function); + var cachedStatus = cache.get(function.function()); if (cachedStatus != null) { - setDisplayedValuesBasedOnStatus(function, cachedStatus); + setDisplayedValuesBasedOnStatus(function.function(), cachedStatus); } else { // TODO: Allow toggling auto decomp mode via local toggle action, for now do it always @@ -184,6 +194,13 @@ public void refresh(Function function) { } public void locationChanged(ProgramLocation loc) { + var service = tool.getService(GhidraRevengService.class); + var analyzedProgram = service.getAnalysedProgram(loc.getProgram()); + if (analyzedProgram.isEmpty()) { + clear(); + return; + } + var functionMgr = loc.getProgram().getFunctionManager(); var newFuncLocation = functionMgr.getFunctionContaining(loc.getAddress()); @@ -193,9 +210,8 @@ public void locationChanged(ProgramLocation loc) { } function = newFuncLocation; - if (function != null && !function.isExternal() && !function.isThunk()) { - refresh(function); - } + var functionWithID = analyzedProgram.get().getIDForFunction(function); + functionWithID.ifPresent(this::refresh); } @@ -225,18 +241,17 @@ private boolean hasPendingDecompilations() { class AIDecompTask extends Task { private final GhidraRevengService service; - private final Function function; + private final GhidraRevengService.FunctionWithID functionWithID; - public AIDecompTask(PluginTool tool, Function function) { + public AIDecompTask(PluginTool tool, GhidraRevengService.FunctionWithID functionWithID) { super("AI Decomp task", true, false, false); service = tool.getService(GhidraRevengService.class); - this.function = function; + this.functionWithID = functionWithID; } @Override public void run(TaskMonitor monitor) throws CancelledException { - var fID = service.getFunctionIDFor(function) - .orElseThrow(() -> new RuntimeException("Function has no associated FunctionID")); + var fID = functionWithID.functionID(); // Check if there is an existing process already, because the trigger API will fail with 400 if there is if (service.getApi().pollAIDecompileStatus(fID).status().equals("uninitialised")) { // Trigger the decompilation @@ -247,7 +262,7 @@ public void run(TaskMonitor monitor) throws CancelledException { } - private void waitForDecomp(FunctionID id, TaskMonitor monitor) throws CancelledException { + private void waitForDecomp(TypedApiInterface.FunctionID id, TaskMonitor monitor) throws CancelledException { var logger = tool.getService(ReaiLoggingService.class); var api = service.getApi(); AIDecompilationStatus lastDecompStatus = null; @@ -256,9 +271,9 @@ private void waitForDecomp(FunctionID id, TaskMonitor monitor) throws CancelledE if (lastDecompStatus == null || !Objects.equals(newStatus.status(), lastDecompStatus.status())) { lastDecompStatus = newStatus; - newStatusForFunction(function, newStatus); + newStatusForFunction(functionWithID.function(), newStatus); } - monitor.setMessage("Waiting for AI Decompilation for %s ... Current status: %s".formatted(function.getName(), lastDecompStatus.status())); + monitor.setMessage("Waiting for AI Decompilation for %s ... Current status: %s".formatted(functionWithID.function().getName(), lastDecompStatus.status())); monitor.checkCancelled(); switch (newStatus.status()) { case "pending": diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java index ea9ac439..2c04d0f5 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/autounstrip/AutoUnstripDialog.java @@ -1,35 +1,21 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.ui.autounstrip; +import ai.reveng.model.AutoUnstripResponse; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.dialog.RevEngDialogComponentProvider; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; -import ai.reveng.toolkit.ghidra.core.services.api.types.AutoUnstripResponse; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionID; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.framework.plugintool.PluginTool; -import ghidra.program.model.address.Address; -import ghidra.program.model.listing.Function; -import ghidra.program.model.listing.Program; -import ghidra.program.model.symbol.SourceType; -import ghidra.util.exception.DuplicateNameException; -import ghidra.util.exception.InvalidInputException; import ghidra.util.task.TaskMonitorComponent; import javax.swing.*; import javax.swing.table.DefaultTableModel; import java.awt.*; -import java.util.ArrayList; -import java.util.List; import java.util.Objects; -import static ai.reveng.toolkit.ghidra.plugins.BinarySimilarityPlugin.REVENG_AI_NAMESPACE; - public class AutoUnstripDialog extends RevEngDialogComponentProvider { - private final AnalysisID analysisID; + private final GhidraRevengService.AnalysedProgram analysedProgram; private final GhidraRevengService revengService; - private final Program program; - private AutoUnstripResponse autoUnstripResponse; // UI components private JLabel statusLabel; @@ -40,23 +26,16 @@ public class AutoUnstripDialog extends RevEngDialogComponentProvider { private final TaskMonitorComponent taskMonitorComponent; private JTable resultsTable; private JScrollPane resultsScrollPane; - private final List renameResults; // Polling configuration private static final int POLL_INTERVAL_MS = 2000; // Poll every 2 seconds - // Inner class to hold rename results - private record RenameResult(String virtualAddress, String originalName, String newName) { - } - - public AutoUnstripDialog(PluginTool tool, ProgramWithBinaryID analysisID) { + public AutoUnstripDialog(PluginTool tool, GhidraRevengService.AnalysedProgram analysedProgram) { super(ReaiPluginPackage.WINDOW_PREFIX + "Auto Unstrip", true); - this.analysisID = analysisID.analysisID(); - this.program = analysisID.program(); + this.analysedProgram = analysedProgram; this.revengService = tool.getService(GhidraRevengService.class); this.taskMonitorComponent = new TaskMonitorComponent(false, true); - this.renameResults = new ArrayList<>(); // Initialize UI addDismissButton(); @@ -86,14 +65,18 @@ private void startAutoUnstrip() { private void pollAutoUnstripStatus() { SwingUtilities.invokeLater(() -> { try { - autoUnstripResponse = revengService.getApi().autoUnstrip(analysisID); - updateUI(); + var autoUnstripResponse = revengService.autoUnstrip(analysedProgram).autoUnstripResponse(); + updateUI(autoUnstripResponse); // Check if we're done - if (autoUnstripResponse.progress() >= 100 || Objects.equals(autoUnstripResponse.status(), "COMPLETED")) { + if (autoUnstripResponse.getProgress() >= 100 || Objects.equals(autoUnstripResponse.getStatus(), "COMPLETED")) { stopPolling(); + // Pull function information (names and types) from the server, instead of dealing with matches + // in the auto unstrip response + var changes = revengService.pullFunctionInfoFromAnalysis(analysedProgram, taskMonitorComponent); taskMonitorComponent.setVisible(false); - importFunctionNames(autoUnstripResponse); + + SwingUtilities.invokeLater(() -> updateResultsTable(changes)); } } catch (Exception e) { handleError("Failed to poll auto unstrip status: " + e.getMessage()); @@ -102,96 +85,36 @@ private void pollAutoUnstripStatus() { }); } - private void importFunctionNames(AutoUnstripResponse autoUnstripResponse) { - var functionMgr = program.getFunctionManager(); - - // Retrieve the mangled names map once outside the transaction - var mangledNameMapOpt = revengService.getFunctionMangledNamesMap(program); - - // Retrieve the function ID map once outside the transaction - var functionMap = revengService.getFunctionMap(program); - - program.withTransaction("Apply Auto-Unstrip Function Names", () -> { - try { - var revengMatchNamespace = program.getSymbolTable().getOrCreateNameSpace( - program.getGlobalNamespace(), - REVENG_AI_NAMESPACE, - SourceType.ANALYSIS - ); - - autoUnstripResponse.matches().forEach(match -> { - Address addr = program.getAddressFactory().getDefaultAddressSpace().getAddress(match.function_vaddr()); - Function func = functionMgr.getFunctionAt(addr); - - var revEngMangledName = match.suggested_name(); - var revEngDemangledName = match.suggested_demangled_name(); - var functionID = functionMap.get(new FunctionID(match.function_id().value())); - - if ( - func != null && - // Do not override user-defined function names - func.getSymbol().getSource() != SourceType.USER_DEFINED && - // Exclude thunks and external functions - !func.isThunk() && - !func.isExternal() && - // Only accept valid names (no spaces) - !revEngMangledName.contains(" ") && - !revEngDemangledName.contains(" ") - // Only rename if the function ID is known (boundaries matched) - && functionID != null - ) { - try { - // Capture original name before renaming - String originalName = func.getName(); - - func.setName(revEngDemangledName, SourceType.ANALYSIS); - func.setParentNamespace(revengMatchNamespace); - - // Update the mangled name map with the RevEng.AI mangled name - mangledNameMapOpt.ifPresent(mangledNameMap -> { - try { - mangledNameMap.add(func.getEntryPoint(), revEngMangledName); - } catch (Exception e) { - handleError("Failed to update mangled name map for function at " + addr + ": " + e.getMessage()); - } - }); - - // Add to rename results - renameResults.add(new RenameResult(String.format("%08x", match.function_vaddr()), originalName, revEngDemangledName)); - } catch (Exception e) { - handleError("Failed to rename function at " + addr + ": " + e.getMessage()); - } - } - }); - } catch (DuplicateNameException | InvalidInputException e) { - throw new RuntimeException(e); - } - } - ); - - // Show results table after import is complete - SwingUtilities.invokeLater(this::updateResultsTable); - } - - private void updateUI() { + private void updateUI(AutoUnstripResponse autoUnstripResponse) { if (autoUnstripResponse == null) return; // Update progress bar - taskMonitorComponent.setProgress(autoUnstripResponse.progress()); - taskMonitorComponent.setMessage(autoUnstripResponse.progress() + "%"); + taskMonitorComponent.setProgress(autoUnstripResponse.getProgress()); + taskMonitorComponent.setMessage(autoUnstripResponse.getProgress() + "%"); // Update status - statusLabel.setText("Status: " + getFriendlyStatusMessage(autoUnstripResponse.status())); + statusLabel.setText("Status: " + getFriendlyStatusMessage(autoUnstripResponse.getStatus())); // Handle error message - dynamically add/remove error panel - if (autoUnstripResponse.error_message() != null && !autoUnstripResponse.error_message().isEmpty()) { - showError(autoUnstripResponse.error_message()); + if (autoUnstripResponse.getErrorMessage() != null && !autoUnstripResponse.getErrorMessage().isEmpty()) { + showError(autoUnstripResponse.getErrorMessage()); } else { hideError(); } + final var map = analysedProgram.getFunctionMap(); + var currentRenameResults = autoUnstripResponse.getMatches().stream().map(match -> { + // This functionID is for the local/origin function and not the matched/neighbour function! + var functionId = new TypedApiInterface.FunctionID(match.getFunctionId()); + var function = map.get(functionId); + return new GhidraRevengService.RenameResult( + function, + function.getName(), + match.getSuggestedName() + ); + }).toList(); // Update results table - updateResultsTable(); + updateResultsTable(currentRenameResults); } /** @@ -212,7 +135,7 @@ private String getFriendlyStatusMessage(String apiStatus) { }; } - private void updateResultsTable() { + private void updateResultsTable(java.util.List renameResults) { DefaultTableModel model = new DefaultTableModel(new Object[]{"Virtual Address", "Original Name", "New Name"}, 0) { @Override public boolean isCellEditable(int row, int column) { @@ -220,8 +143,8 @@ public boolean isCellEditable(int row, int column) { return false; } }; - for (RenameResult result : renameResults) { - model.addRow(new Object[]{result.virtualAddress, result.originalName, result.newName}); + for (GhidraRevengService.RenameResult result : renameResults) { + model.addRow(new Object[]{result.virtualAddress(), result.originalName(), result.newName()}); } resultsTable.setModel(model); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/collectiondialog/DataSetControlPanelComponent.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/collectiondialog/DataSetControlPanelComponent.java index 9d9ea4c8..44cd88bc 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/collectiondialog/DataSetControlPanelComponent.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/collectiondialog/DataSetControlPanelComponent.java @@ -1,8 +1,8 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.ui.collectiondialog; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; import docking.action.builder.ActionBuilder; import generic.theme.GIcon; import ghidra.framework.plugintool.ComponentProviderAdapter; @@ -22,7 +22,7 @@ *

*

*

- * Currently, collections are "flat", and effectively a set of {@link ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID}
+ * Currently, collections are "flat", and effectively a set of {@link TypedApiInterface.AnalysisID}
* In the future they may support hierarchies/nesting
*

* This dialog allows sharing the collection choice logic between all components that need to select a collection, @@ -154,7 +154,7 @@ public void run(TaskMonitor monitor) throws CancelledException { // Get all rows that are already selected to be included var selectedBinaries = binaryTableModel.getModelData().stream().filter(BinaryRowObject::include).toList(); // - Set selectedSet = selectedBinaries.stream().map(row -> row.analysisResult().analysisID() ).collect(Collectors.toSet()); + Set selectedSet = selectedBinaries.stream().map(row -> row.analysisResult().analysisID() ).collect(Collectors.toSet()); binaryTableModel.clearData(); // var searchTerm = analysisSearchTextBox.getText(); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java index 21a2d222..43383aae 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/AbstractFunctionMatchingDialog.java @@ -1,13 +1,17 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.ui.functionmatching; import ai.reveng.model.*; +import ai.reveng.toolkit.ghidra.binarysimilarity.cmds.ApplyMatchCmd; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionID; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.dialog.RevEngDialogComponentProvider; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.components.CollectionSelectionPanel; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.components.BinarySelectionPanel; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.components.SelectableItem; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatch; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatchWithSignature; +import com.google.common.collect.BiMap; import ghidra.program.model.listing.Function; import ghidra.program.model.symbol.SourceType; import ghidra.util.exception.DuplicateNameException; @@ -28,7 +32,7 @@ public abstract class AbstractFunctionMatchingDialog extends RevEngDialogComponentProvider { protected final GhidraRevengService revengService; - protected final ProgramWithBinaryID programWithBinaryID; + protected final GhidraRevengService.AnalysedProgram analyzedProgram; // UI Components protected JPanel contentPanel; @@ -50,43 +54,44 @@ public abstract class AbstractFunctionMatchingDialog extends RevEngDialogCompone // Data protected Basic analysisBasicInfo; - protected FunctionMatchingBatchResponse functionMatchingResponse; - protected final List functionMatchResults; - protected final List filteredFunctionMatchResults; + protected FunctionMatchingResponse functionMatchingResponse; + protected final List functionMatchResults; + protected final List filteredFunctionMatchResults; // Polling configuration protected static final int POLL_INTERVAL_MS = 2000; // Poll every 2 seconds - // Inner class to hold function match results - protected record FunctionMatchResult( - String virtualAddress, - String functionName, - String bestMatchName, - String bestMatchMangledName, - String similarity, - String confidence, - String matchedHash, - String binary, - Long matcherFunctionId - ) { - // Constructor for function-level dialog (without virtual address and function name) - public FunctionMatchResult(String bestMatchName, String bestMatchMangledName, String similarity, - String confidence, String matchedHash, String binary, Long matcherFunctionId) { - this("", "", bestMatchName, bestMatchMangledName, similarity, confidence, matchedHash, binary, matcherFunctionId); - } - } +// /// Inner class to hold function match results +// /// @deprecated {@link ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch} +// public record FunctionMatchResult( +// String virtualAddress, +// String functionName, +// String bestMatchName, +// String bestMatchMangledName, +// String similarity, +// String confidence, +// String matchedHash, +// String binary, +// Long matcherFunctionId +// ) { +// // Constructor for function-level dialog (without virtual address and function name) +// public FunctionMatchResult(String bestMatchName, String bestMatchMangledName, String similarity, +// String confidence, String matchedHash, String binary, Long matcherFunctionId) { +// this("", "", bestMatchName, bestMatchMangledName, similarity, confidence, matchedHash, binary, matcherFunctionId); +// } +// } protected AbstractFunctionMatchingDialog(String title, Boolean isModal, GhidraRevengService revengService, - ProgramWithBinaryID programWithBinaryID) { + GhidraRevengService.AnalysedProgram analyzedProgram) { super(title, isModal); this.revengService = revengService; - this.programWithBinaryID = programWithBinaryID; + this.analyzedProgram = analyzedProgram; this.taskMonitorComponent = new TaskMonitorComponent(false, true); this.functionMatchResults = new ArrayList<>(); this.filteredFunctionMatchResults = new ArrayList<>(); try { - this.analysisBasicInfo = revengService.getBasicDetailsForAnalysis(programWithBinaryID.analysisID()); + this.analysisBasicInfo = revengService.getBasicDetailsForAnalysis(analyzedProgram.analysisID()); } catch (Exception e) { JOptionPane.showMessageDialog(null, "Failed to fetch analysis details: " + e.getMessage(), @@ -123,28 +128,38 @@ protected void startFunctionMatching() { protected abstract void pollFunctionMatchingStatus(); - protected void processFunctionMatchingResults(FunctionMatchingBatchResponse response) { + protected void processFunctionMatchingResults(FunctionMatchingResponse response) { functionMatchResults.clear(); - - var functionMap = revengService.getFunctionMap(programWithBinaryID.program()); + List matches = new ArrayList<>(); + final BiMap functionMap = analyzedProgram.getFunctionMap(); response.getMatches().forEach(matchResult -> { + // Retrieve the local function name + var funcId = new TypedApiInterface.FunctionID(matchResult.getFunctionId()); + Function localFunction = functionMap.get(funcId); + if (localFunction == null) { + // If we can't find the local function, skip this match (boundaries do not match the remote ones) + return; + } // Process each matched function in this result matchResult.getMatchedFunctions().forEach(match -> { - // Retrieve the local function name - Function localFunction = functionMap.get(new FunctionID(matchResult.getFunctionId())); - - if (localFunction == null) { - // If we can't find the local function, skip this match (boundaries do not match the remote ones) - return; - } - // Create function match result using the abstract method - FunctionMatchResult result = createFunctionMatchResult(localFunction, match, matchResult.getFunctionId()); - functionMatchResults.add(result); + FunctionMatch result = FunctionMatch.fromMatchedFunctionAPIType(match, funcId); + GhidraFunctionMatch ghidraResult = new GhidraFunctionMatch(localFunction, result); +// FunctionMatch result = createFunctionMatchResult(localFunction, match, funcId); + matches.add(ghidraResult); }); }); + // Get the type info for all matches + var ghidraResultsWithSignatures = revengService.getSignatures(matches); + + // For all matches we got, create a GhidraFunctionMatchWithSignature object (signature can be null!) + for (GhidraFunctionMatch match : matches) { + var sig = ghidraResultsWithSignatures.get(match); + functionMatchResults.add(new GhidraFunctionMatchWithSignature(match.function(), match.functionMatch(), sig)); + } + // Apply any existing function filter after getting results onFunctionFilterChanged(); @@ -152,7 +167,6 @@ protected void processFunctionMatchingResults(FunctionMatchingBatchResponse resp SwingUtilities.invokeLater(this::updateResultsTable); } - protected abstract FunctionMatchResult createFunctionMatchResult(Function localFunction, MatchedFunction match, Long matcherFunctionId); protected void updateUI() { if (functionMatchingResponse == null) return; @@ -182,7 +196,7 @@ protected void updateUI() { protected void updateResultsTable() { // Determine which results to show based on whether we have an active filter String filterText = functionFilterField != null ? functionFilterField.getText().trim() : ""; - List resultsToShow; + List resultsToShow; if (filterText.isEmpty()) { // No filter text, show all results @@ -199,7 +213,7 @@ public boolean isCellEditable(int row, int column) { return false; } }; - for (FunctionMatchResult result : resultsToShow) { + for (GhidraFunctionMatchWithSignature result : resultsToShow) { model.addRow(getTableRowData(result)); } resultsTable.setModel(model); @@ -235,7 +249,7 @@ public boolean isCellEditable(int row, int column) { } protected abstract String[] getTableColumnNames(); - protected abstract Object[] getTableRowData(FunctionMatchResult result); + protected abstract Object[] getTableRowData(GhidraFunctionMatchWithSignature result); protected abstract String getTableTitle(); protected abstract int getTableSelectionMode(); protected abstract void configureTableColumns(); @@ -717,7 +731,7 @@ protected void onFunctionFilterChanged() { updateResultsTable(); } - protected abstract boolean matchesFilter(FunctionMatchResult result); + protected abstract boolean matchesFilter(GhidraFunctionMatchWithSignature result); // Utility methods public int getThreshold() { @@ -777,10 +791,10 @@ protected void onRenameSelectedButtonClicked() { hideError(); } - List resultsToShow = filteredFunctionMatchResults.isEmpty() ? + List resultsToShow = filteredFunctionMatchResults.isEmpty() ? functionMatchResults : filteredFunctionMatchResults; - List selectedMatches = new ArrayList<>(); + List selectedMatches = new ArrayList<>(); for (int row : selectedRows) { if (row < resultsToShow.size()) { selectedMatches.add(resultsToShow.get(row)); @@ -791,76 +805,26 @@ protected void onRenameSelectedButtonClicked() { importFunctionNames(selectedMatches); } - protected void batchRenameFunctions(List functionMatches) { - var matches = functionMatches.stream() - .map(result -> { - var func = new FunctionRenameMap(); - func.setFunctionId(result.matcherFunctionId()); - func.setNewName(result.bestMatchName()); - func.setNewMangledName(result.bestMatchMangledName()); - return func; - }) - .toList(); - - var functionsListRename = new FunctionsListRename(); - functionsListRename.setFunctions(matches); - + protected void batchRenameFunctions(List functionMatches) { try { - revengService.batchRenameFunctions(functionsListRename); + revengService.batchRenamingGhidraMatchesWithSignatures(functionMatches); } catch (Exception e) { showError("Failed to rename functions: " + e.getMessage()); } } - protected void importFunctionNames(List functionMatches) { - var program = programWithBinaryID.program(); - var mangledNameMapOpt = revengService.getFunctionMangledNamesMap(program); - var functionMap = revengService.getFunctionMap(program); + protected void importFunctionNames(List functionMatches) { + var program = analyzedProgram.program(); program.withTransaction("Apply Function Matching Renames", () -> { - try { - var revengMatchNamespace = program.getSymbolTable().getOrCreateNameSpace( - program.getGlobalNamespace(), - REVENG_AI_NAMESPACE, - SourceType.ANALYSIS - ); - - functionMatches.forEach(match -> { - var funcID = new FunctionID(match.matcherFunctionId()); - Function func = functionMap.get(funcID); - - var revEngMangledName = match.bestMatchMangledName(); - var revEngDemangledName = match.bestMatchName(); - - if (func != null && - func.getSymbol().getSource() != SourceType.USER_DEFINED && - !func.isThunk() && - !func.isExternal() && - !revEngMangledName.contains(" ") && - !revEngDemangledName.contains(" ") - ) { - try { - func.setName(revEngDemangledName, SourceType.ANALYSIS); - func.setParentNamespace(revengMatchNamespace); - - mangledNameMapOpt.ifPresent(mangledNameMap -> { - try { - mangledNameMap.add(func.getEntryPoint(), revEngMangledName); - } catch (Exception e) { - handleError("Failed to update mangled name map for function at " + func.getEntryPoint() + ": " + e.getMessage()); - } - }); - - } catch (Exception e) { - handleError("Failed to rename function at " + func.getEntryPoint() + ": " + e.getMessage()); - } + for (GhidraFunctionMatchWithSignature match : functionMatches) { + // We don't pass a service, because we don't need to update individual functions on the portal + // batchRenameFunctions already does that for us + var cmd = new ApplyMatchCmd(null, analyzedProgram, match, false); + cmd.applyTo(program); } - }); - } catch (DuplicateNameException | InvalidInputException e) { - throw new RuntimeException(e); - } - }); - + } + ); SwingUtilities.invokeLater(this::updateResultsTable); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/BinaryLevelFunctionMatchingDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/BinaryLevelFunctionMatchingDialog.java index 057f1897..2db1e487 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/BinaryLevelFunctionMatchingDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/BinaryLevelFunctionMatchingDialog.java @@ -2,7 +2,9 @@ import ai.reveng.model.*; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatch; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatchWithSignature; import ghidra.framework.plugintool.PluginTool; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.program.model.listing.Function; @@ -15,7 +17,7 @@ public class BinaryLevelFunctionMatchingDialog extends AbstractFunctionMatchingDialog { - public BinaryLevelFunctionMatchingDialog(PluginTool tool, ProgramWithBinaryID programWithBinaryID) { + public BinaryLevelFunctionMatchingDialog(PluginTool tool, GhidraRevengService.AnalysedProgram programWithBinaryID) { super(ReaiPluginPackage.WINDOW_PREFIX + "Function Matching", true, tool.getService(GhidraRevengService.class), programWithBinaryID); } @@ -44,7 +46,7 @@ protected void pollFunctionMatchingStatus() { request.setFilters(filters); - functionMatchingResponse = revengService.getFunctionMatchingForAnalysis(programWithBinaryID.analysisID(), request); + functionMatchingResponse = revengService.getFunctionMatchingForAnalysis(analyzedProgram.analysisID(), request); updateUI(); // Check if we hit an error status @@ -72,47 +74,21 @@ protected void pollFunctionMatchingStatus() { }); } - @Override - protected FunctionMatchResult createFunctionMatchResult(Function localFunction, MatchedFunction match, Long matcherFunctionId) { - String virtualAddress = String.format("%08x", match.getFunctionVaddr()); - String functionName = localFunction.getName(); - String bestMatchName = match.getFunctionName(); - String bestMatchMangledName = match.getMangledName(); - String similarity = match.getSimilarity() != null ? - String.format("%.2f%%", match.getSimilarity().doubleValue()) : "N/A"; - String confidence = match.getConfidence() != null ? - String.format("%.2f%%", match.getConfidence().doubleValue()) : "N/A"; - String matchedHash = match.getSha256Hash(); - String binary = match.getBinaryName(); - - return new FunctionMatchResult( - virtualAddress, - functionName, - bestMatchName, - bestMatchMangledName, - similarity, - confidence, - matchedHash, - binary, - matcherFunctionId - ); - } - @Override protected String[] getTableColumnNames() { return new String[]{"Virtual Address", "Function Name", "Matched Function", "Similarity", "Confidence", "Matched Hash", "Matched Binary"}; } @Override - protected Object[] getTableRowData(FunctionMatchResult result) { + protected Object[] getTableRowData(GhidraFunctionMatchWithSignature result) { return new Object[]{ - result.virtualAddress(), - result.functionName(), - result.bestMatchName(), - result.similarity(), - result.confidence(), - result.matchedHash(), - result.binary() + result.function().getEntryPoint().toString(), + result.function().getName(), + result.functionMatch().nearest_neighbor_function_name(), + result.functionMatch().similarity(), + result.functionMatch().confidence(), + result.functionMatch().nearest_neighbor_sha_256_hash().sha256(), + result.functionMatch().nearest_neighbor_binary_name() }; } @@ -176,12 +152,12 @@ protected String getDialogDescription() { } @Override - protected boolean matchesFilter(FunctionMatchResult result) { + protected boolean matchesFilter(GhidraFunctionMatchWithSignature result) { String filterText = functionFilterField.getText().trim().toLowerCase(); - return result.virtualAddress().toLowerCase().contains(filterText) || - result.functionName().toLowerCase().contains(filterText) || - result.bestMatchName().toLowerCase().contains(filterText) || - result.matchedHash().toLowerCase().contains(filterText) || - result.binary().toLowerCase().contains(filterText); + return result.function().getEntryPoint().toString().toLowerCase().contains(filterText) || + result.function().getName().toLowerCase().contains(filterText) || + result.functionMatch().nearest_neighbor_function_name().toLowerCase().contains(filterText) || + result.functionMatch().nearest_neighbor_sha_256_hash().sha256().toLowerCase().contains(filterText) || + result.functionMatch().nearest_neighbor_binary_name().toLowerCase().contains(filterText); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/FunctionLevelFunctionMatchingDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/FunctionLevelFunctionMatchingDialog.java index a962048b..614bb96f 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/FunctionLevelFunctionMatchingDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/functionmatching/FunctionLevelFunctionMatchingDialog.java @@ -2,7 +2,10 @@ import ai.reveng.model.*; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatch; +import ai.reveng.toolkit.ghidra.core.services.api.types.GhidraFunctionMatchWithSignature; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.listing.Function; @@ -16,7 +19,7 @@ public class FunctionLevelFunctionMatchingDialog extends AbstractFunctionMatchingDialog { private final Function function; - public FunctionLevelFunctionMatchingDialog(PluginTool tool, ProgramWithBinaryID programWithBinaryID, Function function) { + public FunctionLevelFunctionMatchingDialog(PluginTool tool, GhidraRevengService.AnalysedProgram programWithBinaryID, Function function) { super(ReaiPluginPackage.WINDOW_PREFIX + "Function Matching", true, tool.getService(GhidraRevengService.class), programWithBinaryID); this.function = function; @@ -31,7 +34,7 @@ protected void pollFunctionMatchingStatus() { request.setResultsPerFunction(25); // TODO: Make configurable? request.setModelId(analysisBasicInfo.getModelId()); - var functionIDOpt = revengService.getFunctionIDFor(function); + var functionIDOpt = analyzedProgram.getIDForFunction(function); if (functionIDOpt.isEmpty()) { handleError("Could not find function ID for the selected function"); stopPolling(); @@ -39,7 +42,7 @@ protected void pollFunctionMatchingStatus() { } var functionIds = new ArrayList(); - functionIds.add(functionIDOpt.get().value()); + functionIds.add(functionIDOpt.get().functionID().value()); request.setFunctionIds(functionIds); @@ -88,41 +91,19 @@ protected void pollFunctionMatchingStatus() { }); } - @Override - protected FunctionMatchResult createFunctionMatchResult(Function localFunction, MatchedFunction match, Long matcherFunctionId) { - String bestMatchName = match.getFunctionName(); - String bestMatchMangledName = match.getMangledName(); - String similarity = match.getSimilarity() != null ? - String.format("%.2f%%", match.getSimilarity().doubleValue()) : "N/A"; - String confidence = match.getConfidence() != null ? - String.format("%.2f%%", match.getConfidence().doubleValue()) : "N/A"; - String matchedHash = match.getSha256Hash(); - String binary = match.getBinaryName(); - - return new FunctionMatchResult( - bestMatchName, - bestMatchMangledName, - similarity, - confidence, - matchedHash, - binary, - matcherFunctionId - ); - } - @Override protected String[] getTableColumnNames() { return new String[]{"Matched Function", "Similarity", "Confidence", "Matched Hash", "Matched Binary"}; } @Override - protected Object[] getTableRowData(FunctionMatchResult result) { + protected Object[] getTableRowData(GhidraFunctionMatchWithSignature result) { return new Object[]{ - result.bestMatchName(), - result.similarity(), - result.confidence(), - result.matchedHash(), - result.binary() + result.functionMatch().nearest_neighbor_function_name(), + result.functionMatch().similarity(), + result.functionMatch().confidence(), + result.functionMatch().nearest_neighbor_sha_256_hash().sha256(), + result.functionMatch().nearest_neighbor_binary_name() }; } @@ -170,11 +151,11 @@ protected String getDialogDescription() { } @Override - protected boolean matchesFilter(FunctionMatchResult result) { + protected boolean matchesFilter(GhidraFunctionMatchWithSignature result) { String filterText = functionFilterField.getText().trim().toLowerCase(); - return result.bestMatchName().toLowerCase().contains(filterText) || - result.matchedHash().toLowerCase().contains(filterText) || - result.binary().toLowerCase().contains(filterText); + return result.functionMatch().nearest_neighbor_function_name().toLowerCase().contains(filterText) || + result.functionMatch().nearest_neighbor_sha_256_hash().sha256().toLowerCase().contains(filterText) || + result.functionMatch().nearest_neighbor_binary_name().toLowerCase().contains(filterText); } @Override diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java index f34fb412..6ed88e1b 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/misc/AnalysisLogComponent.java @@ -4,9 +4,9 @@ import ai.reveng.toolkit.ghidra.core.AnalysisLogConsumer; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ghidra.framework.plugintool.ComponentProviderAdapter; import ghidra.framework.plugintool.PluginTool; +import ghidra.program.model.listing.Program; import ghidra.util.exception.CancelledException; import ghidra.util.task.Task; import ghidra.util.task.TaskBuilder; @@ -24,9 +24,11 @@ public class AnalysisLogComponent extends ComponentProviderAdapter implements An private final JScrollPane scroll; private TaskMonitorComponent taskMonitorComponent; private final JPanel mainPanel; - private final Map trackedPrograms = new ConcurrentHashMap<>(); + private final Map trackedPrograms = new ConcurrentHashMap<>(); + private final Map storedLogs = new ConcurrentHashMap<>(); public static String NAME = ReaiPluginPackage.WINDOW_PREFIX + "Analysis Log"; + private Program activeProgram; public AnalysisLogComponent(PluginTool tool) { super(tool, NAME, ReaiPluginPackage.NAME); @@ -66,6 +68,18 @@ public TaskMonitor getTaskMonitor() { return taskMonitorComponent; } + public void programActivated(Program program) { + // Switch the active log to the given program + this.activeProgram = program; + if (trackedPrograms.containsKey(program)) { + // If we are tracking this program, show the task monitor + taskMonitorComponent.setVisible(true); + } else { + taskMonitorComponent.setVisible(false); + } + setLogs(storedLogs.getOrDefault(program, "No analysis logs available for the active program.")); + } + public void processEvent(RevEngAIAnalysisStatusChangedEvent event) { // We don't need to display the log window when the user selects an existing analysis because it will be an // already completed analysis. @@ -78,11 +92,11 @@ public void processEvent(RevEngAIAnalysisStatusChangedEvent event) { switch (event.getStatus()) { case Complete, Error -> {} default -> { - if (!trackedPrograms.containsKey(event.getProgramWithBinaryID())){ + if (!trackedPrograms.containsKey(event.getProgram())){ // We aren't tracking this program yet, so we start a new task for it var task = new AnalysisMonitoringTask(event.getProgramWithBinaryID(), this); var builder = TaskBuilder.withTask(task); - trackedPrograms.put(event.getProgramWithBinaryID(), task); + trackedPrograms.put(event.getProgram(), task); builder.launchInBackground(getTaskMonitor()); } } @@ -91,19 +105,24 @@ public void processEvent(RevEngAIAnalysisStatusChangedEvent event) { } @Override - public void consumeLogs(String logs) { - this.setLogs(logs); + public void consumeLogs(String logs, GhidraRevengService.ProgramWithID programWithID) { + storedLogs.put(programWithID.program(), logs); + // If this is the currently active program, update the log display + + if (activeProgram == programWithID.program()) { + setLogs(logs); + } } class AnalysisMonitoringTask extends Task { - private final ProgramWithBinaryID program; + private final GhidraRevengService.ProgramWithID program; private final AnalysisLogConsumer logConsumer; - public AnalysisMonitoringTask(ProgramWithBinaryID programWithBinaryID, AnalysisLogConsumer logConsumer) { - super(programWithBinaryID.toString(), true, false, false); - program = programWithBinaryID; + public AnalysisMonitoringTask(GhidraRevengService.ProgramWithID programWithID, AnalysisLogConsumer logConsumer) { + super(programWithID.toString(), true, false, false); + program = programWithID; this.logConsumer = logConsumer; } @@ -111,10 +130,11 @@ public AnalysisMonitoringTask(ProgramWithBinaryID programWithBinaryID, AnalysisL public void run(TaskMonitor monitor) throws CancelledException { // Wait for the analysis var service = tool.getService(GhidraRevengService.class); - // TODO: Get the log consumer for this specific program service.waitForFinishedAnalysis(monitor, program, this.logConsumer, tool); - // Remove ourselfs from the tracked programs when done - trackedPrograms.remove(program); + /// waitForFinishedAnalysis has already sent the completion event to all plugins + /// so this task just needs to clean up the tracked programs map + // Remove ourselves from the tracked programs when done + trackedPrograms.remove(program.program()); taskMonitorComponent.setVisible(false); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysesTableModel.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysesTableModel.java index 18fba582..c3452d9a 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysesTableModel.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysesTableModel.java @@ -1,6 +1,7 @@ package ai.reveng.toolkit.ghidra.binarysimilarity.ui.recentanalyses; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.core.services.function.export.ExportFunctionBoundariesService; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; @@ -16,10 +17,10 @@ import ghidra.util.task.TaskMonitor; public class RecentAnalysesTableModel extends ThreadedTableModelStub { - private final BinaryHash hash; + private final TypedApiInterface.BinaryHash hash; private final Address imageBase; - public RecentAnalysesTableModel(PluginTool tool, BinaryHash hash, Address imageBase) { + public RecentAnalysesTableModel(PluginTool tool, TypedApiInterface.BinaryHash hash, Address imageBase) { super("Recent Analyses Table Model", tool); this.hash = hash; this.imageBase = imageBase; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java index a51b9e99..d034aeb7 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/binarysimilarity/ui/recentanalyses/RecentAnalysisDialog.java @@ -3,9 +3,8 @@ import ai.reveng.toolkit.ghidra.binarysimilarity.ui.dialog.RevEngDialogComponentProvider; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.LegacyAnalysisResult; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryHash; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.listing.Program; @@ -19,7 +18,7 @@ /** - * Shows a dialog with a table of {@link LegacyAnalysisResult} for a given {@link BinaryHash}, + * Shows a dialog with a table of {@link LegacyAnalysisResult} for a given {@link TypedApiInterface.BinaryHash}, * and fires an event when the user picks an analysis */ public class RecentAnalysisDialog extends RevEngDialogComponentProvider { @@ -35,7 +34,7 @@ public RecentAnalysisDialog(PluginTool tool, Program program) { this.program = program; this.ghidraRevengService = tool.getService(GhidraRevengService.class); - var hash = new BinaryHash(program.getExecutableSHA256()); + var hash = new TypedApiInterface.BinaryHash(program.getExecutableSHA256()); recentAnalysesTableModel = new RecentAnalysesTableModel(tool, hash, this.program.getImageBase()); recentAnalysesTable = new GhidraFilterTable<>(recentAnalysesTableModel); @@ -65,7 +64,8 @@ public void mouseClicked(MouseEvent e) { if ("Analysis ID".equals(columnName)) { LegacyAnalysisResult result = recentAnalysesTable.getModel().getRowObject(row); if (result != null) { - ghidraRevengService.openPortalFor(result); + var analysisID = ghidraRevengService.getApi().getAnalysisIDfromBinaryID(result.binary_id()); + ghidraRevengService.openPortalFor(analysisID); } } } @@ -98,7 +98,7 @@ public void mouseClicked(MouseEvent e) { private void pickAnalysis(LegacyAnalysisResult result) { var service = tool.getService(GhidraRevengService.class); var analysisID = service.getApi().getAnalysisIDfromBinaryID(result.binary_id()); - var programWithID = new ProgramWithBinaryID(program, result.binary_id(), analysisID); + var programWithID = new GhidraRevengService.ProgramWithID(program, analysisID); tool.firePluginEvent( new RevEngAIAnalysisStatusChangedEvent( diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/AnalysisLogConsumer.java b/src/main/java/ai/reveng/toolkit/ghidra/core/AnalysisLogConsumer.java index 17e196de..91891ed6 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/AnalysisLogConsumer.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/AnalysisLogConsumer.java @@ -1,6 +1,8 @@ package ai.reveng.toolkit.ghidra.core; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; + public interface AnalysisLogConsumer { - void consumeLogs(String logs); + void consumeLogs(String logs, GhidraRevengService.ProgramWithID programWithID); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisResultsLoaded.java b/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisResultsLoaded.java index 15e56a57..444ffe0d 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisResultsLoaded.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisResultsLoaded.java @@ -1,6 +1,6 @@ package ai.reveng.toolkit.ghidra.core; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ghidra.framework.plugintool.PluginEvent; /// Indicates that RevEng AI analysis results have been loaded and other components and plugins can now start using them. @@ -9,13 +9,13 @@ /// - Function IDs have been associated with {@link ghidra.program.model.listing.Function}s public class RevEngAIAnalysisResultsLoaded extends PluginEvent { public static final String NAME = "RevEngAIAnalysisResultsLoaded"; - private final ProgramWithBinaryID programWithBinaryID; + private final GhidraRevengService.AnalysedProgram programWithBinaryID; - public RevEngAIAnalysisResultsLoaded(String source, ProgramWithBinaryID programWithBinaryID) { + public RevEngAIAnalysisResultsLoaded(String source, GhidraRevengService.AnalysedProgram programWithBinaryID) { super(NAME, source); this.programWithBinaryID = programWithBinaryID; } - public ProgramWithBinaryID getProgramWithBinaryID() { + public GhidraRevengService.AnalysedProgram getProgramWithBinaryID() { return programWithBinaryID; } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java b/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java index 4dadcf95..4bbb7930 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/RevEngAIAnalysisStatusChangedEvent.java @@ -1,8 +1,8 @@ package ai.reveng.toolkit.ghidra.core; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ghidra.framework.plugintool.PluginEvent; import ghidra.program.model.listing.Program; @@ -21,38 +21,35 @@ */ public class RevEngAIAnalysisStatusChangedEvent extends PluginEvent { private final AnalysisStatus status; - private final ProgramWithBinaryID programWithBinaryID; + private final GhidraRevengService.ProgramWithID programWithID; - public RevEngAIAnalysisStatusChangedEvent(String sourceName, ProgramWithBinaryID programWithBinaryID, AnalysisStatus status) { + public RevEngAIAnalysisStatusChangedEvent(String sourceName, GhidraRevengService.ProgramWithID programWithID, AnalysisStatus status) { super(sourceName, "RevEngAI Analysis Finished"); - if (status == null || programWithBinaryID == null) { + if (status == null || programWithID == null) { throw new IllegalArgumentException("args cannot be null"); } this.status = status; - this.programWithBinaryID = programWithBinaryID; + this.programWithID = programWithID; } public AnalysisStatus getStatus() { return status; } - public ProgramWithBinaryID getProgramWithBinaryID() { - return programWithBinaryID; + public GhidraRevengService.ProgramWithID getProgramWithBinaryID() { + return programWithID; } public Program getProgram() { - return programWithBinaryID.program(); + return programWithID.program(); } - public BinaryID getBinaryID() { - return programWithBinaryID.binaryID(); - } @Override public String toString() { return "RevEngAIAnalysisStatusChangedEvent{" + "status=" + status + - ", programWithBinaryID=" + programWithBinaryID + + ", programWithBinaryID=" + programWithID + '}'; } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilder.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilder.java index 6bba37c8..ad720a97 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilder.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilder.java @@ -1,7 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api; import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisScope; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryHash; import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionBoundary; import ghidra.program.model.listing.Program; import ghidra.util.Msg; @@ -35,7 +34,7 @@ public AnalysisOptionsBuilder functionBoundaries(long base, List statusCache = new HashMap<>(); + private final Map statusCache = new HashMap<>(); private TypedApiInterface api; private ApiInfo apiInfo; private final List collections = new ArrayList<>(); private final List analysisIDFilter = new ArrayList<>(); + /// dedicated functions on the GhidraRevengService should be used instead to enforce assumptions via + /// type level guarantees +// @Deprecated public TypedApiInterface getApi() { return api; } @@ -93,23 +104,41 @@ public URI getServer() { return this.apiInfo.hostURI(); } - public void registerFinishedAnalysisForProgram(ProgramWithBinaryID programWithBinaryID) throws ApiException { - statusCache.put(programWithBinaryID.binaryID(), AnalysisStatus.Complete); - - // Add the binary to the program before loading the function info. Checking that a function is present requires - // the binary ID to be present in the program options. - addBinaryIDtoProgramOptions(programWithBinaryID.program(), programWithBinaryID.binaryID()); + public AnalysedProgram registerFinishedAnalysisForProgram(ProgramWithID programWithID, TaskMonitor monitor) throws CancelledException { + var status = status(programWithID); + if (!status.equals(AnalysisStatus.Complete)){ + throw new IllegalStateException("Analysis %s is not complete yet, current status: %s" + .formatted(programWithID.analysisID(), status)); + } + statusCache.put(programWithID.analysisID, AnalysisStatus.Complete); - loadFunctionInfo(programWithBinaryID.program(), programWithBinaryID.binaryID()); + var analysedProgram = associateFunctionInfo(programWithID); + pullFunctionInfoFromAnalysis(analysedProgram, monitor); + monitor.checkCancelled(); + return analysedProgram; } - public void addBinaryIDtoProgramOptions(Program program, BinaryID binID){ + private ProgramWithID addAnalysisIDtoProgramOptions(Program program, TypedApiInterface.AnalysisID analysisID){ var transactionId = program.startTransaction("Associate Binary ID with Program"); program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY) - .setLong(ReaiPluginPackage.OPTION_KEY_BINID, binID.value()); + .setLong(OPTION_KEY_ANALYSIS_ID, analysisID.id()); program.endTransaction(transactionId, true); + return new ProgramWithID(program, analysisID); } + private Namespace getRevEngAINameSpace(Program program) { + Namespace revengMatchNamespace = null; + try { + revengMatchNamespace = program.getSymbolTable().getOrCreateNameSpace( + program.getGlobalNamespace(), + REVENG_AI_NAMESPACE, + SourceType.ANALYSIS + ); + } catch (DuplicateNameException | InvalidInputException e) { + throw new RuntimeException(e); + } + return revengMatchNamespace; + } /** * Tries to find a BinaryID for a given program * If the program already has a BinaryID associated with it, it will return that @@ -117,26 +146,63 @@ public void addBinaryIDtoProgramOptions(Program program, BinaryID binID){ * @param program * @return */ + @Deprecated public Optional getBinaryIDFor(Program program) { - Optional binID; - try { - binID = getBinaryIDfromOptions(program); - } catch (InvalidBinaryID e) { - Msg.error(this, "Invalid Binary ID found in program options: %s".formatted(e.getMessage())); - return Optional.empty(); - } - return binID; - + return getBinaryIDfromOptions(program); } - public Optional getAnalysisIDFor(Program program){ - return getBinaryIDFor(program).map(binID -> api.getAnalysisIDfromBinaryID(binID)); + @SuppressWarnings("deprecation") // Using deprecated method to support legacy BinaryID + private Optional getAnalysisIDFor(Program program){ + var optAnalysisID = getAnalysisIDFromOptions(program); + if (optAnalysisID.isPresent()){ + return optAnalysisID; + } + // Fallback to getting it from the BinaryID, if one exists + var legacyBinaryID = getBinaryIDFor(program); + if (legacyBinaryID.isPresent()) { + // We have a legacy binary ID, upgrade to AnalysisID + var analysisID = api.getAnalysisIDfromBinaryID(legacyBinaryID.get()); + addAnalysisIDtoProgramOptions(program, analysisID); + program.withTransaction("Remove legacy BinaryID from program options", () -> + program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY) + .setLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID) + ); + return Optional.of(analysisID); + } + return Optional.empty(); } + private Optional getAnalysisIDFromOptions( + Program program + ) { + long bid = program.getOptions( + ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(OPTION_KEY_ANALYSIS_ID, + ReaiPluginPackage.INVALID_ANALYSIS_ID); + if (bid == ReaiPluginPackage.INVALID_ANALYSIS_ID) { + return Optional.empty(); + } + var analysisID = new TypedApiInterface.AnalysisID((int) bid); + if (!statusCache.containsKey(analysisID)) { + // Check that it's really valid in the context of the currently configured API + try { + var status = api.status(analysisID); + statusCache.put(analysisID, status); + } catch (APIAuthenticationException | ApiException e) { + Msg.showError(this, null, "Invalid Analysis ID", + ("The Analysis ID %s stored in the program options is invalid for the currently configured RevEng.AI server %s. " + + "This could be an intermittent error, or you switched the servers") + .formatted(analysisID, this.apiInfo.hostURI()), e); + return Optional.empty(); + } + // Now it's certain that it is a valid binary ID + } + return Optional.of(analysisID); + } - public Optional getBinaryIDfromOptions( + @Deprecated + private Optional getBinaryIDfromOptions( Program program - ) throws InvalidBinaryID { + ) { long bid = program.getOptions( ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID); @@ -144,27 +210,37 @@ public Optional getBinaryIDfromOptions( return Optional.empty(); } var binID = new BinaryID((int) bid); - if (!statusCache.containsKey(binID)) { - // Check that it's really valid in the context of the currently configured API - try { - var status = api.status(binID); - statusCache.put(binID, status); - } catch (APIAuthenticationException | ApiException e) { - throw new InvalidBinaryID(binID, this.apiInfo); - } - // Now it's certain that it is a valid binary ID + // Check that it's really valid in the context of the currently configured API + AnalysisStatus status; + try { + status = api.status(binID); + } catch (APIAuthenticationException | ApiException e) { + Msg.showError(this, null, "Invalid Binary ID", + ("The Binary ID %s stored in the program options is invalid for the currently configured RevEng.AI server %s. " + + "This could be an intermittent error, or you switched servers") + .formatted(binID, this.apiInfo.hostURI()), e); + return Optional.empty(); } + var analysisID = api.getAnalysisIDfromBinaryID(binID); + statusCache.put(analysisID, status); + + // Now it's certain that it is a valid binary ID return Optional.of(binID); } - /** - * Loads the function info into a dedicated user property map. - * Future task: we can potentially merge the function ID map with the mangled name map and have a single map of objects. - */ - private void loadFunctionInfo(Program program, BinaryID binID) throws ApiException { - List functionInfo = api.getFunctionInfo(binID); - var transactionID = program.startTransaction("Load Function Info"); + /// Loads the function info into a dedicated user property map. + /// This method should only concern itself with associating the FunctionID with the Ghidra Function + /// This property is immutable within an Analysis: The function ID will never change unless an entirely different + /// analysis is associated with the program + /// Other function information like the name and signature should be loaded in [#pullFunctionInfoFromAnalysis(AnalysedProgram ,TaskMonitor)] + /// because this information can change on the server, and thus needs a dedicated method to refresh it + private AnalysedProgram associateFunctionInfo(ProgramWithID knownProgram) { + var analysisID = knownProgram.analysisID(); + var program = knownProgram.program(); + List functionInfo = null; + functionInfo = api.getFunctionInfo(analysisID); + var transactionID = program.startTransaction("Associate Function Info"); // Create the FunctionID map LongPropertyMap functionIDMap; @@ -176,148 +252,257 @@ private void loadFunctionInfo(Program program, BinaryID binID) throws ApiExcepti } // Create the function mangled name map - StringPropertyMap mangledNameMap; try { - mangledNameMap = program.getUsrPropertyManager().createStringPropertyMap(REAI_FUNCTION_MANGLED_MAP); + program.getUsrPropertyManager().createStringPropertyMap(REAI_FUNCTION_MANGLED_MAP); } catch (DuplicateNameException e) { program.endTransaction(transactionID, false); throw new RuntimeException("Previous mangled name property map still exists",e); } LongPropertyMap finalFunctionIDMap = functionIDMap; - StringPropertyMap finalMangledNameMap = mangledNameMap; - - AtomicInteger ghidraRenamedFunctions = new AtomicInteger(); - AtomicInteger ghidraBoundariesMatchedFunction = new AtomicInteger(); - functionInfo.forEach( - info -> { - var oFunc = getFunctionFor(info, program); - if (oFunc.isEmpty()){ - Msg.error(this, "Function not found in Ghidra for info: %s".formatted(info)); - return; - } - var func = oFunc.get(); - - // Extract the mangled name from Ghidra - var ghidraMangledName = func.getSymbol().getName(false); - var revEngMangledName = info.functionMangledName(); - var revEngDemangledName = info.functionName(); - // Skip external and thunk functions because we don't support them - if (func.isExternal() || func.isThunk()){ - Msg.debug(this, "Skipping external/thunk function %s".formatted(ghidraMangledName)); - return; - } + int ghidraBoundariesMatchedFunction = 0; + for (FunctionInfo info : functionInfo) { + var oFunc = getFunctionFor(info, program); + if (oFunc.isEmpty()) { + Msg.error(this, "Function not found in Ghidra for info: %s".formatted(info)); + continue; + } + var func = oFunc.get(); + // There are two ways to think about the size of a function + // They diverge for non-contiguous functions + var funcSizeByAddressCount = func.getBody().getNumAddresses(); + var funcSizeByDistance = func.getBody().getMaxAddress().subtract(func.getEntryPoint()) + 1; + + // For unclear reasons the func size is off by one + if (funcSizeByAddressCount - 1 != info.functionSize() && funcSizeByAddressCount != info.functionSize()) { + Msg.warn(this, "Function size mismatch for function %s: %d vs %d".formatted(func.getName(), funcSizeByAddressCount, info.functionSize())); + continue; + } - // Skip invalid function mangled names - if (revEngMangledName.contains(" ") || revEngDemangledName.contains(" ")) { - Msg.warn(this, "Skipping renaming of function %s to invalid name %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); - return; - } + finalFunctionIDMap.add(func.getEntryPoint(), info.functionID().value()); - var funcSize = func.getBody().getNumAddresses(); + ghidraBoundariesMatchedFunction++; + } - // For unclear reasons the func size is off by one - if (funcSize - 1 != info.functionSize() && funcSize != info.functionSize()){ - Msg.warn(this, "Function size mismatch for function %s: %d vs %d".formatted(ghidraMangledName, funcSize, info.functionSize())); - return; - } - // Source types: - // DEFAULT: placeholder name automatically assigned by Ghidra when it doesn’t know the real name. - // ANALYSIS: A name/signature inferred by one of Ghidra’s analysis engines (or demangler) rather than simply “default.” - // IMPORTED: Information taken from an external source — symbols or signatures imported from a file or database. - // USER_DEFINED: A name or signature explicitly set by the analyst. - if (func.getSymbol().getSource() == SourceType.DEFAULT && !revEngMangledName.startsWith("FUN_") ){ - Msg.info(this, "Renaming function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); - try { - // Set demangled name from RevEng.AI - func.setName(revEngDemangledName, SourceType.ANALYSIS); - ghidraRenamedFunctions.getAndIncrement(); - } catch (Exception e) { - throw new RuntimeException(e); - } + program.endTransaction(transactionID, true); - } - finalFunctionIDMap.add(func.getEntryPoint(), info.functionID().value()); - finalMangledNameMap.add(func.getEntryPoint(), revEngMangledName); - - ghidraBoundariesMatchedFunction.getAndIncrement(); - } - ); + var analysedProgram = new AnalysedProgram(program, analysisID); AtomicInteger ghidraFunctionCount = new AtomicInteger(); program.getFunctionManager().getFunctions(true).forEach( func -> { if (!func.isExternal() && !func.isThunk()){ ghidraFunctionCount.getAndIncrement(); - if (getFunctionIDFor(func).isEmpty()) { + if (analysedProgram.getIDForFunction(func).isEmpty()) { Msg.info(this, "Function %s not found in RevEng.AI".formatted(func.getSymbol().getName(false))); } } } ); - program.endTransaction(transactionID, true); - // Print summary Msg.showInfo(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Function loading summary", - ("Found %d functions from RevEng.AI. Renamed %d. Your local Ghidra instance has %d/%d matching function " + - "boundaries. For better results, please start a new analysis from this plugin.").formatted( - functionInfo.size(), - ghidraRenamedFunctions.get(), - ghidraBoundariesMatchedFunction.get(), - ghidraFunctionCount.get() - )); - } + ("Found %d functions from RevEng.AI. Your local Ghidra instance has %d/%d matching function " + + "boundaries. For better results, please start a new analysis from this plugin.").formatted( + functionInfo.size(), + ghidraBoundariesMatchedFunction, + ghidraFunctionCount.get() + )); - /** - * Get the FunctionID for a Ghidra Function, if there is one - * - * There are two cases where a function ID is missing: - * 1. Either the whole program has not been analyzed - * 2. Or the function was not found as part of the analysis on the server - * (because its bounds were not included when the analysis was triggered) - */ - public Optional getFunctionIDFor(Function function){ - return getKnownProgram(function.getProgram()) - .flatMap(knownProgram -> getFunctionIDFor(knownProgram, function)); - } + return analysedProgram; - public Optional getFunctionIDFor(ProgramWithBinaryID knownProgram, Function function){ - Optional functionIDMap = getFunctionIDMap(knownProgram); - return functionIDMap - .flatMap(map -> Optional.ofNullable(map.get(function.getEntryPoint()))) - .map(FunctionID::new); } - private Optional getFunctionIDMap(ProgramWithBinaryID program){ - return Optional.ofNullable(program.program().getUsrPropertyManager().getLongPropertyMap(REAI_FUNCTION_PROP_MAP)); - } - public Optional getFunctionMangledNamesMap(Program program) { - return Optional.ofNullable(program.getUsrPropertyManager().getStringPropertyMap(REAI_FUNCTION_MANGLED_MAP)); + public record RenameResult(Function func, String originalName, String newName) { + public String virtualAddress() { + return func.getEntryPoint().toString(); + } } + /// Pull the server side information about the functions from a remote Analysis and update the local {@link Program} + /// based on it + /// This currently includes: + /// * the name of the function + /// * the type signature of the function + /// + /// It assumes that the initial load already happened, i.e. the functions have an associated FunctionID already. + /// The initial association happens in {@link #associateFunctionInfo(ProgramWithID)} + /// + public List pullFunctionInfoFromAnalysis(AnalysedProgram analysedProgram, TaskMonitor monitor) { + var transactionId = analysedProgram.program().startTransaction("RevEng.AI: Pull Function Info from Analysis"); + + List renameResults = new ArrayList<>(); + + int failedRenames = 0; + + var revEngNamespace = getRevEngAINameSpace(analysedProgram.program()); + + Map functionInfoMap = api.getFunctionInfo(analysedProgram.analysisID()).stream() + .collect( + Collectors.toMap( + FunctionInfo::functionID, + fi -> fi + ) + ); - public BiMap getFunctionMap(Program program){ - var propMap = program.getUsrPropertyManager().getLongPropertyMap(REAI_FUNCTION_PROP_MAP); - BiMap functionMap = HashBiMap.create(); - propMap.getPropertyIterator().forEachRemaining( - addr -> { - var func = program.getFunctionManager().getFunctionAt(addr); - try { - functionMap.put(new FunctionID(propMap.getLong(addr)), func); - } catch (NoValueException e) { - // This should never happen, because we're iterating over the keys - throw new RuntimeException(e); + Map signatureMap = api.listFunctionDataTypesForAnalysis(analysedProgram.analysisID).getItems() + .stream() + .filter(item -> item.getStatus().equals("completed")) + .filter(item -> item.getDataTypes().getFuncTypes() != null) + .collect( + Collectors.toMap( + item -> new TypedApiInterface.FunctionID(item.getFunctionId()), + fdtStatus -> fdtStatus + ) + ); + + for (Function function : analysedProgram.program().getFunctionManager().getFunctions(true)) { + if (monitor.isCancelled()) { + continue; + } + var ghidraMangledName = function.getSymbol().getName(false); + // Skip external and thunk functions because we don't support them + if (function.isExternal() || function.isThunk()) { + Msg.debug(this, "Skipping external/thunk function %s".formatted(ghidraMangledName)); + continue; + } + + var fID = analysedProgram.getIDForFunction(function); + if (fID.isEmpty()) { + Msg.info(this, "Function %s has no associated FunctionID, skipping".formatted(function.getName())); + continue; + } + + // Get the current name on the server side +// FunctionDetails details = api.getFunctionDetails(fID.get().functionID); + FunctionInfo details = functionInfoMap.get(fID.get().functionID); + + // Extract the mangled name from Ghidra + var revEngMangledName = details.functionMangledName(); + var revEngDemangledName = details.functionName(); + + // Skip invalid function mangled names + if (revEngMangledName.contains(" ") || revEngDemangledName.contains(" ")) { + Msg.warn(this, "Skipping renaming of function %s to invalid name %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); + continue; + } + + var sig = Optional.ofNullable(signatureMap.get(fID.get().functionID)); + // Get the type information on the server side + Optional functionSignatureMessageOpt = sig + // Try getting the data types if they are available + // If they are available, try converting them to a Ghidra signature + // If the conversion fails, act like there is no signature available + .flatMap (item -> Optional.ofNullable(item.getDataTypes())) + .flatMap((functionDataTypeMessage -> { + try { + return getFunctionSignature(functionDataTypeMessage); + } catch (DataTypeDependencyException e) { + // Something went wrong loading the data type dependencies + // just skip applying the signature and treat it like none being available + Msg.error(this, "Could not get parse signature for function %s".formatted(function.getName())); + return Optional.empty(); + } + })); + + + analysedProgram.setMangledNameForFunction(function, revEngMangledName); + + /// Source types: + /// DEFAULT: placeholder name automatically assigned by Ghidra when it doesn’t know the real name. + /// ANALYSIS: A name/signature inferred by one of Ghidra’s analysis engines (or demangler) rather than simply “default.” + /// IMPORTED: Information taken from an external source — symbols or signatures imported from a file or database. + /// USER_DEFINED: A name or signature explicitly set by the analyst. + /// See {@link ghidra.program.model.symbol.SourceType} for more details + if (function.getSymbol().getSource() == SourceType.DEFAULT) { + if (functionSignatureMessageOpt.isEmpty()) { + // We don't have signature information for this function, so we can only try renaming it + if (function.getSymbol().getSource() == SourceType.DEFAULT && !revEngMangledName.startsWith("FUN_")) { + // The local function has the default name, so we can rename it + // The following check should never fail because it is a default name, + // and we checked above that the server name is not a default name + // but just to be safe and make that assumption explicit we check it explicitly + if (!function.getSymbol().getName(false).equals(revEngDemangledName)) { + Msg.info(this, "Renaming function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); + try { + function.setParentNamespace(revEngNamespace); + } catch (DuplicateNameException | InvalidInputException | CircularDependencyException e) { + throw new RuntimeException(e); + } + var success = new SetFunctionNameCmd(function.getEntryPoint(), revEngDemangledName, SourceType.ANALYSIS) + .applyTo(analysedProgram.program()); + if (success) { + renameResults.add(new RenameResult( + function, + ghidraMangledName, + revEngDemangledName + )); + } else { + failedRenames++; + Msg.error(this, "Failed to rename function %s to %s [%s]".formatted(ghidraMangledName, revEngMangledName, revEngDemangledName)); + } + } + } + + } else { + /// We could use {@link ghidra.program.model.listing.FunctionSignature#isEquivalentSignature(FunctionSignature)} + /// if we expect the server to have changing signatures at any point in time. + /// For now, we only apply signatures to functions that have the default signature + if (function.getSignatureSource() == SourceType.DEFAULT) { + var success = new ApplyFunctionSignatureCmd( + function.getEntryPoint(), + functionSignatureMessageOpt.get(), + SourceType.ANALYSIS + ).applyTo(analysedProgram.program(), monitor); + // For unclear reasons the signature source is not set by the command in Ghidra 11.2.x and lower + if (success) { + renameResults.add(new RenameResult( + function, + ghidraMangledName, + revEngDemangledName + )); + } else { + Msg.error(this, "Failed to apply signature to function %s".formatted(function.getName())); + failedRenames++; + } } } - ); - return functionMap; + } + + + } + // Done iterating over all functions. If nothing changed, discard the transaction, to keep undo history clean + analysedProgram.program().endTransaction(transactionId, !renameResults.isEmpty() && !monitor.isCancelled()); + if (failedRenames > 0){ + Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Function Update Summary", + ("Failed to update %d functions from RevEng.AI. Please check the error log for details.").formatted( + failedRenames + )); + } + return renameResults; + } + + /** + * Get the FunctionID for a Ghidra Function, if there is one + * There are two cases where a function ID is missing: + * 1. Either the whole program has not been analyzed + * (because its bounds were not included when the analysis was triggered) + * + * @deprecated Use {@link AnalysedProgram#getIDForFunction(Function)} instead. It forces the caller to prove that they know that the {@link Program} is indeed known on the server and associated by having to provide a {@link AnalysedProgram} instance. + */ + @Deprecated + public Optional getFunctionIDFor(Function function){ + return getAnalysedProgram(function.getProgram()) + .flatMap(knownProgram -> knownProgram.getIDForFunction(function).map(fidWithStatus -> fidWithStatus.functionID)); } - public Optional getFunctionFor(FunctionInfo functionInfo, Program program){ + /** + * Get the Ghidra Function for a given FunctionInfo if there is one + */ + private Optional getFunctionFor(FunctionInfo functionInfo, Program program){ // These addresses used to be relative, but are now absolute again var defaultAddressSpace = program.getAddressFactory().getDefaultAddressSpace(); var funcAddress = defaultAddressSpace.getAddress(functionInfo.functionVirtualAddress()); @@ -326,46 +511,34 @@ public Optional getFunctionFor(FunctionInfo functionInfo, Program prog return Optional.ofNullable(func); } - public List searchForHash(BinaryHash hash){ + @Deprecated + public List searchForHash(TypedApiInterface.BinaryHash hash){ return api.search(hash); } - public boolean isKnownProgram(Program program){ - var storedBinID = program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID); - return storedBinID != ReaiPluginPackage.INVALID_BINARY_ID; - } - public void removeProgramAssociation(Program program){ - BinaryID binID; - try { - var maybebinID = getBinaryIDfromOptions(program); - if (maybebinID.isEmpty()){ - Msg.warn(this, "No binary ID found for program, cannot remove association"); - return; - } - binID = maybebinID.get(); - } catch (InvalidBinaryID e) { - // The program has an invalid binary ID, which can happen if the server was changed - // This is a very good reason to remove the association, so we unpack the id from the error - binID = e.getBinaryID(); - } // Clear all function ID data - program.getUsrPropertyManager().removePropertyMap(REAI_FUNCTION_PROP_MAP); program.getUsrPropertyManager().removePropertyMap(REAI_FUNCTION_MANGLED_MAP); - statusCache.remove(binID); - program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY).setLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID); + var reaiOptions = program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY); + //noinspection deprecation + reaiOptions.setLong(ReaiPluginPackage.OPTION_KEY_BINID, ReaiPluginPackage.INVALID_BINARY_ID); + reaiOptions.setLong(OPTION_KEY_ANALYSIS_ID, ReaiPluginPackage.INVALID_ANALYSIS_ID); + // Clear the entire cache. Getting the correct ID is not worth the effort in terms of edge cases to handle + // because this method should still work even if the analysis ID or binary ID that was associated is invalid + statusCache.clear(); + } - public boolean isProgramAnalysed(Program program){ + /// This method is private to the service, because it only concerns itself with how the service determines + /// this internally + /// Plugin code that wants to know if a program is known should use {@link #getAnalysedProgram(Program)} and check + /// if the result is present + private boolean isProgramAnalysed(Program program){ return program.getUsrPropertyManager().getLongPropertyMap(REAI_FUNCTION_PROP_MAP) != null && program.getUsrPropertyManager().getStringPropertyMap(REAI_FUNCTION_MANGLED_MAP) != null; } - public boolean isKnownFunction(Function function){ - return getFunctionIDFor(function).isPresent(); - } - public static List exportFunctionBoundaries(Program program){ List result = new ArrayList<>(); Address imageBase = program.getImageBase(); @@ -379,12 +552,12 @@ public static List exportFunctionBoundaries(Program program){ return result; } - private BinaryHash hashOfProgram(Program program) { + private TypedApiInterface.BinaryHash hashOfProgram(Program program) { // TODO: we break the guarantee that a BinaryHash implies that a file of this hash has already been uploaded - return new BinaryHash(program.getExecutableSHA256()); + return new TypedApiInterface.BinaryHash(program.getExecutableSHA256()); } - public BinaryHash upload(Program program) { + public TypedApiInterface.BinaryHash upload(Program program) { // TODO: Check if the program is already uploaded on the server // But this requires a dedicated API to do cleanly @@ -414,7 +587,7 @@ public BinaryHash upload(Program program) { } } - public BinaryHash upload(Path path) { + public TypedApiInterface.BinaryHash upload(Path path) { try { return api.upload(path); } catch (FileNotFoundException | ApiException e) { @@ -422,6 +595,7 @@ public BinaryHash upload(Path path) { } } + @Deprecated public AnalysisStatus pollStatus(BinaryID bid) { try { return api.status(bid); @@ -430,11 +604,27 @@ public AnalysisStatus pollStatus(BinaryID bid) { } } - public String decompileFunctionViaAI(Function function, TaskMonitor monitor, AIDecompilationdWindow window) { + /// Use this method if you just have an AnalysisID and it is not clear yet if it can be accessed + public AnalysisStatus pollStatus(TypedApiInterface.AnalysisID id) throws ApiException { + return api.status(id); + } + + public AnalysisStatus status(ProgramWithID program) { + try { + return api.status(program.analysisID()); + } catch (ApiException e) { + // This should never happen given that `ProgramWithID` guarantees a valid analysis ID + throw new RuntimeException(e); + } + } + + + + public String decompileFunctionViaAI(FunctionWithID functionWithID, TaskMonitor monitor, AIDecompilationdWindow window) { monitor.setMaximum(100 * 50); - var fID = getFunctionIDFor(function) - .orElseThrow(() -> new RuntimeException("Function has no associated FunctionID")); // Check if there is an existing process already, because the trigger API will fail with 400 if there is + var fID = functionWithID.functionID; + var function = functionWithID.function; if (api.pollAIDecompileStatus(fID).status().equals("uninitialised")){ // Trigger the decompilation api.triggerAIDecompilationForFunctionID(fID); @@ -475,74 +665,93 @@ public String decompileFunctionViaAI(Function function, TaskMonitor monitor, AID } } - public ProgramWithBinaryID analyse(Program program, AnalysisOptionsBuilder analysisOptionsBuilder, TaskMonitor monitor) throws CancelledException, ApiException { + + /// This method analyses a program by uploading it (if necessary), triggering an analysis, and _blocking_ + /// until the analysis is complete. This is for scripts and tests, and must not be used on the UI thread + /// It does not upload the program, this must be done beforehand, and the hash must be associated via {@link AnalysisOptionsBuilder#hash(TypedApiInterface.BinaryHash)} + public AnalysedProgram analyse(Program program, AnalysisOptionsBuilder analysisOptionsBuilder, TaskMonitor monitor) throws CancelledException, ApiException { + // Check if we are on the swing thread var programWithBinaryID = startAnalysis(program, analysisOptionsBuilder); - waitForFinishedAnalysis(monitor, programWithBinaryID, null, null); - registerFinishedAnalysisForProgram(programWithBinaryID); - return programWithBinaryID; + var finalStatus = waitForFinishedAnalysis(monitor, programWithBinaryID, null, null); + // TODO: Check final status for errors, and do something appropriate on failure + var analysedProgram = registerFinishedAnalysisForProgram(programWithBinaryID, monitor); + if (getKnownProgram(program).isEmpty()){ + throw new IllegalStateException("Program is not known after finished analysis. Something seriously went wrong."); + } + return analysedProgram; } - public Optional getKnownProgram(Program program) { - return getBinaryIDFor(program).map(binID -> { - var analysisID = api.getAnalysisIDfromBinaryID(binID); - return new ProgramWithBinaryID(program, binID, analysisID); - } - ); + /// Get the {@link ProgramWithID} for a known program + /// This only guarantees an associated analysis, not that it is finished + public Optional getKnownProgram(Program program) { + var analysisID = getAnalysisIDFor(program); + return analysisID.map(id -> new ProgramWithID(program, id)); } - public Optional getFunctionSignatureArtifact(BinaryID binID, FunctionID functionID) { - var analysisID = api.getAnalysisIDfromBinaryID(binID); - return api.getFunctionDataTypes(analysisID, functionID).flatMap(FunctionDataTypeStatus::data_types); + /// Get the {@link AnalysedProgram} for a known and analysed program + public Optional getAnalysedProgram(Program program) { + var kProg = getKnownProgram(program); + if (kProg.isEmpty()){ + return Optional.empty(); + } + if (isProgramAnalysed(kProg.get().program())){ + return Optional.of(new AnalysedProgram(kProg.get().program(), kProg.get().analysisID())); + } + return Optional.empty(); } /** - * Create a {@link FunctionDefinitionDataType} from a @{@link FunctionDataTypeMessage} in isolation + * Create a {@link FunctionDefinitionDataType} from a @{@link FunctionInfoOutput} in isolation * - * All the required dependency types should be stored in the DataTypeManager that is associated with this + * All the required dependency types will be stored in the DataTypeManager that is associated with this * FunctionDefinitionDataType * * @param functionDataTypeMessage The message containing the function signature, received from the API * @return Self-contained signature for the function */ - public static FunctionDefinitionDataType getFunctionSignature(FunctionDataTypeMessage functionDataTypeMessage) throws DataTypeDependencyException { - - // TODO: Do we need the program or data type manager? - // Or can we just create a new one with all the necessary types and then they get merged? + public static Optional getFunctionSignature(FunctionInfoOutput functionDataTypeMessage) throws DataTypeDependencyException { // Create Data Type Manager with all dependencies - var dtm = loadDependencyDataTypes(functionDataTypeMessage.func_deps()); + var d = FunctionDependencies.fromOpenAPI(functionDataTypeMessage.getFuncDeps()); + var dtm = loadDependencyDataTypes(d); - FunctionDefinitionDataType f = new FunctionDefinitionDataType(functionDataTypeMessage.functionName(), dtm); + if (functionDataTypeMessage.getFuncTypes() == null){ + return Optional.empty(); + } + var funcName = functionDataTypeMessage.getFuncTypes().getName(); + FunctionDefinitionDataType f = new FunctionDefinitionDataType(funcName, dtm); try { - f.setName(functionDataTypeMessage.functionName()); + f.setName(funcName); } catch (InvalidNameException e) { throw new RuntimeException(e); } - ParameterDefinitionImpl[] args = Arrays.stream(functionDataTypeMessage.func_types().header().args()).map( + ParameterDefinitionImpl[] args = functionDataTypeMessage.getFuncTypes().getHeader().getArgs().values().stream().map( arg -> { DataType ghidraType = null; try { - ghidraType = loadDataType(dtm, arg.type(), functionDataTypeMessage.func_deps()); + var scopedName = TypePathAndName.fromString(arg.getType()); + ghidraType = loadDataType(dtm, scopedName); } catch (DataTypeDependencyException e) { Msg.error(GhidraRevengService.class, - "Couldn't find type '%s' for param of %s".formatted(arg.type(), functionDataTypeMessage.functionName()) + ("" + + "Couldn't find type '%s' for param of %s").formatted(arg.getType(), funcName) ); - ghidraType = Undefined.getUndefinedDataType(arg.size()); + ghidraType = Undefined.getUndefinedDataType(arg.getSize()); } // Add the type to the DataTypeManager - return new ParameterDefinitionImpl(arg.name(), ghidraType, null); + return new ParameterDefinitionImpl(arg.getName(), ghidraType, null); }).toArray(ParameterDefinitionImpl[]::new); f.setArguments(args); DataType returnType = null; - returnType = loadDataType(dtm, functionDataTypeMessage.func_types().header().type(), functionDataTypeMessage.func_deps()); + returnType = loadDataType(dtm, TypePathAndName.fromString(functionDataTypeMessage.getFuncTypes().getHeader().getType())); f.setReturnType(returnType); - return f; + return Optional.of(f); } public static DataTypeManager loadDependencyDataTypes(FunctionDependencies dependencies){ @@ -587,7 +796,8 @@ public static DataTypeManager loadDependencyDataTypes(FunctionDependencies depen var path = TypePathAndName.fromString(typeDef.name()); DataType type; try { - type = dataTypeParser.parse(typeDef.type()); + var scopedType = TypePathAndName.fromString(typeDef.type()); + type = dataTypeParser.parse(scopedType.name()); } catch (InvalidDataTypeException e) { // The type wasn't available in the DataTypeManager yet, try again later typeDefsToAdd.add(typeDef); @@ -596,7 +806,7 @@ public static DataTypeManager loadDependencyDataTypes(FunctionDependencies depen } catch (CancelledException e) { throw new RuntimeException(e); } - TypedefDataType typedefDataType = new TypedefDataType(new CategoryPath(CategoryPath.ROOT, path.path()), path.name(), type, null); + TypedefDataType typedefDataType = new TypedefDataType(path.toCategoryPath(), path.name(), type, null); dtm.addDataType(typedefDataType, DataTypeConflictHandler.REPLACE_EMPTY_STRUCTS_OR_RENAME_AND_ADD_HANDLER); } @@ -611,7 +821,7 @@ public static DataTypeManager loadDependencyDataTypes(FunctionDependencies depen binSyncStructMember -> { DataType fieldType = null; try { - fieldType = loadDataType(dtm, binSyncStructMember.type(), dependencies); + fieldType = loadDataType(dtm, TypePathAndName.fromString(binSyncStructMember.type())); } catch (DataTypeDependencyException e) { Msg.error( GhidraRevengService.class, @@ -639,7 +849,7 @@ public static DataTypeManager loadDependencyDataTypes(FunctionDependencies depen return dtm; } - private static DataType loadDataType(DataTypeManager dtm, String name, FunctionDependencies dependencies) throws DataTypeDependencyException { + private static DataType loadDataType(DataTypeManager dtm, TypePathAndName type) throws DataTypeDependencyException { DataTypeParser dataTypeParser = new DataTypeParser( dtm, null, @@ -647,17 +857,17 @@ private static DataType loadDataType(DataTypeManager dtm, String name, FunctionD DataTypeParser.AllowedDataTypes.ALL); DataType dataType; try { - dataType = dataTypeParser.parse(name); + dataType = dataTypeParser.parse(type.name()); } catch (InvalidDataTypeException e) { // The type wasn't available in the DataTypeManager, so we have to find it in the dependencies - throw new DataTypeDependencyException("Data type not found in DataTypeManager: %s".formatted(name), e); + throw new DataTypeDependencyException("Data type not found in DataTypeManager: %s".formatted(type), e); } catch (CancelledException e) { throw new RuntimeException(e); } return dataType; } - public String getAnalysisLog(AnalysisID analysisID) { + public String getAnalysisLog(TypedApiInterface.AnalysisID analysisID) { return api.getAnalysisLogs(analysisID); } @@ -676,7 +886,7 @@ public BoxPlot getNameScoreForMatch(GhidraFunctionMatch functionMatch) { public static final String PORTAL_FUNCTIONS = "function/"; - public void openFunctionInPortal(FunctionID functionID) { + public void openFunctionInPortal(TypedApiInterface.FunctionID functionID) { openPortal(PORTAL_FUNCTIONS, String.valueOf(functionID.value())); } @@ -687,16 +897,20 @@ public void openCollectionInPortal(Collection collection) { public void openPortalFor(Collection c){ openCollectionInPortal(c); } - public void openPortalFor(FunctionID f){ + public void openPortalFor(TypedApiInterface.FunctionID f){ openFunctionInPortal(f); } public void openPortalFor(AnalysisResult analysisResult) { - openPortal("analyses", String.valueOf(analysisResult.analysisID().id())); + openPortalFor(analysisResult.analysisID()); + } + + public void openPortalFor(ProgramWithID programWithID) { + openPortalFor(programWithID.analysisID()); } - public void openPortalFor(LegacyAnalysisResult analysisResult) { - openPortal("analyses", String.valueOf(analysisResult.binary_id().value())); + public void openPortalFor(TypedApiInterface.AnalysisID analysisID) { + openPortal("analyses", String.valueOf(analysisID.id())); } public void openPortal(String... subPath) { @@ -756,11 +970,12 @@ public List getActiveAnalysisIDsFilter() { } /** + * @param tool The UI tool for firing an event on status changes. Can be null * @return The final AnalysisStatus, should be either Complete or Error */ public AnalysisStatus waitForFinishedAnalysis( TaskMonitor monitor, - ProgramWithBinaryID programWithID, + ProgramWithID programWithID, @Nullable AnalysisLogConsumer logger, @Nullable PluginTool tool @@ -770,12 +985,12 @@ public AnalysisStatus waitForFinishedAnalysis( // TODO: In the future this can be made smarter and e.g. wait longer if the analysis log hasn't changed AnalysisStatus lastStatus = null; while (true) { - AnalysisStatus currentStatus = this.api.status(programWithID.analysisID()); + AnalysisStatus currentStatus = this.status(programWithID); if (currentStatus != AnalysisStatus.Queued) { // Analysis log endpoint only starts to return data after the analysis is processing String logs = this.getAnalysisLog(programWithID.analysisID()); if (logger != null) { - logger.consumeLogs(logs); + logger.consumeLogs(logs, programWithID); } var logsLines = logs.lines().toList(); var lastLine = logsLines.get(logsLines.size() - 1); @@ -801,18 +1016,16 @@ public AnalysisStatus waitForFinishedAnalysis( } } - public ProgramWithBinaryID startAnalysis(Program program, AnalysisOptionsBuilder analysisOptionsBuilder) throws ApiException { - var binaryID = api.analyse(analysisOptionsBuilder); - AnalysisID analysisID = api.getAnalysisIDfromBinaryID(binaryID); - addBinaryIDtoProgramOptions(program, binaryID); - return new ProgramWithBinaryID(program, binaryID, analysisID); + public ProgramWithID startAnalysis(Program program, AnalysisOptionsBuilder analysisOptionsBuilder) throws ApiException { + var analysisID = api.analyse(analysisOptionsBuilder); + return addAnalysisIDtoProgramOptions(program, analysisID); } public Map getNameScores(java.util.Collection values) { // Get the confidence scores for each match in the input List r = api.getNameScores(values.stream().map(GhidraFunctionMatch::functionMatch).toList(), false); // Collect to a Map from the FunctionID to the actual score - Map plots = r.stream().collect(Collectors.toMap(FunctionNameScore::functionID, FunctionNameScore::score)); + Map plots = r.stream().collect(Collectors.toMap(FunctionNameScore::functionID, FunctionNameScore::score)); return values.stream().collect(Collectors.toMap( match -> match, match -> plots.get(match.functionMatch().origin_function_id()) @@ -824,23 +1037,43 @@ public Map getNameScores(java.util.Collection getSignatures(java.util.Collection values) { + public Map getSignatures(java.util.Collection values) { + - DataTypeList dataTypesList = this.api.getFunctionDataTypes(values.stream().map(GhidraFunctionMatch::nearest_neighbor_id).toList()); - Map signatureMap = Arrays.stream(dataTypesList.dataTypes()) - .filter(FunctionDataTypeStatus::completed) - .filter(status -> status.data_types().isPresent()) + // Get all data type info for the neighbour functions + var dataTypesList = this.api.listFunctionDataTypesForFunctions( + values.stream().map(GhidraFunctionMatch::nearest_neighbor_id).toList() + ); + // Create a map from FunctionID to FunctionInfoOutput for easy lookup, only for completed signatures + Map signatureMap = dataTypesList.getItems().stream() + // Only keep completed signatures + .filter(FunctionDataTypesListItem::getCompleted) + // Double check that there is a data type available + .filter(functionDataTypesListItem -> functionDataTypesListItem.getDataTypes() != null) .collect(Collectors.toMap( - FunctionDataTypeStatus::functionID, - status -> status.data_types().get() + item -> new TypedApiInterface.FunctionID(item.getFunctionId()), + FunctionDataTypesListItem::getDataTypes )); - return values.stream() + Map matchMap = values.stream() .filter(match -> signatureMap.containsKey(match.functionMatch().nearest_neighbor_id())) .collect(Collectors.toMap( match -> match, match -> signatureMap.get(match.functionMatch().nearest_neighbor_id()) )); + + // Now parse all signatures + Map result = new HashMap<>(); + for (var entry : matchMap.entrySet()){ + try { + var funcDefOpt = getFunctionSignature(entry.getValue()); + funcDefOpt.ifPresent(funcDef -> result.put(entry.getKey(), funcDef)); + } catch (DataTypeDependencyException e) { + Msg.error(this, "Could not parse signature for function %s".formatted(entry.getKey().functionMatch()), e); + } + } + return result; + } public CompletableFuture> searchCollectionsWithIds(String query, String modelName) { @@ -893,19 +1126,176 @@ public CompletableFuture> searchBinariesWithIds(String quer }); } - public Basic getBasicDetailsForAnalysis(AnalysisID analysisID) throws ApiException { + public Basic getBasicDetailsForAnalysis(TypedApiInterface.AnalysisID analysisID) throws ApiException { return api.getAnalysisBasicInfo(analysisID); } - public FunctionMatchingBatchResponse getFunctionMatchingForAnalysis(AnalysisID analysisID, AnalysisFunctionMatchingRequest request) throws ApiException { + public FunctionMatchingResponse getFunctionMatchingForAnalysis(TypedApiInterface.AnalysisID analysisID, AnalysisFunctionMatchingRequest request) throws ApiException { return api.analysisFunctionMatching(analysisID, request); } - public FunctionMatchingBatchResponse getFunctionMatchingForFunction(FunctionMatchingRequest request) throws ApiException { + public FunctionMatchingResponse getFunctionMatchingForFunction(FunctionMatchingRequest request) throws ApiException { return api.functionFunctionMatching(request); } - public void batchRenameFunctions(FunctionsListRename functionsList) throws ApiException { - api.batchRenameFunctions(functionsList); + public void batchRenamingGhidraMatchesWithSignatures(List functionsList) throws ApiException { + // Pushing types to the portal is not supported yet, so we just extract the function matches and call the other method + batchRenameMatches(functionsList.stream() + .map(GhidraFunctionMatchWithSignature::functionMatch) + .toList()); + + } + + public void batchRenameGhidraMatches(List functionsList) throws ApiException { + var matches = functionsList.stream() + .map(GhidraFunctionMatch::functionMatch) + .toList(); + batchRenameMatches(matches); } + + public void batchRenameMatches(List functionsList) throws ApiException { + var matches = functionsList.stream() + .map(result -> { + var func = new FunctionRenameMap(); + func.setFunctionId(result.origin_function_id().value()); + func.setNewName(result.nearest_neighbor_function_name()); + func.setNewMangledName(result.nearest_neighbor_mangled_function_name()); + return func; + }) + .toList(); + + var functionsListRename = new FunctionsListRename(); + functionsListRename.setFunctions(matches); + + api.batchRenameFunctions(functionsListRename); + } + + + public TypedApiInterface.TypedAutoUnstripResponse autoUnstrip(AnalysedProgram program) throws ApiException { + return api.autoUnstrip(program.analysisID); + } + + + /// Old Helper Datatype that encapsulates a Ghidra program with a binary ID and analysis ID + /// This only guarantees that the program has an associated analysis, but not that the analysis is finished + /// The id of this should also be stored in the program options, but this is currently not enforced yet to allow easier testing + public record ProgramWithID( + Program program, + TypedApiInterface.AnalysisID analysisID + ){} + + + + + /// All functions that require a program to have a finished analysis on the portal can use this to encode this assumption into the type system + /// This guarantees that Ghidra Functions that exist on the server can be mapped to Function IDs + /// This rules out the two cases: + /// * the program has no associated analysis on the server + /// * the program has an associated analysis, but analysis hasn't finished yet + public static class AnalysedProgram { + + private final Program program; + private final TypedApiInterface.AnalysisID analysisID; + + + /// The constructor is private to enforce that only the GhidraRevengService class + /// can create instances of this class, ensuring the guarantees hold + private AnalysedProgram( + Program program, + TypedApiInterface.AnalysisID analysisID + ) { + this.program = program; + this.analysisID = analysisID; + + } + + public Program program() { + return program; + } + + public TypedApiInterface.AnalysisID analysisID() { + return analysisID; + } + + private LongPropertyMap getFunctionIDPropertyMap(AnalysedProgram program){ + var map = program.program().getUsrPropertyManager().getLongPropertyMap(REAI_FUNCTION_PROP_MAP); + if (map == null){ + throw new IllegalStateException("Function ID property map not found for supposedly known program %s".formatted(program.program().getName())); + } + return map; + } + + /// Only returns Optional.Empty if the function is not known on the server (e.g. because it's a thunk) + public Optional getIDForFunction(Function function) { + if (function == null) { + Msg.error(AnalysedProgram.class, "Function provided to getIDForFunction is null"); + return Optional.empty(); + } + if (function.getProgram() != this.program){ + throw new IllegalArgumentException("Function %s does not belong to program %s".formatted(function, this.program.getName())); + } + LongPropertyMap functionIDMap = getFunctionIDPropertyMap(this); + var rawId = functionIDMap.get(function.getEntryPoint()); + return Optional + .ofNullable(rawId) + .map(TypedApiInterface.FunctionID::new) + .map( + functionID -> new FunctionWithID(function, functionID) + ); + } + + public BiMap getFunctionMap(){ + var propMap = getFunctionIDPropertyMap(this); + + BiMap functionMap = HashBiMap.create(); + propMap.getPropertyIterator().forEachRemaining( + addr -> { + var func = program.getFunctionManager().getFunctionAt(addr); + + try { + functionMap.put(new TypedApiInterface.FunctionID(propMap.getLong(addr)), func); + } catch (NoValueException e) { + // This should never happen, because we're iterating over the keys + throw new RuntimeException(e); + } + } + ); + return functionMap; + } + + + public Optional getFunctionForID(TypedApiInterface.FunctionID functionID) { + throw new UnsupportedOperationException("Not implemented yet"); + } + + public void setMangledNameForFunction(Function function, String mangledName) { + if (function.getProgram() != this.program){ + throw new IllegalArgumentException("Function %s does not belong to program %s".formatted(function, this.program.getName())); + } + StringPropertyMap mangledNameMap = this.program.getUsrPropertyManager().getStringPropertyMap(REAI_FUNCTION_MANGLED_MAP); + if (mangledNameMap == null){ + throw new IllegalStateException("Mangled name property map not found for supposedly known program %s".formatted(this.program.getName())); + } + mangledNameMap.add(function.getEntryPoint(), mangledName); + } + + public String getMangledNameForFunction(Function function) { + if (function.getProgram() != this.program){ + throw new IllegalArgumentException("Function %s does not belong to program %s".formatted(function, this.program.getName())); + } + StringPropertyMap mangledNameMap = this.program.getUsrPropertyManager().getStringPropertyMap(REAI_FUNCTION_MANGLED_MAP); + if (mangledNameMap == null){ + throw new IllegalStateException("Mangled name property map not found for supposedly known program %s".formatted(this.program.getName())); + } + return mangledNameMap.getString(function.getEntryPoint()); + } + + + } + + /// Holding this object serves as the proof that a Function has an associated FunctionID + public static record FunctionWithID( + Function function, + TypedApiInterface.FunctionID functionID + ) {} } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java index 660b8125..b8fc0aeb 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiImplementation.java @@ -3,8 +3,8 @@ import ai.reveng.api.*; import ai.reveng.model.*; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.services.api.types.AutoUnstripResponse; import ai.reveng.toolkit.ghidra.core.services.api.types.Collection; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; import ai.reveng.toolkit.ghidra.core.services.api.types.LegacyCollection; import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIAuthenticationException; import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIConflictException; @@ -39,15 +39,12 @@ import static ai.reveng.toolkit.ghidra.core.services.api.Utils.mapJSONArray; import static java.net.http.HttpClient.Version.HTTP_1_1; -/** - * The main implementation of the RevEng HTTP API - * Design notes: - * - every method should correspond to a single API endpoint - * - every method should simply execute the request and return the response - * - i.e. no smart checks relying on other API calls to check if e.g. a binary has already been uploaded - * - * - */ +/// The main implementation of the RevEng HTTP API +/// It partially relies on the old manual implementation, but should be migrated to the OpenAPI generated client over time +/// Design notes: +/// - every method should correspond to a single API endpoint +/// - every method should simply execute the request and return the response +/// - i.e. no smart checks relying on other API calls to check if e.g. a binary has already been uploaded public class TypedApiImplementation implements TypedApiInterface { private final HttpClient httpClient; private final String baseUrl; @@ -60,8 +57,10 @@ public class TypedApiImplementation implements TypedApiInterface { private final FunctionsCoreApi functionsCoreApi; private final FunctionsRenamingHistoryApi functionsRenamingHistoryApi; private final FunctionsAiDecompilationApi functionsAiDecompilationApi; + private final FunctionsDataTypesApi functionsDataTypesApi; // Cache for binary ID to analysis ID mappings + @Deprecated private final Map binaryToAnalysisCache = new HashMap<>(); // Cache for analysis basic info to avoid repeated API calls @@ -101,6 +100,7 @@ public TypedApiImplementation(String baseUrl, String apiKey) { this.functionsCoreApi = new FunctionsCoreApi(apiClient); this.functionsRenamingHistoryApi = new FunctionsRenamingHistoryApi(apiClient); this.functionsAiDecompilationApi = new FunctionsAiDecompilationApi(apiClient); + this.functionsDataTypesApi = new FunctionsDataTypesApi(apiClient); this.baseUrl = baseUrl + "/"; this.httpClient = HttpClient.newBuilder() @@ -139,6 +139,7 @@ public BinaryHash upload(Path binPath) throws FileNotFoundException, ApiExceptio Not all parameters are required, for example /search?search=sha_256_hash: only searches for binaries and collection with hashes like . */ + @Deprecated public List search(BinaryHash hash) { Map params = new HashMap<>(); params.put("sha256_hash", hash.sha256()); @@ -195,13 +196,13 @@ private JSONObject sendRequest(HttpRequest request) throws APIAuthenticationExce } @Override - public BinaryID analyse(AnalysisOptionsBuilder builder) throws ApiException { - var analysisRequest = builder.toAnalysisCreateRequest(); - var result = this.analysisCoreApi.createAnalysis(analysisRequest); - - return new BinaryID(result.getData().getBinaryId()); + public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + var analysisRequest = options.toAnalysisCreateRequest(); + var result = this.analysisCoreApi.createAnalysis(analysisRequest, null); + return new AnalysisID(result.getData().getAnalysisId()); } + @Deprecated @Override public AnalysisStatus status(BinaryID binaryID) throws ApiException { var analysisID = this.getAnalysisIDfromBinaryID(binaryID); @@ -212,18 +213,20 @@ public AnalysisStatus status(BinaryID binaryID) throws ApiException { } @Override - public AnalysisStatus status(AnalysisID analysisID) { - var request = requestBuilderForEndpoint("analyses/%s/status".formatted(analysisID.id())) - .GET() - .build(); - return AnalysisStatus.valueOf(sendVersion2Request(request).getJsonData().getString("analysis_status")); + public AnalysisStatus status(AnalysisID analysisID) throws ApiException { + var status = analysisCoreApi.getAnalysisStatus(analysisID.id()); + return AnalysisStatus.valueOf(status.getData().getAnalysisStatus()); } @Override - public List getFunctionInfo(BinaryID binaryID) throws ApiException { - var analysisID = this.getAnalysisIDfromBinaryID(binaryID); + public List getFunctionInfo(AnalysisID analysisID) { - var response = this.analysesResultsMetadataApi.getFunctionsList(analysisID.id(), null, null, null); + BaseResponseAnalysisFunctions response = null; + try { + response = this.analysesResultsMetadataApi.getFunctionsList(analysisID.id(), null, null, null); + } catch (ApiException e) { + throw new RuntimeException("Could not find analysis with ID: " + analysisID.id(), e); + } return response.getData().getFunctions().stream().map(f -> ( new FunctionInfo( @@ -340,6 +343,7 @@ private HttpRequest.Builder requestBuilderForEndpoint(String... endpointPaths){ * @return the analysis id */ @Override + @Deprecated public AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID){ // Check cache first AnalysisID cachedResult = binaryToAnalysisCache.get(binaryID); @@ -384,7 +388,6 @@ public DataTypeList generateFunctionDataTypes(AnalysisID analysisID, List functionIDS) { String queryString = functionIDS.stream().map( f -> "function_ids=" + f.value() ).reduce((a, b) -> a + "&" + b).orElseThrow(); - var request = requestBuilderForEndpoint("functions", "data_types?", queryString) .GET() .header("Content-Type", "application/json" ) @@ -393,6 +396,33 @@ public DataTypeList getFunctionDataTypes(List functionIDS) { return DataTypeList.fromJson(response.getJsonData()); } + public FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID id, List ids) { + try { + List functionIds = null; + if (ids == null) { + functionIds = null; + } else { + functionIds = ids.stream().map(FunctionID::value).map(Long::intValue).toList(); + } + var r = functionsDataTypesApi.listFunctionDataTypesForAnalysis(id.id(), functionIds); + var data = r.getData(); + return data; + } catch (ApiException e) { + throw new RuntimeException(e); + } + } + + @Override + public FunctionDataTypesList listFunctionDataTypesForFunctions(List functionIDs) { + try { + var r = functionsDataTypesApi.listFunctionDataTypesForFunctions(functionIDs.stream().map(FunctionID::value).map(Long::intValue).toList()); + var data = r.getData(); + return data; + } catch (ApiException e) { + throw new RuntimeException(e); + } + } + @Override public Optional getFunctionDataTypes(AnalysisID analysisID, FunctionID functionID) { // https://api.reveng.ai/v2/analyses/{analysis_id}/info/functions/{function_id}/data_types @@ -505,11 +535,19 @@ public List getNameScores(List matches, Boolea */ @Override public AnalysisResult getInfoForAnalysis(AnalysisID id) { - var request = requestBuilderForEndpoint("analyses", String.valueOf(id.id())) - .GET() - .build(); - var response = sendVersion2Request(request); - return AnalysisResult.fromJSONObject(this, response.getJsonData()); + try { + var response = analysisCoreApi.getAnalysisBasicInfo(id.id()); + var data = response.getData(); + if (data == null) { + throw new RuntimeException("Unexpected null data for analysis ID: " + id.id()); + } + return new AnalysisResult( + id, + data + ); + } catch (ApiException e) { + throw new IllegalArgumentException("Could not find analysis with ID: " + id.id()); + } } /** @@ -519,40 +557,32 @@ public AnalysisResult getInfoForAnalysis(AnalysisID id) { */ @Override public FunctionDetails getFunctionDetails(FunctionID id) { - var request = requestBuilderForEndpoint("functions", String.valueOf(id.value())) - .GET() - .build(); - var response = sendVersion2Request(request); - return FunctionDetails.fromJSON(response.getJsonData()); + BaseResponseFunctionsDetailResponse dets = null; + try { + dets = functionsCoreApi.getFunctionDetails((int) id.value()); + } catch (ApiException e) { + throw new RuntimeException(e); + } + return FunctionDetails.fromServerResponse(dets.getData()); + } @Override - public AutoUnstripResponse autoUnstrip(AnalysisID analysisID) { - JSONObject params = new JSONObject(); - params.put("apply", true); - params.put("min_similarity", 90); // 90% - params.put("confidence_threshold", 90); // 90% - params.put("min_group_size", 1); // At least 1 function in the group - - var request = requestBuilderForEndpoint("analyses", String.valueOf(analysisID.id()), "functions", "auto-unstrip") - .POST(HttpRequest.BodyPublishers.ofString(params.toString())) - .header("Content-Type", "application/json" ) - .build(); - - return AutoUnstripResponse.fromJSONObject(sendRequest(request)); + public TypedAutoUnstripResponse autoUnstrip(AnalysisID analysisID) { + try { + return new TypedAutoUnstripResponse(functionsCoreApi.autoUnstrip(analysisID.id(), new AutoUnstripRequest())); + } catch (ApiException e) { + throw new RuntimeException(e); + } } @Override - public AutoUnstripResponse aiUnstrip(AnalysisID analysisID) { - JSONObject params = new JSONObject(); - params.put("apply", true); - - var request = requestBuilderForEndpoint("analyses", String.valueOf(analysisID.id()), "functions", "ai-unstrip") - .POST(HttpRequest.BodyPublishers.ofString(params.toString())) - .header("Content-Type", "application/json" ) - .build(); - - return AutoUnstripResponse.fromJSONObject(sendRequest(request)); + public TypedAutoUnstripResponse aiUnstrip(AnalysisID analysisID) { + try { + return new TypedAutoUnstripResponse(functionsCoreApi.aiUnstrip(analysisID.id(), new AiUnstripRequest())); + } catch (ApiException e) { + throw new RuntimeException(e); + } } @Override @@ -596,12 +626,12 @@ public ai.reveng.model.Basic getAnalysisBasicInfo(AnalysisID analysisID) throws } @Override - public FunctionMatchingBatchResponse analysisFunctionMatching(AnalysisID analysisID, AnalysisFunctionMatchingRequest request) throws ApiException { + public FunctionMatchingResponse analysisFunctionMatching(AnalysisID analysisID, AnalysisFunctionMatchingRequest request) throws ApiException { return this.functionsCoreApi.analysisFunctionMatching(analysisID.id(), request); } @Override - public FunctionMatchingBatchResponse functionFunctionMatching(FunctionMatchingRequest request) throws ApiException { + public FunctionMatchingResponse functionFunctionMatching(FunctionMatchingRequest request) throws ApiException { return this.functionsCoreApi.batchFunctionMatching(request); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java index d7cae31b..e7b1250f 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/TypedApiInterface.java @@ -3,12 +3,12 @@ import java.io.FileNotFoundException; import java.nio.file.Path; import java.util.List; -import java.util.Map; import java.util.Optional; import ai.reveng.model.*; +import ai.reveng.model.AutoUnstripResponse; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.services.api.types.AutoUnstripResponse; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionMatch; import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.InvalidAPIInfoException; import javax.annotation.Nullable; @@ -22,33 +22,45 @@ * * It aims to stick close to the API functions themselves. * E.g. if a feature is implemented via two API calls, it should be implemented as two methods here. - * + * "Typed" refers to using special types for IDs like {@link AnalysisID} and {@link FunctionID}, rather than raw integers or strings. * Wrapping this feature into one conceptual method should then happen inside the {@link ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService} - * + * This exists as an interface so tests can mock it out more easily. * */ public interface TypedApiInterface { - // Analysis - BinaryID analyse(AnalysisOptionsBuilder binHash) throws ApiException; + /// Data type to represent the RevEng.AI API concept of a function ID + record FunctionID(long value){} + /// This is a special box type for an analysis ID + /// It enforces that the integer is specifically an analysis ID, + /// and it implies that the user has (at least read) access to this ID + record AnalysisID(int id) {} + // TODO: could add a box type for an analysis that the user has _write_ access to - default Object delete(BinaryID binID) { - throw new UnsupportedOperationException("delete not implemented yet"); - } + record CollectionID(int id) {} + /// Data type for all reveng API responses or parameters that are a binary hash (as returned by the upload method) + /// The existence of a BinaryHash implies that there is a binary with this hash on the server + record BinaryHash(String sha256) {} + + default AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + throw new UnsupportedOperationException("analyse not implemented yet"); + } - default AnalysisStatus status(AnalysisID analysisID) { + default AnalysisStatus status(AnalysisID analysisID) throws ApiException { throw new UnsupportedOperationException("status not implemented yet"); } - default List getFunctionInfo(BinaryID binaryID) throws ApiException { + default List getFunctionInfo(AnalysisID analysisID) { throw new UnsupportedOperationException("getFunctionInfo not implemented yet"); } - default List recentAnalyses() { - throw new UnsupportedOperationException("recentAnalyses not implemented yet"); + @Deprecated + default List getFunctionInfo(BinaryID binID) throws ApiException { + return getFunctionInfo(getAnalysisIDfromBinaryID(binID)); } + @Deprecated default AnalysisStatus status(BinaryID binID) throws ApiException { throw new UnsupportedOperationException("status not implemented yet"); }; @@ -56,6 +68,7 @@ default AnalysisStatus status(BinaryID binID) throws ApiException { /** * https://docs.reveng.ai/#/Utility/get_search */ + @Deprecated default List search(BinaryHash hash) { throw new UnsupportedOperationException("search not implemented yet"); } @@ -65,6 +78,19 @@ default BinaryHash upload(Path binPath) throws FileNotFoundException, ai.reveng. throw new UnsupportedOperationException("upload not implemented yet"); } + + /** + * Special filters for the collection search endpoint + * https://api.reveng.ai/v2/docs#tag/Collections/operation/list_collections_v2_collections_get + */ + enum SearchFilter { + official_only, + user_only, + team_only, + public_only, + hide_empty + } + default List searchCollections(String searchTerm, @Nullable List filter, int limit, @@ -97,7 +123,19 @@ default Optional getFunctionDataTypes(AnalysisID analysi throw new UnsupportedOperationException("getFunctionDataTypes not implemented yet"); } + default FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID) { + return listFunctionDataTypesForAnalysis(analysisID, null); + } + + default FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID, @Nullable List ids) { + throw new UnsupportedOperationException("listFunctionDataTypesForAnalysis not implemented yet"); + } + + default FunctionDataTypesList listFunctionDataTypesForFunctions(List functionIDs) { + throw new UnsupportedOperationException("listFunctionDataTypesForFunctions not implemented yet"); + } + @Deprecated default AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { throw new UnsupportedOperationException("getAnalysisIDfromBinaryID not implemented yet"); } @@ -132,11 +170,22 @@ default FunctionDetails getFunctionDetails(FunctionID id) { throw new UnsupportedOperationException("getFunctionInfo not implemented yet"); } - default AutoUnstripResponse autoUnstrip(AnalysisID analysisID) { + /// + /// Typed Box Placeholder for {@link AutoUnstripResponse} + record TypedAutoUnstripResponse( + AutoUnstripResponse autoUnstripResponse + ) { + } + + /// {@link MatchedFunctionSuggestion} + record TypedAutoUnstripMatch(MatchedFunctionSuggestion suggestedFunction) { } + + + default TypedAutoUnstripResponse autoUnstrip(AnalysisID analysisID) { throw new UnsupportedOperationException("autoUnstrip not implemented yet"); } - default AutoUnstripResponse aiUnstrip(AnalysisID analysisID) { + default TypedAutoUnstripResponse aiUnstrip(AnalysisID analysisID) { throw new UnsupportedOperationException("aiUnstrip not implemented yet"); } @@ -156,11 +205,11 @@ default ai.reveng.model.Basic getAnalysisBasicInfo(AnalysisID analysisID) throws throw new UnsupportedOperationException("getAnalysisBasicInfo not implemented yet"); } - default FunctionMatchingBatchResponse analysisFunctionMatching(AnalysisID analysisID, AnalysisFunctionMatchingRequest request) throws ApiException { + default FunctionMatchingResponse analysisFunctionMatching(AnalysisID analysisID, AnalysisFunctionMatchingRequest request) throws ApiException { throw new UnsupportedOperationException("analysisFunctionMatching not implemented yet"); } - default FunctionMatchingBatchResponse functionFunctionMatching(FunctionMatchingRequest request) throws ApiException { + default FunctionMatchingResponse functionFunctionMatching(FunctionMatchingRequest request) throws ApiException { throw new UnsupportedOperationException("functionFunctionMatching not implemented yet"); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/MockApi.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/MockApi.java index ceb8e298..90f1bf56 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/MockApi.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/MockApi.java @@ -1,20 +1,16 @@ package ai.reveng.toolkit.ghidra.core.services.api.mocks; -import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; -import ai.reveng.toolkit.ghidra.core.services.api.ModelName; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIAuthenticationException; import org.json.JSONObject; -import javax.annotation.Nullable; import java.io.FileNotFoundException; -import java.math.BigDecimal; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.Map; +@Deprecated public class MockApi implements TypedApiInterface { @Override public BinaryHash upload(Path binPath) throws FileNotFoundException { @@ -22,16 +18,7 @@ public BinaryHash upload(Path binPath) throws FileNotFoundException { } @Override - public Object delete(BinaryID binID) { - return TypedApiInterface.super.delete(binID); - } - - @Override - public List recentAnalyses() { - return TypedApiInterface.super.recentAnalyses(); - } - - @Override + @Deprecated public List search(BinaryHash hash) { if (hash.equals(new BinaryHash("b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6"))) { return List.of(new LegacyAnalysisResult( @@ -55,11 +42,6 @@ public AnalysisStatus status(BinaryID binID) { return AnalysisStatus.Complete; } - @Override - public BinaryID analyse(AnalysisOptionsBuilder binHash) { - return new BinaryID(17920); - } - @Override public String getAnalysisLogs(AnalysisID analysisID) { return ""; @@ -75,7 +57,7 @@ public void renameFunction(FunctionID id, String newName) { } @Override - public List getFunctionInfo(BinaryID binaryID) { + public List getFunctionInfo(AnalysisID analysisID) { var r = """ { "success": true, diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/ProcessingLimboApi.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/ProcessingLimboApi.java deleted file mode 100644 index 8d9a66fe..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/ProcessingLimboApi.java +++ /dev/null @@ -1,35 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.mocks; - -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; - -/** - * An API mock to simulate a server that is never finished with processing a binary. - */ -public class ProcessingLimboApi extends UnimplementedAPI { - - private final AnalysisStatus status; - - private int logCounter = 0; - - public ProcessingLimboApi() { - super(); - status = AnalysisStatus.Processing; - } - - public ProcessingLimboApi(AnalysisStatus status) { - super(); - this.status = status; - } - - @Override - public AnalysisStatus status(BinaryID binID) { - return status; - } - - @Override - public String getAnalysisLogs(AnalysisID analysisID) { - return "Analysis Logs: " + logCounter++; - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/SimpleMatchesApi.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/SimpleMatchesApi.java deleted file mode 100644 index aaa5466e..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/SimpleMatchesApi.java +++ /dev/null @@ -1,15 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.mocks; - -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; - -/** - * An API mock to simulate a server that is never finished with processing a binary. - */ -public class SimpleMatchesApi extends UnimplementedAPI { - @Override - public AnalysisStatus status(BinaryID binID) { - return AnalysisStatus.Complete; - } - -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/TypeGenerationMock.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/TypeGenerationMock.java deleted file mode 100644 index 7d74c75c..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/TypeGenerationMock.java +++ /dev/null @@ -1,65 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.mocks; - -import ai.reveng.toolkit.ghidra.core.services.api.types.*; - -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; - -public class TypeGenerationMock extends UnimplementedAPI { - - Set generatedFunctions = new HashSet<>(); - @Override - public DataTypeList generateFunctionDataTypes(AnalysisID analysisID, List functionIDS) { - var statuses = functionIDS.stream() - .map(id -> new FunctionDataTypeStatus( - false, - Optional.empty(), - "UNKNOWN", - null, - id - )) - .toList(); - return new DataTypeList( - functionIDS.size(), 0, statuses.toArray(new FunctionDataTypeStatus[0]) - ); - } - - @Override - public DataTypeList getFunctionDataTypes(List functionIDS) { - for (FunctionID functionID : functionIDS) { - if (generatedFunctions.contains(functionID)) continue; - generatedFunctions.add(functionID); - break; - } - - var statuses = functionIDS.stream() - .map(id -> new FunctionDataTypeStatus( - generatedFunctions.contains(id), - Optional.empty(), - generatedFunctions.contains(id) ? "completed" : "UNKNOWN", - null, - id - )) - .toList(); - - return new DataTypeList( - functionIDS.size(), 0, statuses.toArray(new FunctionDataTypeStatus[0]) - ); - } - - @Override - public FunctionDetails getFunctionDetails(FunctionID id) { - return new FunctionDetails( - id, - "placeholder_for_%s".formatted(id), - 0L, - 10L, - new AnalysisID(1337), - new BinaryID(1337), - "placeholder_for_%s".formatted(id), - new BinaryHash("placeholder_for_%s".formatted(id)) - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java index cff35217..6a8c3394 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/mocks/UnimplementedAPI.java @@ -1,12 +1,10 @@ package ai.reveng.toolkit.ghidra.core.services.api.mocks; -import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; -import ai.reveng.toolkit.ghidra.core.services.api.ModelName; +import ai.reveng.model.FunctionDataTypesList; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.InvalidAPIInfoException; +import org.jetbrains.annotations.Nullable; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -14,7 +12,6 @@ import java.security.NoSuchAlgorithmException; import java.util.HexFormat; import java.util.List; -import java.util.Map; import java.util.Objects; public class UnimplementedAPI implements TypedApiInterface { @@ -28,11 +25,6 @@ protected AnalysisStatus getNextStatus(AnalysisStatus previousStatus) { }; } - @Override - public BinaryID analyse(AnalysisOptionsBuilder binHash) { - return new BinaryID(1337); - } - @Override public String getAnalysisLogs(AnalysisID analysisID) { return "ANALYSIS LOGS"; @@ -62,8 +54,10 @@ public BinaryHash upload(Path binPath) { } + /// This gets called when registering the initial mock analysis + /// it just pretends that there is no type info available @Override - public AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { - return new AnalysisID(1337); + public FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID, @Nullable List ids) { + return new FunctionDataTypesList(); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisID.java deleted file mode 100644 index 9441dcec..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisID.java +++ /dev/null @@ -1,4 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -public record AnalysisID(int id) { -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java index d7eb50eb..639c490e 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AnalysisResult.java @@ -1,33 +1,20 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.model.Basic; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; -import org.json.JSONObject; -/* - */ +/// This is a remnant of an older class that contained the analysis result data directly. +/// Now it's a wrapper around the generated Basic class with some shim methods for convenience. public record AnalysisResult( - AnalysisID analysisID, - String binary_name, - String creation, - Integer model_id, - String model_name, - BinaryHash sha_256_hash, - AnalysisStatus status -// AnalysisScope analysis_scope, + TypedApiInterface.AnalysisID analysisID, + Basic base_response_basic ) { - public static AnalysisResult fromJSONObject(TypedApiInterface api, JSONObject json) { -// throw new UnsupportedOperationException("fromJSONObject not implemented yet"); - var analysisId = new AnalysisID(json.getInt("analysis_id")); - var analysisStatus = api.status(analysisId); - return new AnalysisResult( - analysisId, - json.getString("binary_name"), - json.getString("creation"), - json.has("model_id") ? json.getInt("model_id") : null, - json.getString("model_name"), - new BinaryHash(json.getString("sha_256_hash")), - analysisStatus - ); + public TypedApiInterface.BinaryHash sha_256_hash() { + return new TypedApiInterface.BinaryHash(base_response_basic().getSha256Hash()); + } + + public String binary_name() { + return base_response_basic.getBinaryName(); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AutoUnstripResponse.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AutoUnstripResponse.java deleted file mode 100644 index 410c140d..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/AutoUnstripResponse.java +++ /dev/null @@ -1,73 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - - -import org.json.JSONArray; -import org.json.JSONObject; -import java.util.List; -import java.util.ArrayList; - -/** - * Structured response for the auto unstrip endpoint - { - "progress": 0, - "status": "string", - "total_time": 0, - "matches": [ - { - "function_id": 0, - "function_vaddr": 0, - "suggested_name": "string" - } - ], - "applied": true, - "error_message": "string" - } - * - */ -public record AutoUnstripResponse( - int progress, - String status, - int total_time, - List matches, - boolean applied, - String error_message -) { - - /** - * Represents a single match in the auto unstrip response - */ - public record Match( - FunctionID function_id, - long function_vaddr, - String suggested_name, - String suggested_demangled_name - ) { - public static Match fromJSONObject(JSONObject json) { - return new Match( - new FunctionID(json.getInt("function_id")), - json.getLong("function_vaddr"), - json.getString("suggested_name"), - json.getString("suggested_demangled_name") - ); - } - } - - public static AutoUnstripResponse fromJSONObject(JSONObject json) { - List matches = new ArrayList<>(); - if (!json.isNull("matches")) { - JSONArray matchesArray = json.getJSONArray("matches"); - for (int i = 0; i < matchesArray.length(); i++) { - matches.add(Match.fromJSONObject(matchesArray.getJSONObject(i))); - } - } - - return new AutoUnstripResponse( - json.getInt("progress"), - json.getString("status"), - json.getInt("total_time"), - matches, - json.getBoolean("applied"), - !json.isNull("error_message") ? json.getString("error_message") : null - ); - } -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryHash.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryHash.java deleted file mode 100644 index 43aaeecf..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryHash.java +++ /dev/null @@ -1,20 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -import org.json.JSONObject; - -/* - * Data type for all reveng API responses or parameters that are a binary hash (as returned by the upload method) - * The existence of a BinaryHash implies that there is a binary with this hash on the server! - * - * This could later be enforced via package private methods - * - */ -public record BinaryHash(String sha256) { - public static BinaryHash fromJsonString(String json) { - return new BinaryHash(new JSONObject(json).getString("sha_256_hash")); - } - public static BinaryHash fromJSONObject(JSONObject json) { - return new BinaryHash(json.getString("sha_256_hash")); - } - -} \ No newline at end of file diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryID.java index af198920..17a6174b 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryID.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/BinaryID.java @@ -6,6 +6,7 @@ * They are called binary ID in the API doc, but they should be thought of as _analysis_ ids * for a single binary (identified by hash), there can be multiple analyses, which are distinguished by this ID */ +@Deprecated public record BinaryID(int value) implements Comparable { public BinaryID { if (value < 0) { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/Collection.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/Collection.java index 56553567..4225621d 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/Collection.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/Collection.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; import java.util.List; @@ -18,7 +19,7 @@ * @param tags */ public record Collection( - CollectionID collectionID, + TypedApiInterface.CollectionID collectionID, String collectionName, String description, Integer modelID, @@ -26,11 +27,11 @@ public record Collection( String collectionScope, String creationDate, List tags, - List binaries + List binaries ) { public static Collection fromJSONObject(JSONObject json){ return new Collection( - new CollectionID(json.getInt("collection_id")), + new TypedApiInterface.CollectionID(json.getInt("collection_id")), json.getString("collection_name"), json.getString("description"), json.getInt("model_id"), @@ -38,7 +39,7 @@ public static Collection fromJSONObject(JSONObject json){ json.getString("collection_scope"), json.getString("created_at"), json.has("tags") ? json.getJSONArray("tags").toList().stream().map(Object::toString).toList() : null, - json.has("binaries") ? json.getJSONArray("binaries").toList().stream().map( rawID -> new AnalysisID((Integer) rawID)).toList() : null + json.has("binaries") ? json.getJSONArray("binaries").toList().stream().map( rawID -> new TypedApiInterface.AnalysisID((Integer) rawID)).toList() : null ); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/CollectionID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/CollectionID.java deleted file mode 100644 index 583b556a..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/CollectionID.java +++ /dev/null @@ -1,4 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -public record CollectionID(int id) { -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/DataTypeList.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/DataTypeList.java index 207a656f..a2efa258 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/DataTypeList.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/DataTypeList.java @@ -1,10 +1,14 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.model.FunctionDataTypesList; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; /** * Example in data_types_batch_response.json + * @deprecated Use OpenAPI {@link FunctionDataTypesList} */ +@Deprecated public record DataTypeList( int totalCount, int totalDataTypesCount, @@ -22,7 +26,7 @@ public static DataTypeList fromJson(JSONObject json) { return new DataTypeList(totalCount, totalDataTypesCount, dataTypes); } - public FunctionDataTypeStatus statusForFunction(FunctionID functionID) { + public FunctionDataTypeStatus statusForFunction(TypedApiInterface.FunctionID functionID) { for (FunctionDataTypeStatus status : dataTypes) { if (status.functionID().equals(functionID)) { return status; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDataTypeStatus.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDataTypeStatus.java index eb5c9a3b..e2ca65cf 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDataTypeStatus.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDataTypeStatus.java @@ -1,5 +1,7 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.model.FunctionDataTypes; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.FunctionDataTypeMessage; import org.json.JSONObject; @@ -44,14 +46,16 @@ * }, * "status": "completed" * } + * @deprecated Use {@link FunctionDataTypes} instead */ +@Deprecated public record FunctionDataTypeStatus( boolean completed, Optional data_types, // JSONObject data_types, String status, @Nullable Integer dataTypesVersion, - @Nullable FunctionID functionID + @Nullable TypedApiInterface.FunctionID functionID ) { public static FunctionDataTypeStatus fromJson(JSONObject json) { @@ -67,7 +71,7 @@ public static FunctionDataTypeStatus fromJson(JSONObject json) { !json.isNull("data_types") ? Optional.of(FunctionDataTypeMessage.fromJsonObject(json.getJSONObject("data_types"))) : Optional.empty(), json.getString("status"), dataTypesVersion, - json.has("function_id") ? new FunctionID(json.getInt("function_id")) : null + json.has("function_id") ? new TypedApiInterface.FunctionID(json.getInt("function_id")) : null ); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java index 105ac770..b1c7414c 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionDetails.java @@ -1,31 +1,33 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; -import org.json.JSONObject; +import ai.reveng.model.FunctionsDetailResponse; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; /** * Record representing detailed function information from the RevEng.AI API */ public record FunctionDetails( - FunctionID functionId, - String functionName, + TypedApiInterface.FunctionID functionId, + String mangledFunctionName, Long functionVaddr, Long functionSize, - AnalysisID analysisId, - BinaryID binaryId, + TypedApiInterface.AnalysisID analysisId, String binaryName, - BinaryHash sha256Hash + TypedApiInterface.BinaryHash sha256Hash, + String demangledName ) { - public static FunctionDetails fromJSON(JSONObject json) { + + public static FunctionDetails fromServerResponse(FunctionsDetailResponse response) { return new FunctionDetails( - new FunctionID(json.getInt("function_id")), - json.getString("function_name_mangled"), - json.getLong("function_vaddr"), - json.getLong("function_size"), - new AnalysisID(json.getInt("analysis_id")), - new BinaryID(json.getInt("binary_id")), - json.getString("binary_name"), - new BinaryHash(json.getString("sha_256_hash")) + new TypedApiInterface.FunctionID(response.getFunctionId()), + response.getFunctionNameMangled(), + response.getFunctionVaddr(), + response.getFunctionSize().longValue(), + new TypedApiInterface.AnalysisID(response.getAnalysisId()), + response.getBinaryName(), + new TypedApiInterface.BinaryHash(response.getSha256Hash()), + response.getFunctionName() ); } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionID.java deleted file mode 100644 index 6a03f7c6..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionID.java +++ /dev/null @@ -1,10 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -/** - * Data type to represent the RevEng.AI API concept of a function ID - */ -public record FunctionID( - long value - -) { -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionInfo.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionInfo.java index a8c28eec..8817019f 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionInfo.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionInfo.java @@ -1,9 +1,10 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; public record FunctionInfo( - FunctionID functionID, + TypedApiInterface.FunctionID functionID, String functionName, String functionMangledName, // This is an absolute address @@ -12,7 +13,7 @@ public record FunctionInfo( ) { public static FunctionInfo fromJSONObject(JSONObject json) { return new FunctionInfo( - new FunctionID(json.getInt("function_id")), + new TypedApiInterface.FunctionID(json.getInt("function_id")), json.getString("function_name"), json.getString("function_mangled_name"), json.getLong("function_vaddr"), diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionMatch.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionMatch.java index 171f8ced..1d143518 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionMatch.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionMatch.java @@ -1,39 +1,42 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.model.MatchedFunction; +import ai.reveng.toolkit.ghidra.binarysimilarity.ui.functionmatching.AbstractFunctionMatchingDialog; import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; -/** - * @param origin_function_id - * @param nearest_neighbor_id - * @param nearest_neighbor_function_name - * @param nearest_neighbor_binary_name - * @param nearest_neighbor_sha_256_hash - * @param nearest_neighbor_binary_id - * @param nearest_neighbor_debug - * @param similarity - */ +import javax.annotation.Nullable; +import java.math.BigDecimal; + +/// The Typed Class to represent a FunctionMatch returned by the RevEng.AI API +/// At the very least it needs to contain the origin_function_id and nearest_neighbor_id +/// Other fields may be null depending on the context in which the FunctionMatch was returned +/// They can be derived from the ids via the API if needed +/// For representing the combination of a local Ghidra Function and its FunctionMatch, use {@link GhidraFunctionMatch} public record FunctionMatch( - FunctionID origin_function_id, - FunctionID nearest_neighbor_id, + TypedApiInterface.FunctionID origin_function_id, + TypedApiInterface.FunctionID nearest_neighbor_id, String nearest_neighbor_function_name, + String nearest_neighbor_mangled_function_name, String nearest_neighbor_binary_name, - BinaryHash nearest_neighbor_sha_256_hash, - BinaryID nearest_neighbor_binary_id, + TypedApiInterface.BinaryHash nearest_neighbor_sha_256_hash, Boolean nearest_neighbor_debug, - double similarity + BigDecimal similarity, + BigDecimal confidence + ) { - public static FunctionMatch fromJSONObject(JSONObject json) { + + public static FunctionMatch fromMatchedFunctionAPIType(MatchedFunction matchedFunction, TypedApiInterface.FunctionID originFunctionID) { return new FunctionMatch( - new FunctionID(json.getInt("origin_function_id")), - new FunctionID(json.getInt("nearest_neighbor_id")), - json.getString("nearest_neighbor_function_name"), - json.getString("nearest_neighbor_binary_name"), - new BinaryHash(json.getString("nearest_neighbor_sha_256_hash")), - new BinaryID(json.getInt("nearest_neighbor_binary_id")), - json.has("nearest_neighbor_debug") ? json.getBoolean("nearest_neighbor_debug") : null, - // This is called confidence for legacy reasons, but it is actually the similarity - json.getDouble("confidence") + originFunctionID, + new TypedApiInterface.FunctionID(matchedFunction.getFunctionId()), + matchedFunction.getFunctionName(), + matchedFunction.getMangledName(), + matchedFunction.getBinaryName(), + new TypedApiInterface.BinaryHash(matchedFunction.getSha256Hash()), + matchedFunction.getDebug(), + matchedFunction.getSimilarity(), + matchedFunction.getConfidence() ); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionNameScore.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionNameScore.java index 298c770b..837a5286 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionNameScore.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/FunctionNameScore.java @@ -1,15 +1,16 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; public record FunctionNameScore( - FunctionID functionID, + TypedApiInterface.FunctionID functionID, BoxPlot score ) { public static FunctionNameScore fromJSONObject(JSONObject jsonObject) { var boxplotJson = jsonObject.getJSONObject("box_plot"); return new FunctionNameScore( - new FunctionID(jsonObject.getInt("function_id")), + new TypedApiInterface.FunctionID(jsonObject.getInt("function_id")), BoxPlot.fromJSONObject(boxplotJson) ); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatch.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatch.java index e9fc5ac2..8dfb3fc6 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatch.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatch.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ghidra.program.model.listing.Function; /** @@ -11,20 +12,9 @@ public record GhidraFunctionMatch( Function function, FunctionMatch functionMatch ) { - - public String nearest_neighbor_function_name() { - return functionMatch.nearest_neighbor_function_name(); - } - - public String nearest_neighbor_binary_name() { - return functionMatch.nearest_neighbor_binary_name(); - } - - public FunctionID nearest_neighbor_id() { + // The following methods are just convenience methods to access the FunctionMatch fields + // This simplifies using this class in a stream with method references + public TypedApiInterface.FunctionID nearest_neighbor_id() { return functionMatch.nearest_neighbor_id(); } - public double similarity() { - return functionMatch.similarity(); - } - } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatchWithSignature.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatchWithSignature.java index aed0691d..e31f7a9f 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatchWithSignature.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/GhidraFunctionMatchWithSignature.java @@ -1,69 +1,21 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.FunctionDataTypeMessage; +import ghidra.program.model.data.FunctionDefinitionDataType; import ghidra.program.model.listing.Function; +import javax.annotation.Nullable; import java.util.Optional; /** - * The signature is not final because it can be computed on demand - * * @param function The local function that we searched for matches for * @param functionMatch A match that was found and returned by the RevEng.AI server * @param signature The optional signature of the function match */ -public class GhidraFunctionMatchWithSignature { - private final Function function; - private final FunctionMatch functionMatch; - private Optional signature; - private final Optional nameScore; - - public GhidraFunctionMatchWithSignature( - Function function, - FunctionMatch functionMatch, - Optional signature, - Optional nameScore) { - if (function == null) { - throw new IllegalArgumentException("Function cannot be null"); - } - if (functionMatch == null) { - throw new IllegalArgumentException("FunctionMatch cannot be null"); - } - if (nameScore == null) { - throw new IllegalArgumentException("NameScore cannot be null, use Optional.empty() instead"); - } - this.function = function; - this.functionMatch = functionMatch; - this.signature = signature; - this.nameScore = nameScore; - } - - public GhidraFunctionMatchWithSignature(GhidraFunctionMatch functionMatch, FunctionDataTypeMessage signature, BoxPlot nameScore) { - this(functionMatch.function(), functionMatch.functionMatch(), Optional.ofNullable(signature), Optional.ofNullable(nameScore)); - } - - public Function function() { - return function; - } - - public FunctionMatch functionMatch() { - return functionMatch; - } - - public Optional signature() { - return signature; - } - - public void setSignature(Optional signature) { - this.signature = signature; - } - - public Optional nameScore() { - return nameScore; - } - - public FunctionID nearest_neighbor_id() { - return functionMatch.nearest_neighbor_id(); - } -} \ No newline at end of file +public record GhidraFunctionMatchWithSignature( + Function function, + FunctionMatch functionMatch, + @Nullable FunctionDefinitionDataType signature +){} \ No newline at end of file diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/InvalidBinaryID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/InvalidBinaryID.java deleted file mode 100644 index cc7453da..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/InvalidBinaryID.java +++ /dev/null @@ -1,21 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - - - -/** - * Exception thrown when a binary ID is invalid or not accessible under a certain config - */ -public class InvalidBinaryID extends Exception{ - private final BinaryID binaryID; - private final ApiInfo config; - - public InvalidBinaryID(BinaryID binaryID, ApiInfo config) { - super("Binary ID " + binaryID + " is invalid under config " + config); - this.binaryID = binaryID; - this.config = config; - } - - public BinaryID getBinaryID() { - return binaryID; - } -}; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java index c2583e4a..568c0eb5 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyAnalysisResult.java @@ -1,33 +1,35 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; -/* -{"analyses":[ -{"analysis_scope":"PRIVATE","binary_id":27665,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:57:18 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27664,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:55:28 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27663,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:53:33 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27662,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:32:34 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27661,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:24:44 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27660,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:23:54 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27633,"binary_name":"true","creation":"Thu, 18 Apr 2024 17:34:42 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27632,"binary_name":"true","creation":"Thu, 18 Apr 2024 17:33:36 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27631,"binary_name":"ls","creation":"Thu, 18 Apr 2024 17:17:48 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"2e7ef3da2b295c77820a0782b00c9d607cd48d5c8f7458a76b0921bec20a30ae","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27630,"binary_name":"ls","creation":"Thu, 18 Apr 2024 17:16:48 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"2e7ef3da2b295c77820a0782b00c9d607cd48d5c8f7458a76b0921bec20a30ae","status":"Error"}]} - */ +/// {"analyses":[ +/// {"analysis_scope":"PRIVATE","binary_id":27665,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:57:18 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27664,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:55:28 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27663,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:53:33 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27662,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:32:34 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27661,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:24:44 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27660,"binary_name":"true","creation":"Fri, 19 Apr 2024 08:23:54 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27633,"binary_name":"true","creation":"Thu, 18 Apr 2024 17:34:42 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27632,"binary_name":"true","creation":"Thu, 18 Apr 2024 17:33:36 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"b04c1259718dd16c0ffbd0931aeecf07746775cc2f1cda76e46d51af165f3ba6","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27631,"binary_name":"ls","creation":"Thu, 18 Apr 2024 17:17:48 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"2e7ef3da2b295c77820a0782b00c9d607cd48d5c8f7458a76b0921bec20a30ae","status":"Error"},{"analysis_scope":"PRIVATE","binary_id":27630,"binary_name":"ls","creation":"Thu, 18 Apr 2024 17:16:48 GMT","model_id":1,"model_name":"binnet-0.2-x86-linux","sha_256_hash":"2e7ef3da2b295c77820a0782b00c9d607cd48d5c8f7458a76b0921bec20a30ae","status":"Error"}]} +/// @deprecated Use {@link AnalysisResult)} +@Deprecated public record LegacyAnalysisResult( - AnalysisID analysis_id, + TypedApiInterface.AnalysisID analysis_id, + @Deprecated BinaryID binary_id, String binary_name, String creation, int model_id, String model_name, - BinaryHash sha_256_hash, + TypedApiInterface.BinaryHash sha_256_hash, AnalysisStatus status, long base_address, String function_boundaries_hash ) { public static LegacyAnalysisResult fromJSONObject(JSONObject json) { return new LegacyAnalysisResult( - new AnalysisID(json.getInt("analysis_id")), + new TypedApiInterface.AnalysisID(json.getInt("analysis_id")), new BinaryID(json.getInt("binary_id")), json.getString("binary_name"), json.getString("creation"), json.getInt("model_id"), json.getString("model_name"), - new BinaryHash(json.getString("sha_256_hash")), + new TypedApiInterface.BinaryHash(json.getString("sha_256_hash")), AnalysisStatus.valueOf(json.getString("status")), json.getLong("base_address"), json.getString("function_boundaries_hash") diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyCollection.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyCollection.java index 20dad75c..c2c0ea8b 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyCollection.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/LegacyCollection.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import org.json.JSONObject; import java.util.List; @@ -18,7 +19,7 @@ * @param tags */ public record LegacyCollection( - CollectionID collectionID, + TypedApiInterface.CollectionID collectionID, String collectionScope, String collectionName, String owner, @@ -29,7 +30,7 @@ public record LegacyCollection( ) { public static LegacyCollection fromJSONObject(JSONObject json){ return new LegacyCollection( - new CollectionID(json.getInt("collection_id")), + new TypedApiInterface.CollectionID(json.getInt("collection_id")), json.getString("collection_scope"), json.getString("collection_name"), json.getString("collection_owner"), diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/SearchFilter.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/SearchFilter.java deleted file mode 100644 index a74a97cb..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/SearchFilter.java +++ /dev/null @@ -1,13 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.services.api.types; - -/** - * Special filters for the collection search endpoint - * https://api.reveng.ai/v2/docs#tag/Collections/operation/list_collections_v2_collections_get - */ -public enum SearchFilter { - official_only, - user_only, - team_only, - public_only, - hide_empty -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArtifact.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArtifact.java index 6cb1e6d5..6b105091 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArtifact.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionArtifact.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +import ai.reveng.model.FunctionTypeOutput; import org.json.JSONObject; import java.util.ArrayList; diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDataTypeMessage.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDataTypeMessage.java index 5a84ab76..2225ef05 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDataTypeMessage.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDataTypeMessage.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +import ai.reveng.model.FunctionInfoOutput; import org.json.JSONObject; /** @@ -227,14 +228,17 @@ * * This object isn't part of the BinSync types * The func_deps members are either typedefs or structures + * + * @deprecated see {@link FunctionInfoOutput} */ +@Deprecated public record FunctionDataTypeMessage( FunctionArtifact func_types, FunctionDependencies func_deps ) { public static FunctionDataTypeMessage fromJsonObject(JSONObject dataTypes) { return new FunctionDataTypeMessage( - FunctionArtifact.fromJsonObject(dataTypes.getJSONObject("func_types")), + FunctionArtifact.fromJsonObject(dataTypes. getJSONObject("func_types")), FunctionDependencies.fromJsonObject(dataTypes.getJSONArray("func_deps")) ); diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDependencies.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDependencies.java index 3e751b84..4346766e 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDependencies.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/FunctionDependencies.java @@ -1,16 +1,18 @@ package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +import ai.reveng.model.*; +import ghidra.util.Msg; import org.json.JSONArray; import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Optional; /** * Describes the dependencies of a function. - * Has no corresponding artifact in BinSync, in the RevEng.AI response it's just a list of + * Has no corresponding artifact in BinSync or the RevEng.AI OpenAPI spec. In the RevEng.AI response it's just a list of * type artifacts - * * For ease of use we split it into its own record, and split the type of artifacts into their own * arrays. This needs to happen for deserialization anyway */ @@ -18,6 +20,40 @@ public record FunctionDependencies( Typedef[] typedefs, Struct[] structs ) { + + public static FunctionDependencies fromOpenAPI(List deps) { + if (deps.isEmpty()) { + return null; + } + var typedefs = new ArrayList(); + var structs = new ArrayList(); + + for (var dep : deps) { + var instance = dep.getActualInstance(); + switch (instance) { + case TypeDefinition typedef -> typedefs.add(Typedef.fromOpenAPI(typedef)); + case Structure struct -> structs.add(Struct.fromOpenAPI(struct)); + case Enumeration enumeration -> { + // We don't handle enums for now + } + case GlobalVariable globalVariable -> { + // We don't handle global variables for now + } + default ->{ + Msg.error(FunctionDependencies.class, "Unexpected type dependency: " + instance); + } + + + } + } + + return new FunctionDependencies( + typedefs.toArray(new Typedef[0]), + structs.toArray(new Struct[0]) + ); + + } + public static FunctionDependencies fromJsonObject(JSONArray funcDeps) { // We need to distinguish the object type somehow if (funcDeps.isEmpty()) { diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Struct.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Struct.java index db4ff4be..76101147 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Struct.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Struct.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +import ai.reveng.model.Structure; import org.json.JSONObject; /** @@ -35,4 +36,16 @@ public static boolean matches(JSONObject jsonObject) { } + public static Struct fromOpenAPI(Structure struct) { + StructMember[] members = struct.getMembers().values().stream() + .map(StructMember::fromOpenAPI) + .toList().toArray(new StructMember[0]); + + return new Struct( + struct.getLastChange(), + struct.getName(), + struct.getSize(), + members + ); + } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StructMember.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StructMember.java index faa8ab7b..d89905ba 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StructMember.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/StructMember.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +import ai.reveng.model.StructureMember; import org.json.JSONObject; /** @@ -22,4 +23,14 @@ public static StructMember fromJsonObject(JSONObject jsonObject) { jsonObject.getInt("size") ); } + + public static StructMember fromOpenAPI(StructureMember structureMember) { + return new StructMember( + structureMember.getLastChange(), + structureMember.getName(), + structureMember.getOffset(), + structureMember.getType(), + structureMember.getSize() + ); + } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/TypePathAndName.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/TypePathAndName.java index 9b553307..699c302e 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/TypePathAndName.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/TypePathAndName.java @@ -7,12 +7,28 @@ public record TypePathAndName( String[] path ) { + + + /// based on `ArtifactLifter.parse_scoped_type` from binsync + /// Takes strings like: + /// + /// - "uint32_t" + /// - "stdint::uint32_t" + /// - "DWARF::stdio.h::off_t" + /// @param str + /// @return public static TypePathAndName fromString(String str){ - String[] parts = str.split("/"); - String name = parts[parts.length - 1]; - String[] path = new String[parts.length - 1]; - System.arraycopy(parts, 0, path, 0, parts.length - 1); - return new TypePathAndName(name, path); + // split into path and name on "::" + if (str.contains("::")) { + String[] pathPlusType = str.split("::"); + var baseType = pathPlusType[pathPlusType.length - 1]; + + String[] parts = new String[pathPlusType.length - 1]; + System.arraycopy(pathPlusType, 0, parts, 0, pathPlusType.length - 1); + return new TypePathAndName(baseType, parts); + } else { + return new TypePathAndName(str, new String[0]); + } } public CategoryPath toCategoryPath(){ diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Typedef.java b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Typedef.java index 172a61e0..57671e3f 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Typedef.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/services/api/types/binsync/Typedef.java @@ -1,5 +1,6 @@ package ai.reveng.toolkit.ghidra.core.services.api.types.binsync; +import ai.reveng.model.TypeDefinition; import org.json.JSONObject; import java.util.Set; @@ -26,4 +27,12 @@ public static boolean matches(JSONObject obj) { return obj.keySet().equals(Set.of("last_change", "name", "type")); // return obj.has("type") && obj.has("name"); } + + public static Typedef fromOpenAPI(TypeDefinition typedef) { + return new Typedef( + typedef.getLastChange(), + typedef.getName(), + typedef.getType() + ); + } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/tasks/StartAnalysisTask.java b/src/main/java/ai/reveng/toolkit/ghidra/core/tasks/StartAnalysisTask.java index ad335f9a..8e4d41a5 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/tasks/StartAnalysisTask.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/core/tasks/StartAnalysisTask.java @@ -6,7 +6,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.listing.Program; import ghidra.util.exception.CancelledException; @@ -27,12 +26,6 @@ public class StartAnalysisTask extends Task { private final AnalysisLogConsumer log; private final PluginTool tool; - public ProgramWithBinaryID getProgramWithBinaryID() { - return programWithBinaryID; - } - - private ProgramWithBinaryID programWithBinaryID; - public StartAnalysisTask(Program program, AnalysisOptionsBuilder options, GhidraRevengService reService, @@ -55,8 +48,9 @@ public void run(TaskMonitor monitor) throws CancelledException { monitor.setMessage("Sending Analysis Request"); + GhidraRevengService.ProgramWithID programWithID; try { - programWithBinaryID = reService.startAnalysis(program, options); + programWithID = reService.startAnalysis(program, options); } catch (ApiException e) { monitor.setMessage("Analysis Request Failed"); return; @@ -64,7 +58,7 @@ public void run(TaskMonitor monitor) throws CancelledException { tool.firePluginEvent(new RevEngAIAnalysisStatusChangedEvent( "StartAnalysisTask", - programWithBinaryID, + programWithID, AnalysisStatus.Queued) ); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/core/types/ProgramWithBinaryID.java b/src/main/java/ai/reveng/toolkit/ghidra/core/types/ProgramWithBinaryID.java deleted file mode 100644 index bf642777..00000000 --- a/src/main/java/ai/reveng/toolkit/ghidra/core/types/ProgramWithBinaryID.java +++ /dev/null @@ -1,17 +0,0 @@ -package ai.reveng.toolkit.ghidra.core.types; - -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; -import ghidra.program.model.listing.Program; - - -/** - * Helper Datatype that encapsulates a Ghidra program with a binary ID and analysis ID - * All functions that require a program to be known on the portal can use this to encode this assumption into the type system - */ -public record ProgramWithBinaryID( - Program program, - BinaryID binaryID, - AnalysisID analysisID -) { -} diff --git a/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java b/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java index f5409be8..f5b0ff52 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/devplugin/RevEngMetadataProvider.java @@ -1,10 +1,8 @@ package ai.reveng.toolkit.ghidra.devplugin; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionID; import ghidra.framework.plugintool.ComponentProviderAdapter; import ghidra.framework.plugintool.PluginTool; import ghidra.program.model.listing.Function; @@ -17,9 +15,8 @@ public class RevEngMetadataProvider extends ComponentProviderAdapter { private final JPanel panel; private final JTextField serverField; - BinaryID binaryID; - AnalysisID analysisID; - FunctionID functionID; + TypedApiInterface.AnalysisID analysisID; + TypedApiInterface.FunctionID functionID; Function function; JTextField binaryIDField; @@ -69,7 +66,6 @@ public JComponent getComponent() { } private void clear() { - binaryID = null; analysisID = null; functionID = null; function = null; @@ -82,8 +78,7 @@ public void setProgram(Program program) { var kProg = api.getKnownProgram(program); kProg.ifPresent(p -> { - binaryID = p.binaryID(); - analysisID = api.getApi().getAnalysisIDfromBinaryID(binaryID); + analysisID = p.analysisID(); }); refresh(); @@ -93,7 +88,6 @@ public void setProgram(Program program) { * Updates the component with the latest information */ private void refresh() { - binaryIDField.setText(binaryID == null ? "" : binaryID.toString()); analysisIDField.setText(analysisID == null ? "" : analysisID.toString()); functionIDField.setText(functionID == null ? "" : functionID.toString()); functionField.setText(function == null ? "" : function.toString()); @@ -107,11 +101,11 @@ public void locationChanged(ProgramLocation loc) { var func = loc.getProgram().getFunctionManager().getFunctionContaining(loc.getAddress()); if (func != null) { function = func; - var kProg = revengService.getKnownProgram(loc.getProgram()); + var kProg = revengService.getAnalysedProgram(loc.getProgram()); kProg.ifPresent(p -> { - Optional f = revengService.getFunctionIDFor(p, func); + Optional f = p.getIDForFunction(func); f.ifPresent(functionID -> { - this.functionID = functionID; + this.functionID = functionID.functionID(); }); }); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java index fe7a7f46..2573bf71 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/AnalysisManagementPlugin.java @@ -17,11 +17,9 @@ import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisResultsLoaded; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; -import ai.reveng.toolkit.ghidra.binarysimilarity.ui.about.AboutDialog; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.analysiscreation.RevEngAIAnalysisOptionsDialog; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.misc.AnalysisLogComponent; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.recentanalyses.RecentAnalysisDialog; -import ai.reveng.toolkit.ghidra.binarysimilarity.ui.help.HelpDialog; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.types.*; @@ -29,8 +27,6 @@ import ai.reveng.toolkit.ghidra.core.services.function.export.ExportFunctionBoundariesServiceImpl; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; import ai.reveng.toolkit.ghidra.core.tasks.StartAnalysisTask; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; -import docking.ActionContext; import docking.action.DockingAction; import docking.action.builder.ActionBuilder; import docking.widgets.OptionDialog; @@ -44,6 +40,7 @@ import ghidra.program.util.GhidraProgramUtilities; import ghidra.util.Msg; import ghidra.util.task.TaskBuilder; +import ghidra.util.task.TaskMonitor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -133,10 +130,7 @@ private void setupActions() { // Disable the action if no program is open return false; } - boolean isKnown = revengService.isKnownProgram(currentProgram); - boolean shouldEnable = !isKnown; - - return shouldEnable; + return revengService.getKnownProgram(currentProgram).isEmpty(); }) .onAction(context -> { var program = tool.getService(ProgramManager.class).getCurrentProgram(); @@ -180,8 +174,7 @@ private void setupActions() { if (currentProgram == null) { return false; } - boolean isKnown = revengService.isKnownProgram(currentProgram); - return !isKnown; + return revengService.getKnownProgram(currentProgram).isEmpty(); }) .onAction(context -> { var currentProgram = tool.getService(ProgramManager.class).getCurrentProgram(); @@ -203,12 +196,12 @@ private void setupActions() { // Disable the action if no program is open return false; } - return revengService.isKnownProgram(currentProgram); + return revengService.getKnownProgram(currentProgram).isPresent(); }) .onAction(context -> { var program = tool.getService(ProgramManager.class).getCurrentProgram(); - var analysisID = this.revengService.getAnalysisIDFor(program); - var displayText = analysisID.map(id -> "analysis " + id.id()).get(); + var knownProgram = this.revengService.getKnownProgram(program); + var displayText = knownProgram.map(p -> "analysis " + p.analysisID().id()).orElseThrow(); var result = OptionDialog.showOptionDialogWithCancelAsDefaultButton( tool.getToolFrame(), @@ -238,18 +231,17 @@ private void setupActions() { // Disable the action if no program is open return false; } - return revengService.isKnownProgram(currentProgram); + return revengService.getKnownProgram(currentProgram).isPresent(); }) .onAction(context -> { var currentProgram = tool.getService(ProgramManager.class).getCurrentProgram(); - var binID = revengService.getBinaryIDFor(currentProgram).orElseThrow(); - var analysisID = revengService.getApi().getAnalysisIDfromBinaryID(binID); - var logs = revengService.getAnalysisLog(analysisID); + var knownProgram = revengService.getKnownProgram(currentProgram).orElseThrow(); + var logs = revengService.getAnalysisLog(knownProgram.analysisID()); analysisLogComponent.setLogs(logs); - AnalysisStatus status = revengService.pollStatus(binID); + AnalysisStatus status = revengService.status(knownProgram); tool.getService(ReaiLoggingService.class).info("Check Status: " + status); Msg.showInfo(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Check status", - "Status of analysis " + analysisID.id() + ": " + status); + "Status of analysis " + knownProgram + ": " + status); }) .menuPath(new String[] { ReaiPluginPackage.MENU_GROUP_NAME, "Analysis", "Check status" }) .menuGroup(REAI_ANALYSIS_MANAGEMENT_MENU_GROUP, "400") @@ -262,37 +254,71 @@ private void setupActions() { // Disable the action if no program is open return false; } - return revengService.isKnownProgram(currentProgram); + return revengService.getKnownProgram(currentProgram).isPresent(); }) .onAction(context -> { var currentProgram = tool.getService(ProgramManager.class).getCurrentProgram(); - var binID = revengService.getBinaryIDFor(currentProgram).orElseThrow(); - revengService.openPortal("analyses", String.valueOf(binID.value())); + var knownProgram = revengService.getKnownProgram(currentProgram).orElseThrow(); + revengService.openPortalFor(knownProgram); }) .menuPath(new String[] { ReaiPluginPackage.MENU_GROUP_NAME, "Analysis", "View in portal" }) .menuGroup(REAI_ANALYSIS_MANAGEMENT_MENU_GROUP, "400") .buildAndInstall(tool); } - @Override + @Override + protected void programOpened(Program program) { + super.programOpened(program); + // When a program is opened, we check if it has an associated analysis (not necessarily finished yet) + var knownProgram = revengService.getKnownProgram(program); + if (knownProgram.isPresent()) { + log.info("Opened known program: {}", knownProgram.get()); + var analysedProgram = revengService.getAnalysedProgram(program); + if (analysedProgram.isPresent()) { + // Nothing to do, we already have loaded the function IDs and similar + log.info("Loaded analysed program: {}", analysedProgram); + } else { + // There is an associated program that hasn't been fully loaded yet + // This can happen if the analysis was started in a previous session but hadn't finished when closing Ghidra + // Either the analysis is finished already now, or we want to actively wait for it to finish + log.info("Detected known program that hasn't been fully loaded yet: {}", knownProgram.get()); + var status = revengService.status(knownProgram.get()); + switch (status) { + case Complete -> { + tool.firePluginEvent( + new RevEngAIAnalysisStatusChangedEvent( + "programOpened", + knownProgram.get(), + status)); + } + case Queued, Processing -> { + // This is the same code as above, but it has the implicit assumption that someone + /// Currently this is done by {@link AnalysisLogComponent#processEvent(RevEngAIAnalysisStatusChangedEvent)} + tool.firePluginEvent( + new RevEngAIAnalysisStatusChangedEvent( + "programOpened", + knownProgram.get(), + status)); + } + + + case Error -> { + // The analysis failed on the server side + Msg.showError(this, null, "Analysis Error", + "The RevEng.AI analysis for the program " + knownProgram.get() + " is in error state on the server side."); + } + } + } + } else { + log.info("Opened unknown program: {}", program.getName()); + } + } + + @Override protected void programActivated(Program program) { super.programActivated(program); - - if (!revengService.isKnownProgram(program)){ - var maybeBinID = revengService.getBinaryIDFor(program); - if (maybeBinID.isEmpty()){ - Msg.info(this, "Program has no saved binary ID"); - return; - } - var binID = maybeBinID.get(); - AnalysisStatus status = revengService.pollStatus(binID); - var analysisID = revengService.getApi().getAnalysisIDfromBinaryID(binID); - tool.firePluginEvent( - new RevEngAIAnalysisStatusChangedEvent( - "programActivated", - new ProgramWithBinaryID(program, binID, analysisID), - status)); - } + // Any ComponentProviders that need to refresh based on the current program should be notified here + analysisLogComponent.programActivated(program); } @Override @@ -308,12 +334,14 @@ public void processEvent(PluginEvent event) { // If the analysis is complete, we refresh the function signatures from the server var program = analysisEvent.getProgramWithBinaryID(); try { - revengService.registerFinishedAnalysisForProgram(program); + // TODO: Can we get a better taskmonitor here? + // Or should we never do something here that warrants a monitor in the first place? + var analysedProgram = revengService.registerFinishedAnalysisForProgram(program, TaskMonitor.DUMMY); + tool.firePluginEvent(new RevEngAIAnalysisResultsLoaded("AnalysisManagementPlugin", analysedProgram)); } catch (Exception e) { Msg.error(this, "Error registering finished analysis for program " + program, e); return; } - tool.firePluginEvent(new RevEngAIAnalysisResultsLoaded("AnalysisManagementPlugin", program)); } } } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java index 4750a64d..8ee9a1ca 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/BinarySimilarityPlugin.java @@ -22,6 +22,7 @@ import ai.reveng.toolkit.ghidra.binarysimilarity.ui.functionmatching.FunctionLevelFunctionMatchingDialog; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisResultsLoaded; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.core.services.function.export.ExportFunctionBoundariesService; import ai.reveng.toolkit.ghidra.core.services.logging.ReaiLoggingService; @@ -77,7 +78,7 @@ protected void locationChanged(ProgramLocation loc) { // If no program, or not attached to a complete analysis, do not trigger location change events var program = loc.getProgram(); - if (program == null || !apiService.isKnownProgram(program) || !apiService.isProgramAnalysed(program)) { + if (program == null || apiService.getAnalysedProgram(program).isEmpty()) { return; } @@ -119,7 +120,7 @@ private void setupActions() { .enabledWhen(context -> { var program = tool.getService(ProgramManager.class).getCurrentProgram(); if (program != null) { - return apiService.isKnownProgram(program); + return apiService.getKnownProgram(program).isPresent(); } else { return false; } @@ -127,18 +128,18 @@ private void setupActions() { ) .onAction(context -> { var program = tool.getService(ProgramManager.class).getCurrentProgram(); - if (!apiService.isProgramAnalysed(program)) { + if (apiService.getAnalysedProgram(program).isEmpty()) { Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Auto Unstrip", "Analysis must have completed before running auto unstrip"); return; } - var knownProgram = apiService.getKnownProgram(program); - if (knownProgram.isEmpty()){ + var analysedProgram = apiService.getAnalysedProgram(program); + if (analysedProgram.isEmpty()){ Msg.info(this, "Program has no saved binary ID"); return; } - var autoUnstrip = new AutoUnstripDialog(tool, knownProgram.get()); + var autoUnstrip = new AutoUnstripDialog(tool, analysedProgram.get()); tool.showDialog(autoUnstrip); }) @@ -151,7 +152,7 @@ private void setupActions() { .enabledWhen(context -> { var program = tool.getService(ProgramManager.class).getCurrentProgram(); if (program != null) { - return apiService.isKnownProgram(program); + return apiService.getAnalysedProgram(program).isPresent(); } else { return false; } @@ -159,16 +160,12 @@ private void setupActions() { ) .onAction(context -> { var program = tool.getService(ProgramManager.class).getCurrentProgram(); - if (!apiService.isProgramAnalysed(program)) { + var knownProgram = apiService.getAnalysedProgram(program); + if (knownProgram.isEmpty()){ Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Function Matching", "Analysis must have completed before running function matching"); return; } - var knownProgram = apiService.getKnownProgram(program); - if (knownProgram.isEmpty()){ - Msg.info(this, "Program has no saved binary ID"); - return; - } var functionMatchingDialog = new BinaryLevelFunctionMatchingDialog(tool, knownProgram.get()); tool.showDialog(functionMatchingDialog); @@ -184,25 +181,15 @@ private void setupActions() { // Exclude thunks and external functions because we do not support them in the portal && !func.isExternal() && !func.isThunk() - && apiService.isKnownProgram(context.getProgram()) - && apiService.isProgramAnalysed(context.getProgram()); + && apiService.getAnalysedProgram(context.getProgram()).isPresent(); }) .onAction(context -> { - var program = tool.getService(ProgramManager.class).getCurrentProgram(); - if (!apiService.isProgramAnalysed(program)) { - Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "Match function", - "Analysis must have completed before running function matching"); - return; - } - var knownProgram = apiService.getKnownProgram(program); - if (knownProgram.isEmpty()){ - Msg.info(this, "Program has no saved binary ID"); - return; - } + // We know analysed program is present due to enabledWhen + var knownProgram = apiService.getAnalysedProgram(context.getProgram()).get(); var func = context.getProgram().getFunctionManager().getFunctionContaining(context.getAddress()); - var functionMatchingDialog = new FunctionLevelFunctionMatchingDialog(tool, knownProgram.get(), func); + var functionMatchingDialog = new FunctionLevelFunctionMatchingDialog(tool, knownProgram, func); tool.showDialog(functionMatchingDialog); }) .popupMenuPath(new String[] { "Match function" }) @@ -219,12 +206,13 @@ private void setupActions() { // Exclude thunks and external functions because we do not support them in the portal && !func.isExternal() && !func.isThunk() - && apiService.isKnownProgram(context.getProgram()) - && apiService.isProgramAnalysed(context.getProgram()); + && apiService.getAnalysedProgram(context.getProgram()).isPresent(); }) .onAction(context -> { var func = context.getProgram().getFunctionManager().getFunctionContaining(context.getAddress()); - if (!apiService.isKnownFunction(func)) { + var analysedProgram = apiService.getAnalysedProgram(context.getProgram()).get(); + var functionWithId = analysedProgram.getIDForFunction(func); + if (functionWithId.isEmpty()) { Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "AI Decompilation", "Function is not known to the RevEng.AI API." + "This can happen if the function boundaries do not match.\n" + @@ -234,7 +222,7 @@ private void setupActions() { // Spawn Task to decompile the function tool.getService(ReaiLoggingService.class).info("Requested AI Decompilation via Action for function " + func.getName()); decompiledWindow.setVisible(true); - decompiledWindow.refresh(func); + decompiledWindow.refresh(functionWithId.get()); }) .popupMenuPath(new String[] { "AI Decompilation" }) .popupMenuIcon(ReaiPluginPackage.REVENG_16) @@ -246,13 +234,13 @@ private void setupActions() { .enabledWhen(context -> { var func = context.getProgram().getFunctionManager().getFunctionContaining(context.getAddress()); return func != null - && apiService.isKnownProgram(context.getProgram()) - && apiService.isProgramAnalysed(context.getProgram()); + && apiService.getAnalysedProgram(context.getProgram()).isPresent(); }) .onAction(context -> { var func = context.getProgram().getFunctionManager().getFunctionContaining(context.getAddress()); - var functionID = apiService.getFunctionIDFor(func); - if (!apiService.isKnownFunction(func) || functionID.isEmpty()) { + var analysedProgram = apiService.getAnalysedProgram(context.getProgram()).get(); + var functionWithID = analysedProgram.getIDForFunction(func); + if (functionWithID.isEmpty()) { Msg.showError(this, null, ReaiPluginPackage.WINDOW_PREFIX + "View function in portal", "Function is not known to the RevEng.AI API." + "This can happen if the function boundaries do not match.\n" + @@ -260,7 +248,7 @@ private void setupActions() { return; } - apiService.openFunctionInPortal(functionID.get()); + apiService.openFunctionInPortal(functionWithID.get().functionID()); }) .popupMenuPath(new String[] { "View function in portal" }) .popupMenuIcon(ReaiPluginPackage.REVENG_16) @@ -272,7 +260,7 @@ private void setupActions() { public void readDataState(SaveState saveState) { int[] rawCollectionIDs = saveState.getInts("collectionIDs", new int[0]); var restoredCollections = Arrays.stream(rawCollectionIDs) - .mapToObj(CollectionID::new) + .mapToObj(TypedApiInterface.CollectionID::new) .map(cID -> apiService.getApi().getCollectionInfo(cID)) .toList(); apiService.setActiveCollections(restoredCollections); @@ -280,7 +268,7 @@ public void readDataState(SaveState saveState) { @Override public void writeDataState(SaveState saveState) { - int[] collectionIDs = apiService.getActiveCollections().stream().map(Collection::collectionID).mapToInt(CollectionID::id).toArray(); + int[] collectionIDs = apiService.getActiveCollections().stream().map(Collection::collectionID).mapToInt(TypedApiInterface.CollectionID::id).toArray(); saveState.putInts("collectionIDs", collectionIDs); } diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/DevPlugin.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/DevPlugin.java index cd9fd16c..7c37534f 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/DevPlugin.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/DevPlugin.java @@ -1,7 +1,7 @@ package ai.reveng.toolkit.ghidra.plugins; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.types.exceptions.APIConflictException; import ai.reveng.toolkit.ghidra.devplugin.RevEngMetadataProvider; import docking.action.builder.ActionBuilder; @@ -50,8 +50,8 @@ public DevPlugin(PluginTool tool) { .onAction(e -> { GhidraRevengService reAIService = tool.getService(GhidraRevengService.class); var api = reAIService.getApi(); - AnalysisID analysisID = reAIService.getAnalysisIDFor(currentProgram).get(); - var functionMap = reAIService.getFunctionMap(currentProgram); + var analysedProgram = reAIService.getAnalysedProgram(currentProgram).orElseThrow(); + var functionMap = analysedProgram.getFunctionMap(); var task = new Task("Generate Signatures", true, true, true) { @Override public void run(TaskMonitor monitor) { @@ -60,7 +60,7 @@ public void run(TaskMonitor monitor) { (fID, function) -> { try { monitor.checkCancelled(); - api.generateFunctionDataTypes(analysisID, List.of(fID)); + api.generateFunctionDataTypes(analysedProgram.analysisID(), List.of(fID)); monitor.incrementProgress(1); } catch (APIConflictException e) { // Already requested diff --git a/src/main/java/ai/reveng/toolkit/ghidra/plugins/ReaiPluginPackage.java b/src/main/java/ai/reveng/toolkit/ghidra/plugins/ReaiPluginPackage.java index 2c0a5f57..2b819b75 100644 --- a/src/main/java/ai/reveng/toolkit/ghidra/plugins/ReaiPluginPackage.java +++ b/src/main/java/ai/reveng/toolkit/ghidra/plugins/ReaiPluginPackage.java @@ -22,12 +22,16 @@ public class ReaiPluginPackage extends PluginPackage { public static final String OPTION_KEY_HOSTNAME = PREFIX + "Hostname"; public static final String OPTION_KEY_PORTAL_HOSTNAME = PREFIX + "Portal Hostname"; public static final String OPTION_KEY_MODEL = PREFIX + "Model"; + @Deprecated public static final String OPTION_KEY_BINID = PREFIX + "Binary ID"; + public static final String OPTION_KEY_ANALYSIS_ID = PREFIX + "Analysis ID"; public static final String REAI_OPTIONS_CATEGORY = "RevEngAI Options"; + @Deprecated public static final Integer INVALID_BINARY_ID = -1; + public static final Integer INVALID_ANALYSIS_ID = -1; public static final Icon REVENG_16 = ResourceManager.loadImage("images/reveng_16.png"); diff --git a/src/test/java/ConvertBinSyncArtifactTests.java b/src/test/java/ConvertBinSyncArtifactTests.java index f2c8fee8..511fef13 100644 --- a/src/test/java/ConvertBinSyncArtifactTests.java +++ b/src/test/java/ConvertBinSyncArtifactTests.java @@ -1,42 +1,40 @@ +import ai.reveng.model.FunctionDataTypes; +import ai.reveng.model.FunctionInfoOutput; import ai.reveng.toolkit.ghidra.binarysimilarity.cmds.ComputeTypeInfoTask; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.V2Response; -import ai.reveng.toolkit.ghidra.core.services.api.mocks.TypeGenerationMock; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisID; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionDataTypeStatus; +import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; +import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.services.api.types.DataTypeList; -import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionID; import ghidra.program.model.data.CategoryPath; import ghidra.program.model.data.DataType; import ghidra.program.model.data.DataTypeDependencyException; import ghidra.program.model.data.Structure; -import ghidra.test.AbstractGhidraHeadlessIntegrationTest; import ghidra.util.Msg; import ghidra.util.exception.CancelledException; import ghidra.util.task.TaskMonitor; -import org.json.JSONObject; import org.junit.Ignore; import org.junit.Test; import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; +@Ignore("Integration tests that rely on mock data from files") public class ConvertBinSyncArtifactTests extends AbstractRevEngIntegrationTest { - AnalysisID analysisID = new AnalysisID(1337); + TypedApiInterface.AnalysisID analysisID = new TypedApiInterface.AnalysisID(1337); @Test - public void testSimpleGhidraSignatureGeneration() throws DataTypeDependencyException { + public void testSimpleGhidraSignatureGeneration() throws DataTypeDependencyException, IOException { V2Response mockResponse = getMockResponseFromFile("main_fdupes_77846709.json"); - FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var signature = GhidraRevengService.getFunctionSignature(functionDataTypeStatus.data_types().get()); +// FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); + var funcInfo = FunctionDataTypes.fromJson(mockResponse.getJsonData().toString()); + var signature = GhidraRevengService.getFunctionSignature(funcInfo.getDataTypes()).orElseThrow(); assert signature.getName().equals("main"); assert signature.getReturnType().getName().equals("int"); @@ -74,11 +72,11 @@ public void testDependencyToDtm() { } @Test - public void testComplexGhidraSignatureGeneration() throws DataTypeDependencyException { + public void testComplexGhidraSignatureGeneration() throws DataTypeDependencyException, IOException { var mockResponse = getMockResponseFromFile("confirmmatch_fdupes_77846700.json"); - FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var signature = GhidraRevengService.getFunctionSignature(functionDataTypeStatus.data_types().get()); + var funcInfo = FunctionInfoOutput.fromJson(mockResponse.getJsonData().toString()); + var signature = GhidraRevengService.getFunctionSignature(funcInfo).orElseThrow(); assert signature.getName().equals("confirmmatch"); Msg.info(this, signature); @@ -86,22 +84,24 @@ public void testComplexGhidraSignatureGeneration() throws DataTypeDependencyExce @Test - public void testComplexGhidraSignatureGeneration2() throws DataTypeDependencyException { + public void testComplexGhidraSignatureGeneration2() throws DataTypeDependencyException, IOException { var mockResponse = getMockResponseFromFile("summarizematches_fdupes.json"); FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var signature = GhidraRevengService.getFunctionSignature(functionDataTypeStatus.data_types().get()); + var funcInfo = FunctionInfoOutput.fromJson(mockResponse.getJsonData().toString()); + var signature = GhidraRevengService.getFunctionSignature(funcInfo).orElseThrow(); assert signature.getName().equals("summarizematches"); Msg.info(this, signature); } @Test - public void testComplexGhidraSignatureGeneration3() throws DataTypeDependencyException { + public void testComplexGhidraSignatureGeneration3() throws DataTypeDependencyException, IOException { var mockResponse = getMockResponseFromFile("md5_process_fdupes.json"); FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var signature = GhidraRevengService.getFunctionSignature(functionDataTypeStatus.data_types().get()); + var funcInfo = FunctionInfoOutput.fromJson(mockResponse.getJsonData().toString()); + var signature = GhidraRevengService.getFunctionSignature(funcInfo).orElseThrow(); var dtm = signature.getDataTypeManager(); assert signature.getName().equals("md5_process"); @@ -118,11 +118,11 @@ public void testComplexGhidraSignatureGeneration3() throws DataTypeDependencyExc */ @Ignore("Ignored until it can properly distinguish an infinite loop and an exception") @Test - public void testNoLoopForBrokenDeps() throws DataTypeDependencyException { + public void testNoLoopForBrokenDeps() throws DataTypeDependencyException, IOException { var mockResponse = getMockResponseFromFile("errormsg.json"); - FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var signature = GhidraRevengService.getFunctionSignature(functionDataTypeStatus.data_types().get()); + var funcInfo = FunctionInfoOutput.fromJson(mockResponse.getJsonData().toString()); + var signature = GhidraRevengService.getFunctionSignature(funcInfo).orElseThrow(); assert signature.getName().equals("md5_process"); Msg.info(this, signature); @@ -134,10 +134,11 @@ public void testNoLoopForBrokenDeps() throws DataTypeDependencyException { * The function is `registerpair` from `fdupes`: registerpair */ @Test - public void testFunctionPointerArgument() throws DataTypeDependencyException { + public void testFunctionPointerArgument() throws DataTypeDependencyException, IOException { var mockResponse = getMockResponseFromFile("complex_pointer.json"); - FunctionDataTypeStatus functionDataTypeStatus = FunctionDataTypeStatus.fromJson(mockResponse.getJsonData()); - var signature = GhidraRevengService.getFunctionSignature(functionDataTypeStatus.data_types().get()); + var signature = GhidraRevengService.getFunctionSignature( + FunctionInfoOutput.fromJson(mockResponse.getJsonData().toString()) + ); } @Test @@ -154,7 +155,7 @@ public void testBatchResponse() { var mockResponse = getMockResponseFromFile("data_types_batch_response.json"); DataTypeList batchResponse = DataTypeList.fromJson(mockResponse.getJsonData()); - var r1 = batchResponse.statusForFunction(new FunctionID(266294328)); + var r1 = batchResponse.statusForFunction(new TypedApiInterface.FunctionID(266294328)); assert r1.data_types().orElseThrow().functionName().equals("sort_pairs_by_mtime"); } @@ -163,10 +164,66 @@ public void testDataTypeGenerationTask() throws CancelledException { var mockApi = new TypeGenerationMock(); var task = new ComputeTypeInfoTask( new GhidraRevengService(mockApi), - IntStream.range(0, 5).boxed().map(FunctionID::new).collect(Collectors.toList()), null + IntStream.range(0, 5).boxed().map(TypedApiInterface.FunctionID::new).collect(Collectors.toList()), null ); task.run(TaskMonitor.DUMMY); } + public static class TypeGenerationMock extends UnimplementedAPI { + + Set generatedFunctions = new HashSet<>(); + @Override + public DataTypeList generateFunctionDataTypes(AnalysisID analysisID, List functionIDS) { + var statuses = functionIDS.stream() + .map(id -> new FunctionDataTypeStatus( + false, + Optional.empty(), + "UNKNOWN", + null, + id + )) + .toList(); + return new DataTypeList( + functionIDS.size(), 0, statuses.toArray(new FunctionDataTypeStatus[0]) + ); + } + + @Override + public DataTypeList getFunctionDataTypes(List functionIDS) { + for (FunctionID functionID : functionIDS) { + if (generatedFunctions.contains(functionID)) continue; + generatedFunctions.add(functionID); + break; + } + + var statuses = functionIDS.stream() + .map(id -> new FunctionDataTypeStatus( + generatedFunctions.contains(id), + Optional.empty(), + generatedFunctions.contains(id) ? "completed" : "UNKNOWN", + null, + id + )) + .toList(); + + return new DataTypeList( + functionIDS.size(), 0, statuses.toArray(new FunctionDataTypeStatus[0]) + ); + } + + @Override + public FunctionDetails getFunctionDetails(FunctionID id) { + return new FunctionDetails( + id, + "placeholder_for_%s".formatted(id), + 0L, + 10L, + new AnalysisID(1337), + "placeholder_for_%s".formatted(id), + new BinaryHash("placeholder_for_%s".formatted(id)), + "demangled_placeholder_for_%s".formatted(id) + ); + } + } } diff --git a/src/test/java/HelperTests.java b/src/test/java/HelperTests.java index ddd3fde1..17895b9c 100644 --- a/src/test/java/HelperTests.java +++ b/src/test/java/HelperTests.java @@ -6,7 +6,7 @@ public class HelperTests { @Test public void testPathSplitting(){ - var path = TypePathAndName.fromString("a/b/c"); + var path = TypePathAndName.fromString("a::b::c"); assert path.name().equals("c"); assert path.path().length == 2; assert path.path()[0].equals("a"); diff --git a/src/test/java/ai/reveng/AIDecompilerComponentTest.java b/src/test/java/ai/reveng/AIDecompilerComponentTest.java index 87a4562b..7f7ec1fb 100644 --- a/src/test/java/ai/reveng/AIDecompilerComponentTest.java +++ b/src/test/java/ai/reveng/AIDecompilerComponentTest.java @@ -1,6 +1,8 @@ package ai.reveng; +import ai.reveng.invoker.ApiException; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.aidecompiler.AIDecompilationdWindow; +import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; import ai.reveng.toolkit.ghidra.plugins.BinarySimilarityPlugin; @@ -36,7 +38,7 @@ public AnalysisStatus status(AnalysisID analysisID) { } @Override - public List getFunctionInfo(BinaryID binaryID) { + public List getFunctionInfo(AnalysisID analysisID) { return List.of( new FunctionInfo( new FunctionID(1), @@ -83,6 +85,11 @@ public AIDecompilationStatus pollAIDecompileStatus(FunctionID functionID) { } + @Override + public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + return new AnalysisID(1); + } + @Override public boolean triggerAIDecompilationForFunctionID(FunctionID functionID) { return true; @@ -155,7 +162,7 @@ public void testAIDecompFeedbackMechanism() throws Exception { var func2 = builder.createEmptyFunction(null, "0x2000", 10, Undefined.getUndefinedDataType(4)); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); - +// service.getAnalysedProgram(programWithID); env.showTool(programWithID.program()); waitForSwing(); @@ -190,13 +197,18 @@ static class RatingsAPI extends UnimplementedAPI { public RatingsAPI() { } + @Override + public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + return new AnalysisID(1); + } + @Override public AnalysisStatus status(AnalysisID analysisID) { return AnalysisStatus.Complete; } @Override - public List getFunctionInfo(BinaryID binaryID) { + public List getFunctionInfo(AnalysisID analysisID) { return List.of( new FunctionInfo( new FunctionID(1), diff --git a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java index a753349a..6e22f6d7 100644 --- a/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java +++ b/src/test/java/ai/reveng/PortalAnalysisIntegrationTest.java @@ -1,18 +1,32 @@ package ai.reveng; +import ai.reveng.invoker.ApiException; +import ai.reveng.model.FunctionDataTypesList; +import ai.reveng.model.FunctionDataTypesListItem; +import ai.reveng.model.FunctionInfoOutput; +import ai.reveng.model.FunctionTypeOutput; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisResultsLoaded; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; +import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; +import ai.reveng.toolkit.ghidra.core.services.api.types.binsync.*; import ai.reveng.toolkit.ghidra.plugins.AnalysisManagementPlugin; +import ghidra.framework.Application; +import ghidra.framework.ApplicationVersion; import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; +import ghidra.program.model.symbol.SourceType; +import ghidra.util.task.TaskMonitor; +import org.jetbrains.annotations.Nullable; import org.junit.Assert; import org.junit.Test; +import java.io.IOException; import java.util.List; +import java.util.Optional; import java.util.concurrent.atomic.AtomicBoolean; import static org.junit.Assert.assertEquals; @@ -20,21 +34,122 @@ public class PortalAnalysisIntegrationTest extends RevEngMockableHeadedIntegrationTest { + /// TODO: This test currently tests more things at once than needed and could be split + /// * Test that the checks the event lifecycle works (status changed -> results loaded) + /// * Test that loading the function info/details works correctly + /// @Test public void testInfoLoading() throws Exception { var tool = env.getTool(); addMockedService(tool, new UnimplementedAPI() { @Override - public List getFunctionInfo(BinaryID binaryID) { + public List getFunctionInfo(AnalysisID analysisID) { return List.of( - new FunctionInfo(new FunctionID(1), "portal_name", "portal_name_mangled", 0x4000L, 0x100) + new FunctionInfo(new FunctionID(1), "portal_name_demangled", "portal_name_mangled", 0x4000L, 0x100) ); } + + @Override + public FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID, @Nullable List ids) { + + try { + var list = FunctionDataTypesList.fromJson( + """ + { + "total_count": 1, + "total_data_types_count": 1, + "items": [ + { + "completed": true, + "status": "completed", + "data_types": { + "func_types": { + "addr": 1052960, + "size": 22, + "header": { + "name": "portal_name_demangled", + "addr": 1052960, + "type": "int", + "args": { + "0x0": { + "offset": 0, + "name": "ctx", + "type": "ossl_typ.h::EVP_PKEY_CTX *", + "size": 1 + } + } + }, + "name": "portal_name_demangled", + "type": "int", + "artifact_type": "Function" + }, + "func_deps": [ + { + "name": "evp_pkey_ctx_st", + "size": 0, + "members": {}, + "artifact_type": "Struct" + }, + { + "name": "EVP_PKEY_CTX", + "type": "ossl_typ.h::evp_pkey_ctx_st", + "artifact_type": "Typedef" + } + ] + }, + "function_id": 1 + } + ] + } + """ + ); + return list; + + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public AnalysisStatus status(AnalysisID analysisID) { + return AnalysisStatus.Complete; + } + + @Override + public FunctionDetails getFunctionDetails(FunctionID id) { + return new FunctionDetails( + id, + "portal_name_mangled", + 0x4000L, + 0x100L, + new AnalysisID(1), + "binary_name", + new BinaryHash("dummyhash"), + "portal_name_demangled" + ); + } + + @Override + public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + return new AnalysisID(1); + } }); var builder = new ProgramBuilder("mock", ProgramBuilder._8051, this); // Add an example function var exampleFunc = builder.createEmptyFunction(null, "0x4000", 0x100, Undefined.getUndefinedDataType(8)); + /// Tell Ghidra that the function signature source is just default, + /// as the logic in {@link GhidraRevengService#pullFunctionInfoFromAnalysis(GhidraRevengService.AnalysedProgram, TaskMonitor)} + /// relies on this to decide whether to update the function signature or not + var tId = builder.getProgram().startTransaction("Set function signature source"); + exampleFunc.setSignatureSource(SourceType.DEFAULT); + builder.getProgram().endTransaction(tId, true); + // We need to also create the memory where the function lives, `getFunctions` doesn't find it otherwise + builder.createMemory("test", "0x4000", 0x100); + Assert.assertNotNull(builder.getProgram().getFunctionManager().getFunctionAt(exampleFunc.getEntryPoint())); + assert builder.getProgram().getFunctionManager().getFunctionCount() == 1; + assert builder.getProgram().getFunctionManager().getFunctionAt(exampleFunc.getEntryPoint()) != null; + assert builder.getProgram().getFunctionManager().getFunctions(true).hasNext(); var program = builder.getProgram(); var defaultTool = env.showTool(program); @@ -42,9 +157,13 @@ public List getFunctionInfo(BinaryID binaryID) { env.addPlugin(AnalysisManagementPlugin.class); waitForSwing(); - var id = new ProgramWithBinaryID(program, new BinaryID(1), new AnalysisID(1)); + var service = defaultTool.getService(GhidraRevengService.class); - service.addBinaryIDtoProgramOptions(program, id.binaryID()); + // We start an analysis to get an Analysis ID associated with the program + var id = service.startAnalysis(program, null); + + assert service.getKnownProgram(program).isPresent(); + assert service.getAnalysedProgram(program).isEmpty(); // Register a listener for the results loaded event, to verify that has been fired later AtomicBoolean receivedResultsLoadedEvent = new AtomicBoolean(false); @@ -52,7 +171,7 @@ public List getFunctionInfo(BinaryID binaryID) { receivedResultsLoadedEvent.set(true); }); - // Simulate the analysis status change event + // Simulate the analysis status change event being triggered when the analysis is complete // We have to run this without waiting, otherwise the test case doesn't continue until the dialog is closed runSwing( () -> defaultTool.firePluginEvent( @@ -67,23 +186,35 @@ public List getFunctionInfo(BinaryID binaryID) { waitForSwing(); // Check that we received the results loaded event, i.e. other plugins would have been notified assertTrue(receivedResultsLoadedEvent.get()); + + // Check that an analysed program is now known + assert service.getAnalysedProgram(program).isPresent(); + var analyzedProgram = service.getAnalysedProgram(program).get(); + // Check that the function names have been updated to the one returned by the portal - assertEquals("portal_name", exampleFunc.getName()); + assertEquals("portal_name_demangled", exampleFunc.getName()); - // Check the function ID has been stored in the program options - var funcIDMap = service.getFunctionMap(program); + var signature = exampleFunc.getSignature(true); + assertEquals("int portal_name_demangled(EVP_PKEY_CTX * ctx)", signature.getPrototypeString()); + // For unclear reasons the signature source is not set by the command in Ghidra 11.2.x + // So we only test this for Ghidra 11.3 and above + ApplicationVersion version = new ApplicationVersion(Application.getApplicationVersion()); + if (version.compareTo(new ApplicationVersion("11.3")) > 0) { + assertEquals(SourceType.ANALYSIS, exampleFunc.getSignatureSource()); + } - var storedFunc = funcIDMap.get(new FunctionID(1)); - Assert.assertNotNull(storedFunc); - assertEquals("portal_name", storedFunc.getName()); - // Check the function mangled name has been stored - var mangledNamesMap = service.getFunctionMangledNamesMap(program); + // Check the function ID has been stored in the program options + var funcIDMap = analyzedProgram.getFunctionMap(); + var storedFunc = funcIDMap.get(new TypedApiInterface.FunctionID(1)); - assertTrue(mangledNamesMap.isPresent()); - assertEquals("portal_name_mangled", mangledNamesMap.get().getString(exampleFunc.getEntryPoint())); + Assert.assertNotNull(storedFunc); + assertEquals("portal_name_demangled", storedFunc.getName()); + // Check the function mangled name has been stored +// assertEquals("portal_name_mangled", mangledNamesMap.get().getString(exampleFunc.getEntryPoint())); + assertEquals("portal_name_mangled", analyzedProgram.getMangledNameForFunction(exampleFunc)); // TODO: What else should happen when the analysis is finished? } } diff --git a/src/test/java/ai/reveng/TestAnalysisLogComponent.java b/src/test/java/ai/reveng/TestAnalysisLogComponent.java index 4d7b7b29..51eb1960 100644 --- a/src/test/java/ai/reveng/TestAnalysisLogComponent.java +++ b/src/test/java/ai/reveng/TestAnalysisLogComponent.java @@ -1,12 +1,14 @@ package ai.reveng; import ai.reveng.toolkit.ghidra.binarysimilarity.ui.misc.AnalysisLogComponent; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.TypedApiInterface; import ai.reveng.toolkit.ghidra.plugins.AnalysisManagementPlugin; import ai.reveng.toolkit.ghidra.core.RevEngAIAnalysisStatusChangedEvent; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; import ai.reveng.toolkit.ghidra.core.services.api.types.*; -import ai.reveng.toolkit.ghidra.core.types.ProgramWithBinaryID; import ghidra.program.database.ProgramBuilder; +import ghidra.program.model.listing.Program; import ghidra.util.task.Task; import ghidra.util.task.TaskMonitorComponent; import org.junit.Test; @@ -19,14 +21,13 @@ public class TestAnalysisLogComponent extends RevEngMockableHeadedIntegrationTest { - private ProgramWithBinaryID getPlaceHolderID() throws Exception{ + private GhidraRevengService.ProgramWithID getPlaceHolderID() throws Exception{ var builder = new ghidra.program.database.ProgramBuilder("mock", ProgramBuilder._8051, this); // Add an example function var program = builder.getProgram(); - return new ProgramWithBinaryID( + return new GhidraRevengService.ProgramWithID( program, - new BinaryID(1), - new AnalysisID(1) + new TypedApiInterface.AnalysisID(1) ); } @@ -47,11 +48,11 @@ public AnalysisStatus status(AnalysisID analysisID) { // defaultTool.getService(GhidraRevengService.class); env.addPlugin(AnalysisManagementPlugin.class); - var program = getPlaceHolderID(); + var programWithID = getPlaceHolderID(); defaultTool.firePluginEvent( new RevEngAIAnalysisStatusChangedEvent( "Test", - program, + programWithID, AnalysisStatus.Processing ) ); @@ -59,22 +60,22 @@ public AnalysisStatus status(AnalysisID analysisID) { // The analysis log component should now be visible, and have a task var logComponent = defaultTool.getComponentProvider(AnalysisLogComponent.NAME); assertNotNull(logComponent); - Map trackedPrograms = (Map) getInstanceField("trackedPrograms", logComponent); + Map trackedPrograms = (Map) getInstanceField("trackedPrograms", logComponent); assertNotNull(trackedPrograms); - assertTrue(trackedPrograms.containsKey(program)); + assertTrue(trackedPrograms.containsKey(programWithID.program())); } @Test public void testFullAnalysisFlow() throws Exception { var defaultTool = env.showTool(); - var program = getPlaceHolderID(); + var programWithID = getPlaceHolderID(); addMockedService(defaultTool, new UnimplementedAPI() { private final Map statusMap = new HashMap<>(); { - statusMap.put(program.analysisID(), AnalysisStatus.Queued); + statusMap.put(programWithID.analysisID(), AnalysisStatus.Queued); } @Override @@ -91,7 +92,7 @@ public String getAnalysisLogs(AnalysisID analysisID) { } @Override - public List getFunctionInfo(BinaryID binaryID) { + public List getFunctionInfo(AnalysisID analysisID) { return List.of(); } } @@ -101,19 +102,19 @@ public List getFunctionInfo(BinaryID binaryID) { defaultTool.firePluginEvent( new RevEngAIAnalysisStatusChangedEvent( "Test", - program, + programWithID, AnalysisStatus.Queued ) ); AnalysisLogComponent logComponent = (AnalysisLogComponent) defaultTool.getComponentProvider(AnalysisLogComponent.NAME); - var trackedPrograms = (Map) getInstanceField("trackedPrograms", logComponent); + var trackedPrograms = (Map) getInstanceField("trackedPrograms", logComponent); assertNotNull(trackedPrograms); - assertTrue(trackedPrograms.containsKey(program)); + assertTrue(trackedPrograms.containsKey(programWithID.program())); waitForTasks(); // Check that it is cleared out again after the task finished - assertFalse(trackedPrograms.containsKey(program)); + assertFalse(trackedPrograms.containsKey(programWithID.program())); // Check that the taskmonitor is hidden again and isn't sitting there forever TaskMonitorComponent taskMonitorComponent = (TaskMonitorComponent) getInstanceField("taskMonitorComponent", logComponent); assertFalse(taskMonitorComponent.isVisible()); diff --git a/src/test/java/ai/reveng/TestMockableService.java b/src/test/java/ai/reveng/TestMockableService.java index 92616f8f..f1014d34 100644 --- a/src/test/java/ai/reveng/TestMockableService.java +++ b/src/test/java/ai/reveng/TestMockableService.java @@ -2,9 +2,6 @@ import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; -import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryHash; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; import org.junit.Test; diff --git a/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java b/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java new file mode 100644 index 00000000..7294458a --- /dev/null +++ b/src/test/java/ai/reveng/TestUpgradeFromBinaryID.java @@ -0,0 +1,64 @@ +package ai.reveng; + +import ai.reveng.invoker.ApiException; +import ai.reveng.toolkit.ghidra.core.services.api.GhidraRevengService; +import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; +import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; +import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryID; +import ai.reveng.toolkit.ghidra.plugins.ReaiPluginPackage; +import ghidra.program.database.ProgramBuilder; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Optional; + +@SuppressWarnings("deprecation") // This test ensures upgrade from deprecated Binary ID usage +public class TestUpgradeFromBinaryID extends RevEngMockableHeadedIntegrationTest { + + /// Tests the logic for handling a program that has only a binary ID stored in its properties + @Test + public void test() throws Exception { + var builder = new ProgramBuilder("upgrade-test", ProgramBuilder._8051, this); + var tId = builder.getProgram().startTransaction("Set Binary ID"); + builder.getProgram() + .getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY) + .setLong(ReaiPluginPackage.OPTION_KEY_BINID, 1); + builder.getProgram().endTransaction(tId, true); + addMockedService(env.getTool(), new UnimplementedAPI() { + @Override + public AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { + if (binaryID.value() == 1) { + return new AnalysisID(42); + } + return null; + } + + @Override + public AnalysisStatus status(BinaryID binID) throws ApiException { + if (binID.value() == 1) { + return AnalysisStatus.Complete; + } + return AnalysisStatus.Error; + } + }); + var program = builder.getProgram(); + + env.open(program); + + var service = env.getTool().getService(GhidraRevengService.class); + Optional analysisID = service.getKnownProgram(program); + Assert.assertTrue(analysisID.isPresent()); + Assert.assertEquals(42, analysisID.get().analysisID().id()); + // Verify that after opening, the program has the Analysis ID set and the Binary ID removed + Assert.assertEquals(-1, program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(ReaiPluginPackage.OPTION_KEY_BINID, -1)); + Assert.assertEquals(42, program.getOptions(ReaiPluginPackage.REAI_OPTIONS_CATEGORY).getLong(ReaiPluginPackage.OPTION_KEY_ANALYSIS_ID, -1)); + + + + + + + + + } +} diff --git a/src/test/java/ai/reveng/UnstripTest.java b/src/test/java/ai/reveng/UnstripTest.java index caa47fa2..b48f8eca 100644 --- a/src/test/java/ai/reveng/UnstripTest.java +++ b/src/test/java/ai/reveng/UnstripTest.java @@ -1,12 +1,18 @@ package ai.reveng; +import ai.reveng.invoker.ApiException; +import ai.reveng.model.AutoUnstripResponse; +import ai.reveng.model.FunctionDataTypesList; +import ai.reveng.model.MatchedFunctionSuggestion; import ai.reveng.toolkit.ghidra.core.services.api.AnalysisOptionsBuilder; import ai.reveng.toolkit.ghidra.core.services.api.mocks.UnimplementedAPI; -import ai.reveng.toolkit.ghidra.core.services.api.types.*; +import ai.reveng.toolkit.ghidra.core.services.api.types.AnalysisStatus; +import ai.reveng.toolkit.ghidra.core.services.api.types.FunctionInfo; import ai.reveng.toolkit.ghidra.plugins.BinarySimilarityPlugin; import ghidra.program.database.ProgramBuilder; import ghidra.program.model.data.Undefined; import ghidra.util.task.TaskMonitor; +import org.jetbrains.annotations.Nullable; import org.junit.Test; import java.util.Iterator; @@ -23,27 +29,29 @@ public void testFinishedUnstrip() throws Exception { var tool = env.getTool(); var service = addMockedService(tool, new UnimplementedAPI() { - + boolean autoUnstripCalled = false; @Override - public AutoUnstripResponse autoUnstrip(AnalysisID analysisID) { - return new AutoUnstripResponse( - 100, - "STATUS", - 0, - List.of(new AutoUnstripResponse.Match(new FunctionID(1), 0x1000, "unstripped_function_name", "unstripped_function_name_demangled" ) ), - false, - null - ); + public TypedAutoUnstripResponse autoUnstrip(AnalysisID analysisID) { + var r = new AutoUnstripResponse() + .progress(100) + .status("COMPLETED") + .applied(false) + .totalTime(0) + .matches(List.of( + new MatchedFunctionSuggestion() + .functionId(1L) + .functionVaddr(0x1000L) + .suggestedName("unstripped_function_name") + .suggestedDemangledName("unstripped_function_name_demangled") + ) + ); + autoUnstripCalled = true; + return new TypedAutoUnstripResponse(r); } @Override - public BinaryID analyse(AnalysisOptionsBuilder binHash) { - return new BinaryID(1); - } - - @Override - public AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { - return new AnalysisID(binaryID.value()); + public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + return new AnalysisID(1); } @Override @@ -52,15 +60,20 @@ public AnalysisStatus status(AnalysisID analysisID) { } @Override - public List getFunctionInfo(BinaryID binaryID) { - return List.of(new FunctionInfo(new FunctionID(1), "default_function_info_name", "default_function_info_name_mangled",0x1000L, 10)); + public List getFunctionInfo(AnalysisID analysisID) { + if (!autoUnstripCalled) { + return List.of(new FunctionInfo(new FunctionID(1), "FUN_0x1000", "FUN_0x1000",0x1000L, 10)); + } + else { + return List.of(new FunctionInfo(new FunctionID(1), "unstripped_function_name_demangled", "unstripped_function_name_mangled",0x1000L, 10)); + } } }); addPlugin(tool, BinarySimilarityPlugin.class); var builder = new ProgramBuilder("mock", ProgramBuilder._8051, this); var func = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); - + builder.createMemory("test", "01000", 0x100); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); env.showTool(programWithID.program()); @@ -81,14 +94,13 @@ public List getFunctionInfo(BinaryID binaryID) { public void testProgressingUnstrip() throws Exception { var tool = env.getTool(); Iterator responses = List.of( - new AutoUnstripResponse( - 0, - "QUEUED", - 0, - List.of(), - false, - null - ), + new AutoUnstripResponse() + .progress(0) + .status("QUEUED") + .applied(false) + .totalTime(0) + .matches(List.of()) + , // The poll interval is not configurable, so we can only test two responses before hitting a Ghidra test timeout // new AutoUnstripResponse( // 50, @@ -98,43 +110,52 @@ public void testProgressingUnstrip() throws Exception { // false, // null // ), - new AutoUnstripResponse( - 100, - "COMPLETED", - 0, - List.of(new AutoUnstripResponse.Match(new FunctionID(1), 0x1000, "unstripped_function_name", "unstripped_function_name_demangled") ), - false, - null - ) + new AutoUnstripResponse() + .progress(100) + .status("COMPLETED") + .totalTime(1) + .matches(List.of( + new MatchedFunctionSuggestion() + .functionId(1L) + .functionVaddr(0x1000L) + .suggestedName("unstripped_function_name") + .suggestedDemangledName("unstripped_function_name_demangled") + )) + .applied(false) ).iterator(); var service = addMockedService(tool, new UnimplementedAPI() { @Override - public AutoUnstripResponse autoUnstrip(AnalysisID analysisID) { - return responses.next(); + public TypedAutoUnstripResponse autoUnstrip(AnalysisID analysisID) { + return new TypedAutoUnstripResponse(responses.next()); } @Override - public BinaryID analyse(AnalysisOptionsBuilder binHash) { - return new BinaryID(1); + public AnalysisID analyse(AnalysisOptionsBuilder options) throws ApiException { + return new AnalysisID(1); } @Override - public AnalysisID getAnalysisIDfromBinaryID(BinaryID binaryID) { - return new AnalysisID(binaryID.value()); + public AnalysisStatus status(AnalysisID analysisID) { + return AnalysisStatus.Complete; } @Override - public AnalysisStatus status(AnalysisID analysisID) { - return AnalysisStatus.Complete; + public List getFunctionInfo(AnalysisID analysisID) { + // the function info will return a name, but it will _not_ be the unstripped name by default + if (responses.hasNext()) { + return List.of(new FunctionInfo(new FunctionID(1), "FUN_0x1000", "FUN_0x1000",0x1000L, 10)); + } else { + return List.of(new FunctionInfo(new FunctionID(1), "unstripped_function_name_demangled", "unstripped_function_name_mangled",0x1000L, 10)); + } } @Override - public List getFunctionInfo(BinaryID binaryID) { - // the function info will return a name, but it will _not_ be the unstripped name - return List.of(new FunctionInfo(new FunctionID(1), "default_function_info_name", "default_function_info_name_mangled",0x1000L, 10)); + public FunctionDataTypesList listFunctionDataTypesForAnalysis(AnalysisID analysisID) { + var list = new FunctionDataTypesList(); + return list; } }); @@ -142,7 +163,7 @@ public List getFunctionInfo(BinaryID binaryID) { var builder = new ProgramBuilder("mock", ProgramBuilder._8051, this); // We provide no function name, so ghidra will assign the default "FUN_1000" name var func = builder.createEmptyFunction(null, "0x1000", 10, Undefined.getUndefinedDataType(4)); - + builder.createMemory("test", "01000", 0x100); var programWithID = service.analyse(builder.getProgram(), null, TaskMonitor.DUMMY); env.showTool(programWithID.program()); diff --git a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilderTest.java b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilderTest.java index a3cc9ce2..eb465e96 100644 --- a/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilderTest.java +++ b/src/test/java/ai/reveng/toolkit/ghidra/core/services/api/AnalysisOptionsBuilderTest.java @@ -2,7 +2,6 @@ import ai.reveng.model.AnalysisCreateRequest; import ai.reveng.model.Tag; -import ai.reveng.toolkit.ghidra.core.services.api.types.BinaryHash; import org.junit.Test; import java.util.Arrays; @@ -16,7 +15,7 @@ public class AnalysisOptionsBuilderTest { public void testToAnalysisCreateRequest_WithNoTags() { // Create a builder with no tags AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash("a".repeat(64))) + .hash(new TypedApiInterface.BinaryHash("a".repeat(64))) .fileName("test.bin"); AnalysisCreateRequest request = builder.toAnalysisCreateRequest(); @@ -31,7 +30,7 @@ public void testToAnalysisCreateRequest_WithNoTags() { public void testToAnalysisCreateRequest_WithValidTags() { // Create a builder with valid tags AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash("b".repeat(64))) + .hash(new TypedApiInterface.BinaryHash("b".repeat(64))) .fileName("test.bin") .addTag("malware") .addTag("suspicious"); @@ -54,7 +53,7 @@ public void testToAnalysisCreateRequest_WithValidTags() { public void testToAnalysisCreateRequest_WithEmptyStringTag() { // Create a builder with an empty string tag AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash("c".repeat(64))) + .hash(new TypedApiInterface.BinaryHash("c".repeat(64))) .fileName("test.bin") .addTag("") .addTag(" ") // whitespace only @@ -72,7 +71,7 @@ public void testToAnalysisCreateRequest_WithEmptyStringTag() { public void testToAnalysisCreateRequest_WithMultipleTags() { // Create a builder with multiple tags using addTags method AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash("d".repeat(64))) + .hash(new TypedApiInterface.BinaryHash("d".repeat(64))) .fileName("test.bin") .addTags(Arrays.asList("tag1", "tag2", "tag3")); @@ -87,7 +86,7 @@ public void testToAnalysisCreateRequest_WithMultipleTags() { public void testToAnalysisCreateRequest_WithOnlyEmptyTags() { // Create a builder with only empty/whitespace tags AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash("e".repeat(64))) + .hash(new TypedApiInterface.BinaryHash("e".repeat(64))) .fileName("test.bin") .addTag("") .addTag(" ") @@ -108,7 +107,7 @@ public void testToAnalysisCreateRequest_BasicFields() { String hash = "f".repeat(64); AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash(hash)) + .hash(new TypedApiInterface.BinaryHash(hash)) .fileName(filename); AnalysisCreateRequest request = builder.toAnalysisCreateRequest(); @@ -121,7 +120,7 @@ public void testToAnalysisCreateRequest_BasicFields() { public void testGetTags() { // Test the getTags method AnalysisOptionsBuilder builder = new AnalysisOptionsBuilder() - .hash(new BinaryHash("a".repeat(64))) + .hash(new TypedApiInterface.BinaryHash("a".repeat(64))) .fileName("test.bin"); // Initially should be empty