diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..626dd88 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + - package-ecosystem: "maven" + directory: "/" + schedule: + interval: "daily" + target-branch: "develop" diff --git a/README.md b/README.md index e207e08..03d39c4 100644 --- a/README.md +++ b/README.md @@ -25,25 +25,25 @@ A video demonstrating the end-to-end use of FlowMate is available on our YouTube **FlowMate** is used best during the reconnaissance phase in a security assessment. The following steps explain on how to get started: 1. Load FlowMate into your BurpSuite with a project for your current assessment already created 2. After loading finished add the target application to the BurpSuite internal *Scope*. Only in-scope targets are tracked by FlowMate -3. Activate the detection by checking both boxes on the *Getting Started* tab of FlowMate +3. Activate the detection by checking both boxes on the *Getting Started* tab of FlowMate. You can choose *Live* or *Deferred* matching. *Deferred* is recommended for bigger applications as *Live* matching might slow down the browsing experience if hundreds of parameters are matched against every response. 4. Browse the application following the *General best practices* below 5. Stop the detection before starting manual analysis. This prevents payloads and duplicate values from polluting the graph 6. Profit from the data flow graph created for you! +### General best practices +- Enter *unique* and *long enough* values (generally more than 6 characters) when browsing an application with FlowMate enabled +- Do not enter payloads during this phase +- Browse all user roles and functionality available + ### What can you get from the graph? -1. You can look up the locations where a specific parameter you are testing reappears in the application, including the immediate surroundings of the match, giving a first impression of which payloads might be useful for exploitation -2. You can more easily identify occurrences of a parameter in places that are not directly visible, such as in hidden input fields or when a value is used in resources like stylesheets or scripts for example +1. You can lookup in which locations an specific parameter you are testing reappers in the application including the near surrounding of the match giving a first impression on which payloads might be useful for exploitation +2. You can more easily identify occurrences of a parameter in not directly visible places, such as in hidden input fields or when a value is used in resources like stylesheets or scripts for example 3. In conjunction with the session tracking feature you can track cross-session parameter occurrences. In case of attack vectors like Cross-Site Scripting (XSS) this may lead to attacks on higher privileged accounts (privilege escalation, account takeover) 4. If your target application consists of multiple domains, for example APIs and the actual web frontend, the graph helps to detect cross-domain occurrences of parameter matches 5. You can directly identify unsafe behavior of the application from the graph. Some examples include: - A user password is included in the applications sources in cleartext - Security enhancements such as CSRF tokens are not changed in a secure manner - -### General best practices -- Enter *unique* and *long enough* values (generally more than 6 characters) when browsing an application with FlowMate enabled -- Do not enter payloads during this phase -- Browse all user roles and functionality available - + ## Installation ### If you want to use a pre-built JAR file, follow these steps @@ -54,4 +54,7 @@ A video demonstrating the end-to-end use of FlowMate is available on our YouTube 1. Clone the repository, switch into it and run `mvn package`. The `target` folder then contains a built version of FlowMate 2. Follow the steps to install an extension from JAR file here: [Installing an extension from a file](https://portswigger.net/burp/documentation/desktop/extensions/installing-extensions#installing-an-extension-from-a-file) +## Reporting Issues +If you encounter issues with FlowMate please report to us via a GitHub Issue. Error Logs are written to the logfile in `~/.flowmate` folder on your system running the plugin as well as the error log console in the BurpSuite Extension tab ("Extension" Tab > "Installed" Tab > Select "FlowMate" > "Errors" Tab). +Supplying us with this information as well as a short description of when the error occurs helps us to troubleshoot and fix the issue. Furthermore, keeping the BurpState might help if we have questions back to you. diff --git a/pom.xml b/pom.xml index d3ffc77..6981bc7 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ de.usd flowmate - 1.1 + 1.1.1 17 @@ -18,7 +18,7 @@ maven-assembly-plugin - 3.6.0 + 3.7.1 src/assembly/jar-descriptor.xml @@ -34,6 +34,16 @@ + + org.codehaus.mojo + versions-maven-plugin + 2.7 + + + net.portswigger.burp.extensions + + + @@ -47,89 +57,78 @@ org.neo4j neo4j-bolt - 5.16.0 + 2025.07.1 org.neo4j neo4j - 5.16.0 + 2025.07.1 org.neo4j neo4j-ogm-core - 4.0.9 + 4.0.19 org.neo4j neo4j-ogm-bolt-driver - 4.0.9 + 4.0.19 org.neo4j.client neo4j-browser - 5.15.0 + 5.24.0 org.neo4j.app neo4j-server - 5.16.0 + 2025.07.1 org.neo4j neo4j-native - 5.16.0 + 2025.07.1 org.jsoup jsoup - 1.17.2 + 1.21.1 org.json json - 20231013 + 20250517 org.apache.commons commons-lang3 - 3.14.0 + 3.18.0 org.apache.commons commons-text - 1.11.0 + 1.14.0 com.miglayout miglayout 3.7.4 - - com.opencsv - opencsv - 5.9 - org.projectlombok lombok - 1.18.30 + 1.18.34 provided - - - org.openjfx - javafx-controls - 21.0.1 - org.commonmark commonmark - 0.21.0 + 0.22.0 @@ -141,7 +140,7 @@ commons-io commons-io - 2.15.1 + 2.20.0 diff --git a/src/main/java/audit/CrossSessionAudit.java b/src/main/java/audit/CrossSessionAudit.java index 0c7b99f..870a503 100644 --- a/src/main/java/audit/CrossSessionAudit.java +++ b/src/main/java/audit/CrossSessionAudit.java @@ -1,15 +1,10 @@ package audit; -import db.DBModel; import db.MatchHelperClass; -import db.entities.InputValue; import db.entities.ParameterMatch; -import db.entities.Session; import gui.AuditFindingView; -import org.neo4j.ogm.model.Result; -import scala.reflect.macros.Universe; -import java.util.*; +import java.util.Vector; public class CrossSessionAudit { diff --git a/src/main/java/burp/BurpExtender.java b/src/main/java/burp/BurpExtender.java index e547a44..ff16f38 100644 --- a/src/main/java/burp/BurpExtender.java +++ b/src/main/java/burp/BurpExtender.java @@ -48,6 +48,7 @@ public class BurpExtender implements BurpExtension { private RegexMatcher regexMatcher; private DeferMatching deferMatching; private HttpListener listener; + private RetroactiveParser retroactiveParser; private MontoyaApi api; public Neo4JDB neo4j; @@ -93,8 +94,11 @@ public void initialize(MontoyaApi api) { queryView = new QueryView(this.listener.parameterHandler, this.listener.matchHandler, this.api); queryViewModel = new QueryViewModel(); queryViewController = new QueryViewController(api, queryView, queryViewModel, this.listener.parameterHandler, this.listener.matchHandler, sessionViewController); + this.retroactiveParser = new RetroactiveParser(this.api, this.listener.parameterHandler, this.queryViewController); gettingStartedView = new GettingStartedView(this.api, this.propertiesHandler, this.deferMatching, this.listener.parameterHandler, - this.listener.matchHandler, this.noiseReductionController, this.queryViewController, this.auditFindingView, this.sessionViewController); + this.listener.matchHandler, this.noiseReductionController, this.queryViewController, this.auditFindingView, this.sessionViewController, this.retroactiveParser); + + this.api.userInterface().registerContextMenuItemsProvider(new HistoryContextMenu(this.api, gettingStartedView)); if (this.propertiesHandler.isFirstTimeLoading) { historyStart = this.api.proxy().history().size() + 1; @@ -105,6 +109,8 @@ public void initialize(MontoyaApi api) { noiseReductionController.addRuleContainerListener(queryViewController); deferMatching.addDeferMatchingFinishedListener(sessionViewController); + deferMatching.addDeferMatchingFinishedListener(queryViewController); + listener.addItemsAddedListener(queryViewController); api.extension().registerUnloadingHandler(new MyExtensionUnloadHandler()); diff --git a/src/main/java/burp/ContainerConverter.java b/src/main/java/burp/ContainerConverter.java index 2bc6509..4228923 100644 --- a/src/main/java/burp/ContainerConverter.java +++ b/src/main/java/burp/ContainerConverter.java @@ -4,7 +4,10 @@ import db.CypherQueryHandler; import db.DBModel; import db.MatchHandler; -import db.entities.*; +import db.entities.InputParameter; +import db.entities.InputValue; +import db.entities.MatchValue; +import db.entities.ParameterMatch; import gui.container.*; import org.neo4j.ogm.model.Result; import utils.MessageHashToProxyId; @@ -33,7 +36,7 @@ public Vector parameterToContainer(List inpu var name = parameter.getName(); var type = parameter.getType(); Map values = Collections.singletonMap("name", name); - String query = "MATCH (n:InputParameter {name: $name, type: \"%s\"})-[OCCURS_WITH_VALUE]-(m:InputValue) RETURN m".formatted(type); + String query = "MATCH (n:InputParameter {name: $name, type: \"%s\"})-[OCCURS_WITH_VALUE]-(o:InputValue) RETURN o".formatted(type); Result result = DBModel.query(query, values); List inputValueList = new ArrayList<>(CypherQueryHandler.getOccurrencesFromQueryResult(result)); int occurrences = inputValueList.size(); @@ -55,7 +58,7 @@ public Vector parameterToContainerSessionDef(List values = Map.of("name", match.getName(), "type", match.getType(), "value", match.getValue()); - String valueQuery = "MATCH (p:InputParameter {name: $name, type: $type})-[OCCURS_WITH_VALUE]-(m:InputValue {type: $type, value: $value}) RETURN m"; + String valueQuery = "MATCH (p:InputParameter {name: $name, type: $type})-[OCCURS_WITH_VALUE]-(o:InputValue {type: $type, value: $value}) RETURN o"; Result result = DBModel.query(valueQuery, values); for (var parameterValue : CypherQueryHandler.getOccurrencesFromQueryResult(result).stream().distinct().toList()) { String type = parameterValue.getType(); @@ -119,7 +122,7 @@ public Vector entryOccurrenceToContainer(List o } public int getNumberOfMatchesForParameterName(String paramName, String type) { - List parameterMatchList = new ArrayList<>(this.matchHandler.observableParameterMatchList.stream().toList()); + List parameterMatchList = new ArrayList<>(this.matchHandler.parameterMatchStorage.values()); List duplicates = new ArrayList<>(); List correctParameterMatchList = new ArrayList<>(); for (var match : parameterMatchList) { @@ -136,7 +139,7 @@ public int getNumberOfMatchesForSessionTabForParameterName(String paramName, Str Map values = Map.of("sessionName", sessionName, "value", value, "type", type); String query = "MATCH (m:ParameterMatch {session: $sessionName, value: $value, type: $type}) RETURN m"; Result result = DBModel.query(query, values); - List parameterMatchList = new ArrayList<>(CypherQueryHandler.getMatchesFromQueryResult(result)); + List parameterMatchList = new ArrayList<>(CypherQueryHandler.getParameterMatchesFromQueryResult(result)); return parameterMatchList.stream().distinct().filter(e -> e.getName().equals(paramName)).toList().size(); } diff --git a/src/main/java/burp/HistoryContextMenu.java b/src/main/java/burp/HistoryContextMenu.java new file mode 100644 index 0000000..458f419 --- /dev/null +++ b/src/main/java/burp/HistoryContextMenu.java @@ -0,0 +1,93 @@ +package burp; + +import burp.api.montoya.MontoyaApi; +import burp.api.montoya.http.message.requests.HttpRequest; +import burp.api.montoya.ui.contextmenu.ContextMenuEvent; +import burp.api.montoya.ui.contextmenu.ContextMenuItemsProvider; +import gui.GettingStartedView; +import utils.Hashing; +import utils.MessageHashToProxyId; + +import javax.swing.*; +import java.awt.*; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; +import java.util.ArrayList; +import java.util.List; + +public class HistoryContextMenu implements ContextMenuItemsProvider { + + private MontoyaApi api; + private GettingStartedView gettingStartedView; + + private boolean isStartSet; + private boolean isEndSet; + private int startValue; + private int endValue; + + public HistoryContextMenu(MontoyaApi api, GettingStartedView gettingStartedView) { + this.api = api; + this.gettingStartedView = gettingStartedView; + this.isStartSet = false; + this.isEndSet = false; + } + + @Override + public List provideMenuItems(ContextMenuEvent event) { + + List menuItems = new ArrayList<>(); + JMenuItem setStartPointMenu = new JMenuItem("Set as start point for Retroactive Parsing"); + JMenuItem setEndPointMenu = new JMenuItem("Set as end point for Retroactive Parsing"); + + menuItems.add(setStartPointMenu); + menuItems.add(setEndPointMenu); + + setStartPointMenu.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent actionEvent) { + isStartSet = true; + HttpRequest selectedRequest = event.selectedRequestResponses().get(0).request(); + String messageHash = Hashing.sha1(event.selectedRequestResponses().get(0).request().toByteArray().getBytes()); + startValue = MessageHashToProxyId.getInstance(api).calculateId(messageHash); + if (!selectionsAreValid(selectedRequest)) { + isStartSet = false; + startValue = -1; + return; + } + + gettingStartedView.setStartValue(startValue); + } + }); + + setEndPointMenu.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent actionEvent) { + isEndSet = true; + HttpRequest selectedRequest = event.selectedRequestResponses().get(0).request(); + String messageHash = Hashing.sha1(event.selectedRequestResponses().get(0).request().toByteArray().getBytes()); + endValue = MessageHashToProxyId.getInstance(api).calculateId(messageHash); + if (!selectionsAreValid(selectedRequest)) { + isEndSet = false; + endValue = -1; + return; + } + + gettingStartedView.setEndValue(endValue); + } + }); + + return menuItems; + } + + private boolean selectionsAreValid(HttpRequest selectedRequest) { + if (!selectedRequest.isInScope()) { + JOptionPane.showMessageDialog(null, "Selected request is not in scope!", "Warning", JOptionPane.WARNING_MESSAGE); + return false; + } + if ((isEndSet && isStartSet) && endValue < startValue) { + JOptionPane.showMessageDialog(null, "Start value is greater than End value", "Warning", JOptionPane.WARNING_MESSAGE); + return false; + } + return true; + } +} diff --git a/src/main/java/burp/HttpListener.java b/src/main/java/burp/HttpListener.java index a31a4d3..d81101f 100644 --- a/src/main/java/burp/HttpListener.java +++ b/src/main/java/burp/HttpListener.java @@ -11,6 +11,10 @@ import db.ParameterHandler; import db.entities.InputParameter; import db.entities.SessionParameter; +import events.DeferMatchingFinishedEvent; +import events.DeferMatchingFinishedListener; +import events.ItemsAddedEvent; +import events.ItemsAddedListener; import utils.Hashing; import java.util.ArrayList; @@ -31,6 +35,8 @@ public class HttpListener implements HttpHandler { private String messageHash; private int messageId; + private List listeners; + public HttpListener(MontoyaApi api, CrossSessionAudit crossSessionAuditor, CrossContentTypeAudit crossContentTypeAuditor, CrossScopeAudit crossScopeAuditor, HeaderMatchAudit headerMatchAuditor, LongDistanceMatchAudit longDistanceMatchAuditor, @@ -44,18 +50,34 @@ public HttpListener(MontoyaApi api, CrossSessionAudit crossSessionAuditor, Cross this.messageId = 0; this.messageHash = ""; monitoredParameter = new ArrayList<>(); + this.listeners = new ArrayList<>(); + } + + public void removeItemsAddedListener(ItemsAddedListener listener) { + this.listeners.remove(listener); + } + + public void addItemsAddedListener(ItemsAddedListener listener) { + this.listeners.add(listener); + } + + private void fireItemsAddedEvent() { + ItemsAddedEvent event = new ItemsAddedEvent(this); + for (ItemsAddedListener listener : this.listeners) { + listener.onItemsAddedEvent(); + } } @Override public RequestToBeSentAction handleHttpRequestToBeSent(HttpRequestToBeSent requestToBeSent) { - if(detectionIsActive) { + if (detectionIsActive) { // Ignores the probe request as the originate from toolFlag 1024 == Burp Extender (Alternative could be by using the User-Agent header in the request) - if(requestToBeSent.toolSource().isFromTool(ToolType.EXTENSIONS) || requestToBeSent.toolSource().isFromTool(ToolType.REPEATER)) + if (requestToBeSent.toolSource().isFromTool(ToolType.EXTENSIONS) || requestToBeSent.toolSource().isFromTool(ToolType.REPEATER)) return continueWith(requestToBeSent); // Ignore requests out of scope - if(!api.scope().isInScope(requestToBeSent.url())) + if (!api.scope().isInScope(requestToBeSent.url())) return continueWith(requestToBeSent); this.messageHash = Hashing.sha1(requestToBeSent.toByteArray().getBytes()); @@ -63,11 +85,15 @@ public RequestToBeSentAction handleHttpRequestToBeSent(HttpRequestToBeSent reque var reqParsed = this.parseRequest(requestToBeSent); if (reqParsed != null) { - parameterHandler.addParameters(reqParsed.parameterHelpers, messageHash); + List parametersList = parameterHandler.addParameters(reqParsed.parameterHelpers, messageHash); + DBModel.saveBulk(parametersList); + } if (hasActiveSession) { this.monitorSessionParameters(requestToBeSent); } + + fireItemsAddedEvent(); } return continueWith(requestToBeSent); } @@ -79,13 +105,13 @@ public ResponseReceivedAction handleHttpResponseReceived(HttpResponseReceived re if (responseReceived.toolSource().isFromTool(ToolType.EXTENSIONS) || responseReceived.toolSource().isFromTool(ToolType.REPEATER)) return ResponseReceivedAction.continueWith(responseReceived); - var reqParsed = this.parseRequest(responseReceived.initiatingRequest()); - var respParsed = this.parseResponse(responseReceived, responseReceived.initiatingRequest()); - // Ignore requests out of scope - if(!api.scope().isInScope(responseReceived.initiatingRequest().url())) + if (!api.scope().isInScope(responseReceived.initiatingRequest().url())) return ResponseReceivedAction.continueWith(responseReceived); + var reqParsed = this.parseRequest(responseReceived.initiatingRequest()); + var respParsed = this.parseResponse(responseReceived, responseReceived.initiatingRequest()); + var requestDomain = reqParsed.Url.getHost(); // Searching for matching parameters List noiseReeducatedParameters = this.parameterHandler.getRelevant(requestDomain); @@ -93,6 +119,8 @@ public ResponseReceivedAction handleHttpResponseReceived(HttpResponseReceived re List matchesList = matchHandler.addMatches(newMatches); DBModel.saveBulk(matchesList); + fireItemsAddedEvent(); + SessionViewController.updateParameterListInActiveSession(); } return ResponseReceivedAction.continueWith(responseReceived); @@ -135,4 +163,8 @@ public static void setMonitoredParameter(List params) { public static void setLiveMatchingIsActive(boolean isActive) { liveMatchingIsActive = isActive; } + + public static void setHasActiveSession(boolean hasActiveSession) { + HttpListener.hasActiveSession = hasActiveSession; + } } diff --git a/src/main/java/burp/HttpRequest.java b/src/main/java/burp/HttpRequest.java index 7b628be..1d4500a 100644 --- a/src/main/java/burp/HttpRequest.java +++ b/src/main/java/burp/HttpRequest.java @@ -11,10 +11,10 @@ public class HttpRequest { public URL Url; public Collection parameterHelpers; - public HttpRequest(String method, URL url, Collection oldParameters){ + public HttpRequest(String method, URL url, Collection parsedParameterHelper){ this.Method = method; this.Url = url; - this.parameterHelpers = oldParameters; + this.parameterHelpers = parsedParameterHelper; } @Override diff --git a/src/main/java/burp/HttpRequestParser.java b/src/main/java/burp/HttpRequestParser.java index 18f1846..d61760e 100644 --- a/src/main/java/burp/HttpRequestParser.java +++ b/src/main/java/burp/HttpRequestParser.java @@ -42,13 +42,13 @@ public HttpRequest parse(burp.api.montoya.http.message.requests.HttpRequest requ private Collection sortParameters(List params, URL url){ - var parameters2 = new Vector(); + var parameters = new Vector(); for (var p: params) { var burpType = p.type(); ParameterType inferredType; //Map burp param types to my own types - switch(burpType){ + switch (burpType) { case BODY: inferredType = ParameterType.BODY; break; @@ -67,13 +67,12 @@ private Collection sortParameters(List Headers; public URL AssociatedRequestUrl; public int StatusCode; - private String identifier; - - public HttpResponse(int statusCode, String body, String contentType, List headers){ + public HttpResponse(int statusCode, String body, String contentType, List headers) { this.StatusCode = statusCode; this.Body = body; this.ContentType = contentType; this.Headers = headers; - this.identifier = null; } - public HttpResponse(int statusCode, String body, String contentType, List headers, URL requestUrl){ + public HttpResponse(int statusCode, String body, String contentType, List headers, URL requestUrl) { this(statusCode, body, contentType, headers); this.AssociatedRequestUrl = requestUrl; } - - public String getResponseIdentifier(){ - if(identifier == null){ - calculateIdentifier(); - } - return this.identifier; - } - - private void calculateIdentifier(){ - List headerAsString = Headers.stream().map(HttpHeader::toString).collect(Collectors.toList()); - String content = String.join("~", headerAsString); - content += String.format("~BODY~%s", this.Body); - this.identifier = Hashing.getSha512(content); - } - - } diff --git a/src/main/java/burp/HttpResponseParser.java b/src/main/java/burp/HttpResponseParser.java index 7c6c30e..aac26d0 100644 --- a/src/main/java/burp/HttpResponseParser.java +++ b/src/main/java/burp/HttpResponseParser.java @@ -18,38 +18,32 @@ public class HttpResponseParser { private IParser jsonParser; private IParser defaultParser; - public HttpResponseParser(MontoyaApi api){ + public HttpResponseParser(MontoyaApi api) { this.api = api; this.htmlParser = new HtmlParser(this.api); this.jsonParser = new JsonParser(); this.defaultParser = new DefaultParser(); } - public HttpResponse parseResponse(burp.api.montoya.http.message.responses.HttpResponse responseReceived, HttpRequest request){ + public HttpResponse parseResponse(burp.api.montoya.http.message.responses.HttpResponse responseReceived, HttpRequest request) { var raw = responseReceived.toByteArray().getBytes(); - - var resp = this.parseResponseBase(responseReceived, raw, URLExtension.stringToUrl(request.url())); - - // Parse from burp.api request to own request type - HttpRequestParser requestParser = new HttpRequestParser(this.api); - burp.HttpRequest requestParsed = requestParser.parse(request); - - return resp; + return this.parseResponseBase(responseReceived, raw, URLExtension.stringToUrl(request.url())); } + // Takes response, request as bytes, request url - public HttpResponse parseResponseBase(burp.api.montoya.http.message.responses.HttpResponse responseReceived, byte[] raw, URL requestUrl){ + public HttpResponse parseResponseBase(burp.api.montoya.http.message.responses.HttpResponse responseReceived, byte[] raw, URL requestUrl) { var headers = responseReceived.headers(); int statusCode = responseReceived.statusCode(); - //Extracts the Burp inferred Mime-Type as shown in the "MIME Type" column - var contentType = responseReceived.inferredMimeType().toString(); + //Extracts the Burp Stated Mime-Type as shown in the "MIME Type" column + var contentType = responseReceived.statedMimeType().toString(); var body = extractBody(responseReceived, raw); return new HttpResponse(statusCode, body, contentType, headers, requestUrl); } private String extractBody(burp.api.montoya.http.message.responses.HttpResponse responseReceived, byte[] raw){ var bodyOffset = responseReceived.bodyOffset(); - if(bodyOffset > 0 && bodyOffset < raw.length){ + if (bodyOffset > 0 && bodyOffset < raw.length) { var rawBody = Arrays.copyOfRange(raw, bodyOffset, raw.length); return new String(rawBody); } @@ -58,26 +52,26 @@ private String extractBody(burp.api.montoya.http.message.responses.HttpResponse } } - public List getMatches(HttpResponse resp, List inputParameters, String messageHash){ + public List getMatches(HttpResponse resp, List inputParameters, String messageHash) { return getMatchesSynchronized(resp, inputParameters, messageHash); } - private List getMatchesSynchronized(HttpResponse resp, List inputParameters, String messageHash){ + private List getMatchesSynchronized(HttpResponse resp, List inputParameters, String messageHash) { var matches = new Vector(); IParser usedParser; - if(resp.ContentType.equals("HTML")){ + if (resp.ContentType.equals("HTML")) { usedParser = this.htmlParser; } - else if(resp.ContentType.equals("JSON")){ + else if (resp.ContentType.equals("JSON")) { usedParser = this.jsonParser; } - else{ + else { usedParser = this.defaultParser; } usedParser.initialize(resp); - for(var param : inputParameters) { + for (var param : inputParameters) { if (param.isExcludedByNoiseReduction()) { continue; } diff --git a/src/main/java/burp/ParameterType.java b/src/main/java/burp/ParameterType.java index 7bf9e83..228f11c 100644 --- a/src/main/java/burp/ParameterType.java +++ b/src/main/java/burp/ParameterType.java @@ -7,8 +7,8 @@ public enum ParameterType { JSON, OTHER; - public String getName(){ - switch(this){ + public String getName() { + switch (this) { case URL: return "GET"; case BODY: @@ -21,5 +21,4 @@ public String getName(){ return "OTHER"; } } - } \ No newline at end of file diff --git a/src/main/java/burp/PropertiesHandler.java b/src/main/java/burp/PropertiesHandler.java index e51a889..05e45de 100644 --- a/src/main/java/burp/PropertiesHandler.java +++ b/src/main/java/burp/PropertiesHandler.java @@ -135,24 +135,28 @@ public List loadNoiseReductionRules() { List rules = new ArrayList<>(); for (String key : keys) { List rule = this.api.persistence().extensionData().getStringList(key); - // List order of Elements is name, regex, affectsParameterNames, affectsParameterValues, - // affectsHeader, affectsBody, affectsCookie, active, caseSensitive, hash (10 Elements) - String name = rule.get(0); - String regex = rule.get(1); - boolean affectsParameterNames = Boolean.parseBoolean(rule.get(2)); - boolean affectsParameterValues = Boolean.parseBoolean(rule.get(3)); - boolean affectsHeader = Boolean.parseBoolean(rule.get(4)); - boolean affectsBody = Boolean.parseBoolean(rule.get(5)); - boolean affectsCookie = Boolean.parseBoolean(rule.get(6)); - boolean active = Boolean.parseBoolean(rule.get(7)); - boolean caseInsensitive = Boolean.parseBoolean(rule.get(8)); - RuleContainer ruleContainer = new RuleContainer(name, regex, affectsParameterNames, - affectsParameterValues, affectsHeader, affectsBody, affectsCookie, active, caseInsensitive); + RuleContainer ruleContainer = extractRuleContainerFromList(rule); rules.add(ruleContainer); } return rules; } + private RuleContainer extractRuleContainerFromList(List ruleList) { + // List order of Elements is name, regex, affectsParameterNames, affectsParameterValues, + // affectsHeader, affectsBody, affectsCookie, active, caseSensitive, hash (10 Elements) + String name = ruleList.get(0); + String regex = ruleList.get(1); + boolean affectsParameterNames = Boolean.parseBoolean(ruleList.get(2)); + boolean affectsParameterValues = Boolean.parseBoolean(ruleList.get(3)); + boolean affectsHeader = Boolean.parseBoolean(ruleList.get(4)); + boolean affectsBody = Boolean.parseBoolean(ruleList.get(5)); + boolean affectsCookie = Boolean.parseBoolean(ruleList.get(6)); + boolean active = Boolean.parseBoolean(ruleList.get(7)); + boolean caseInsensitive = Boolean.parseBoolean(ruleList.get(8)); + return new RuleContainer(name, regex, affectsParameterNames, + affectsParameterValues, affectsHeader, affectsBody, affectsCookie, active, caseInsensitive); + } + public void setDefaultRulesOnFirstLoad() { if (isFirstTimeLoading && loadNoiseReductionRules().isEmpty()) { ObjectMapper objectMapper = new ObjectMapper(); diff --git a/src/main/java/burp/RegexMatcher.java b/src/main/java/burp/RegexMatcher.java index 63f0768..0d3b82a 100644 --- a/src/main/java/burp/RegexMatcher.java +++ b/src/main/java/burp/RegexMatcher.java @@ -2,7 +2,9 @@ import db.entities.InputParameter; import db.entities.InputValue; +import db.entities.Url; import gui.container.RuleContainer; +import org.eclipse.jetty.http.HttpTester; import java.util.ArrayList; import java.util.List; @@ -66,73 +68,69 @@ public static void excludeInputValue(InputValue inputValue) { } public static void excludeParametersForSingleRule(List parameters, RuleContainer rule) { - for (InputParameter parameter : parameters) { - for (InputValue inputValue : parameter.getOccurrenceEntities()) { - Pattern pattern = Pattern.compile(rule.getRegex(), rule.isCaseInsensitive() ? Pattern.CASE_INSENSITIVE : 0); - if (rule.affectsParameterNames()) { - // Apply the rule on parameter names - if (rule.affectsQueryString() && parameter.getType().equals("GET")) { - Matcher matcher = pattern.matcher(parameter.getName()); - if (matcher.find()) { - parameter.setExcludedByNoiseReduction(rule.isActive()); - } - } else if (rule.affectsBody() && parameter.getType().equals("POST")) { - Matcher matcher = pattern.matcher(parameter.getName()); - if (matcher.find()) { - parameter.setExcludedByNoiseReduction(rule.isActive()); - } - } else if (rule.affectsCookie() && parameter.getType().equals("COOKIE")) { - Matcher matcher = pattern.matcher(parameter.getName()); - if (matcher.find()) { - parameter.setExcludedByNoiseReduction(rule.isActive()); - } - } - } + Pattern pattern = Pattern.compile(rule.getRegex(), rule.isCaseInsensitive() ? Pattern.CASE_INSENSITIVE : 0); + + // Process Parameters from the Parameter list + parameters.parallelStream() + .forEach(parameter -> { + processParameter(parameter, pattern, rule); if (rule.affectsParameterValues()) { - // Apply the rule on parameter values - if (rule.affectsQueryString() && inputValue.getType().equals("GET")) { - Matcher matcher = pattern.matcher(inputValue.getValue()); - if (matcher.find()) { - inputValue.setExcludedByNoiseReduction(rule.isActive()); - } - } else if (rule.affectsBody() && inputValue.getType().equals("POST")) { - Matcher matcher = pattern.matcher(inputValue.getValue()); - if (matcher.find()) { - inputValue.setExcludedByNoiseReduction(rule.isActive()); - } - } else if (rule.affectsCookie() && inputValue.getType().equals("COOKIE")) { - Matcher matcher = pattern.matcher(inputValue.getValue()); - if (matcher.find()) { - inputValue.setExcludedByNoiseReduction(rule.isActive()); - } - } + parameter.getOccurrenceEntities().parallelStream().forEach(inputValue -> checkWhichParameterValueIsAffected(rule, pattern, inputValue)); } + }); + } + + private static void processParameter(InputParameter parameter, Pattern pattern, RuleContainer rule) { + if (rule.affectsParameterNames()) { + // Apply the rule on parameter names + if (rule.affectsQueryString() && parameter.getType().equals("GET")) { + Matcher matcher = pattern.matcher(parameter.getName()); + if (matcher.find()) { + parameter.setExcludedByNoiseReduction(rule.isActive()); + } + } else if (rule.affectsBody() && parameter.getType().equals("POST")) { + Matcher matcher = pattern.matcher(parameter.getName()); + if (matcher.find()) { + parameter.setExcludedByNoiseReduction(rule.isActive()); + } + } else if (rule.affectsCookie() && parameter.getType().equals("COOKIE")) { + Matcher matcher = pattern.matcher(parameter.getName()); + if (matcher.find()) { + parameter.setExcludedByNoiseReduction(rule.isActive()); + } } } + if (rule.affectsParameterValues()) { + parameter.getOccurrenceEntities().parallelStream().forEach(inputValue -> checkWhichParameterValueIsAffected(rule, pattern, inputValue)); + } } public static void excludeInputValuesForSingleRule(List inputValues, RuleContainer rule) { for (InputValue inputValue : inputValues) { Pattern pattern = Pattern.compile(rule.getRegex(), rule.isCaseInsensitive() ? Pattern.CASE_INSENSITIVE : 0); if (rule.affectsParameterValues()) { - // Apply the rule on parameter values - if (rule.affectsQueryString() && inputValue.getType().equals("GET")) { - Matcher matcher = pattern.matcher(inputValue.getValue()); - if (matcher.find()) { - inputValue.setExcludedByNoiseReduction(rule.isActive()); - } - } else if (rule.affectsBody() && inputValue.getType().equals("POST")) { - Matcher matcher = pattern.matcher(inputValue.getValue()); - if (matcher.find()) { - inputValue.setExcludedByNoiseReduction(rule.isActive()); - } - } else if (rule.affectsCookie() && inputValue.getType().equals("COOKIE")) { - Matcher matcher = pattern.matcher(inputValue.getValue()); - if (matcher.find()) { - inputValue.setExcludedByNoiseReduction(rule.isActive()); - } - } + checkWhichParameterValueIsAffected(rule, pattern, inputValue); + } + } + } + + private static void checkWhichParameterValueIsAffected(RuleContainer rule, Pattern pattern, InputValue inputValue) { + // Apply the rule on parameter values + if (rule.affectsQueryString() && inputValue.getType().equals("GET")) { + Matcher matcher = pattern.matcher(inputValue.getValue()); + if (matcher.find()) { + inputValue.setExcludedByNoiseReduction(rule.isActive()); + } + } else if (rule.affectsBody() && inputValue.getType().equals("POST")) { + Matcher matcher = pattern.matcher(inputValue.getValue()); + if (matcher.find()) { + inputValue.setExcludedByNoiseReduction(rule.isActive()); + } + } else if (rule.affectsCookie() && inputValue.getType().equals("COOKIE")) { + Matcher matcher = pattern.matcher(inputValue.getValue()); + if (matcher.find()) { + inputValue.setExcludedByNoiseReduction(rule.isActive()); } } } diff --git a/src/main/java/burp/RetroactiveParser.java b/src/main/java/burp/RetroactiveParser.java new file mode 100644 index 0000000..f36fc83 --- /dev/null +++ b/src/main/java/burp/RetroactiveParser.java @@ -0,0 +1,119 @@ +package burp; + +import burp.api.montoya.MontoyaApi; +import burp.api.montoya.http.message.requests.HttpRequest; +import burp.api.montoya.proxy.ProxyHttpRequestResponse; +import controller.QueryViewController; +import db.DBModel; +import db.ParameterHandler; +import gui.ProgressDialog; +import utils.Hashing; +import utils.Logger; + +import javax.swing.*; +import java.beans.PropertyChangeEvent; +import java.beans.PropertyChangeListener; +import java.util.*; +import java.util.List; + +public class RetroactiveParser implements PropertyChangeListener { + + private MontoyaApi api; + private HttpRequestParser parser; + private ParameterHandler parameterHandler; + + private QueryViewController queryViewController; + private ParseTask task; + private ProgressDialog progressDialog; + private int historySize; + + public RetroactiveParser(MontoyaApi api, ParameterHandler parameterHandler, QueryViewController queryViewController) { + this.api = api; + this.parser = new HttpRequestParser(this.api); + this.parameterHandler = parameterHandler; + this.queryViewController = queryViewController; + } + + public void init(int startValue, int endValue) { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + historySize = endValue - startValue; + progressDialog = new ProgressDialog("Parsing Parameters..."); + progressDialog.init(); + } + }); + task = new ParseTask(startValue, endValue); + task.addPropertyChangeListener(this); + task.execute(); + } + + @Override + public void propertyChange(PropertyChangeEvent evt) { + if ("progress".equals(evt.getPropertyName())) { + int progress = (Integer) evt.getNewValue(); + double historyProgress = (((double) progress * historySize / 100) - 1); + progressDialog.updateProgressBarValue(progress); + progressDialog.appendTaskOutput((String.format( + "Completed %d/%d of Burp history.\n", (int) historyProgress, historySize))); + } + } + + class ParseTask extends SwingWorker { + + private int startValue; + private int endValue; + + public ParseTask(int startValue, int endValue) { + this.startValue = startValue; + this.endValue = endValue; + this.execute(); + } + + @Override + protected Void doInBackground() { + try { + List proxyHttpRequestResponseList = api.proxy().history().subList(startValue - 1, endValue); + List requestList = new ArrayList<>(proxyHttpRequestResponseList.stream().map(ProxyHttpRequestResponse::finalRequest).toList()); + + int progress = 0; + setProgress(0); + int listSize = requestList.size(); + + List parametersToSave = new ArrayList<>(); + + for (int i = 0; i < requestList.size(); i++) { + HttpRequest request = requestList.get(i); + + // Ignore requests out of scope + if (!api.scope().isInScope(request.url())) { + continue; + } + + String messageHash = Hashing.sha1(request.toByteArray().getBytes()); + + progress = (int) ((i + 1) / (double) listSize * 100); + setProgress(progress); + + var reqParsed = parser.parse(request); + if (reqParsed != null) { + parametersToSave.addAll(parameterHandler.addParametersThreaded(reqParsed.parameterHelpers, messageHash)); + } + } + DBModel.saveBulk(parametersToSave); + return null; + } catch (Exception e) { + Logger.getInstance().logToError(Arrays.toString(e.getStackTrace())); + e.printStackTrace(); + return null; + } + } + @Override + protected void done() { + queryViewController.updateParameters(); + progressDialog.updateDialogDefaultCloseOperation(JDialog.HIDE_ON_CLOSE); + progressDialog.updateProgressBarValue(100); + progressDialog.setTaskOutputText("Completed!\n"); + } + } +} diff --git a/src/main/java/controller/NoiseReductionController.java b/src/main/java/controller/NoiseReductionController.java index 10db1a7..0d6f564 100644 --- a/src/main/java/controller/NoiseReductionController.java +++ b/src/main/java/controller/NoiseReductionController.java @@ -38,45 +38,15 @@ public NoiseReductionController(NoiseReductionView view, NoiseReductionModel mod @Override public void actionPerformed(ActionEvent actionEvent) { if (actionEvent.getSource().equals(view.newRuleButton)) { - this.isEditing = false; - view.ruleList.getSelectionModel().clearSelection(); - view.nameTextField.requestFocus(); - clearAllTextFields(); - clearAllCheckBoxes(); - view.editorPanel.setBorder(BorderFactory.createTitledBorder("Rule editor - *Adding new rule*")); + setEditorInNewRuleMode(); } else if (actionEvent.getSource().equals(view.saveButton)) { - RuleContainer rule = getSelectedValuesFromView(); - view.regexInvalidLabel.setVisible(false); - if (!isRegexValid(rule.getRegex())) { - view.regexInvalidLabel.setVisible(true); - return; - } - if (isEditing) { - saveEditedRule(rule.getName(), rule.getRegex(), rule.affectsParameterNames(), rule.affectsParameterValues(), - rule.affectsQueryString(), rule.affectsBody(), rule.affectsCookie(), rule.isActive(), rule.isCaseInsensitive()); - } else { - saveNewRule(rule.getName(), rule.getRegex(), rule.affectsParameterNames(), rule.affectsParameterValues(), - rule.affectsQueryString(), rule.affectsBody(), rule.affectsCookie(), rule.isActive(), rule.isCaseInsensitive()); - } + saveEditorContent(); } else if (actionEvent.getSource().equals(view.deleteButton)) { - int index = view.ruleList.getSelectedIndex(); - if (index != -1) { - DefaultListModel listModel = (DefaultListModel) view.ruleList.getModel(); - RuleContainer selectedRuleContainer = view.ruleList.getSelectedValue(); - listModel.removeElement(selectedRuleContainer); - model.deleteRuleInState(selectedRuleContainer); - fireRuleContainerChanged(selectedRuleContainer, true); - clearAllTextFields(); - clearAllCheckBoxes(); - } + deleteSelectedRule(); } else if (actionEvent.getSource().equals(view.activatedCheckBox)) { - if (isEditing) { - RuleContainer rule = getSelectedValuesFromView(); - updateActiveStatus(rule.isActive()); - } + changeActivatedStatus(); } else if (actionEvent.getSource().equals(view.purgeAndRematchButton)) { - model.purgeDbAndStartRematch(); - auditFindingView.clearDataAndFields(); + purgeAndRematch(); } } @@ -114,6 +84,56 @@ private void clearAllCheckBoxes() { view.activatedCheckBox.setSelected(false); } + private void setEditorInNewRuleMode() { + this.isEditing = false; + view.ruleList.getSelectionModel().clearSelection(); + view.nameTextField.requestFocus(); + clearAllTextFields(); + clearAllCheckBoxes(); + view.editorPanel.setBorder(BorderFactory.createTitledBorder("Rule editor - *Adding new rule*")); + } + + private void saveEditorContent() { + RuleContainer rule = getSelectedValuesFromView(); + view.regexInvalidLabel.setVisible(false); + if (!isRegexValid(rule.getRegex())) { + view.regexInvalidLabel.setVisible(true); + return; + } + if (isEditing) { + saveEditedRule(rule.getName(), rule.getRegex(), rule.affectsParameterNames(), rule.affectsParameterValues(), + rule.affectsQueryString(), rule.affectsBody(), rule.affectsCookie(), rule.isActive(), rule.isCaseInsensitive()); + } else { + saveNewRule(rule.getName(), rule.getRegex(), rule.affectsParameterNames(), rule.affectsParameterValues(), + rule.affectsQueryString(), rule.affectsBody(), rule.affectsCookie(), rule.isActive(), rule.isCaseInsensitive()); + } + } + + private void deleteSelectedRule() { + int index = view.ruleList.getSelectedIndex(); + if (index != -1) { + DefaultListModel listModel = (DefaultListModel) view.ruleList.getModel(); + RuleContainer selectedRuleContainer = view.ruleList.getSelectedValue(); + listModel.removeElement(selectedRuleContainer); + model.deleteRuleInState(selectedRuleContainer); + fireRuleContainerChanged(selectedRuleContainer, true); + clearAllTextFields(); + clearAllCheckBoxes(); + } + } + + private void changeActivatedStatus() { + if (isEditing) { + RuleContainer rule = getSelectedValuesFromView(); + updateActiveStatus(rule.isActive()); + } + } + + private void purgeAndRematch() { + model.purgeDbAndStartRematch(); + auditFindingView.clearDataAndFields(); + } + private void setValuesInEditor() { RuleContainer ruleContainer = view.ruleList.getSelectedValue(); if (ruleContainer != null) { @@ -168,7 +188,12 @@ private void saveEditedRule(String name, String regex, boolean affectsParamName, int index = view.ruleList.getSelectedIndex(); if (ruleContainer != null && index != -1) { String oldRuleHash = String.valueOf(ruleContainer.getHash()); + // Deactivate the old rule and apply deactivation + ruleContainer.setActive(false); + fireRuleContainerChanged(ruleContainer, false); + // Set properties of new rule ruleContainer.updateValues(name, regex, affectsParamName, affectsParamValue, affectsHeader, affectsBody, affectsCookie, active, caseInsensitive); + // Apply changes model.updateRuleInState(ruleContainer, oldRuleHash); updateListPanel(); fireRuleContainerChanged(ruleContainer, false); diff --git a/src/main/java/controller/QueryViewController.java b/src/main/java/controller/QueryViewController.java index 60e893a..77c4079 100644 --- a/src/main/java/controller/QueryViewController.java +++ b/src/main/java/controller/QueryViewController.java @@ -1,19 +1,15 @@ package controller; import burp.ContainerConverter; -import burp.RegexMatcher; import burp.api.montoya.MontoyaApi; import db.DBModel; import db.MatchHandler; import db.ParameterHandler; import db.entities.InputParameter; import db.entities.MatchValue; -import db.entities.ParameterMatch; -import events.RuleContainerEvent; -import events.RuleContainerListener; +import events.*; import gui.QueryView; import gui.container.*; -import javafx.collections.ListChangeListener; import model.QueryViewModel; import utils.MessageHashToProxyId; @@ -22,14 +18,12 @@ import javax.swing.event.ListSelectionListener; import java.awt.*; import java.awt.event.*; -import java.util.ArrayList; -import java.util.Comparator; import java.util.List; -import java.util.Vector; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.IntStream; -public class QueryViewController implements ActionListener, ListSelectionListener, RuleContainerListener { +public class QueryViewController implements ActionListener, ListSelectionListener, RuleContainerListener, DeferMatchingFinishedListener, ItemsAddedListener { private MontoyaApi api; private QueryView view; @@ -76,6 +70,7 @@ public void focusLost(FocusEvent focusEvent) { } }); + // MouseListener on parameterJList for contextmenu to send selected parameter to session definition view.parameterJList.addMouseListener(new MouseAdapter() { @Override public void mouseClicked(MouseEvent e) { @@ -88,54 +83,19 @@ public void mouseClicked(MouseEvent e) { }); view.sendToSessionDef.addActionListener(this); - - // Listener for InputParameter and Matchlist. If an item gets added to any of these list, the parameterlist needs - // to be updated - this.parameterHandler.observableInputParameterList.addListener(new ListChangeListener() { - @Override - public void onChanged(Change change) { - while (change.next()) { - if (change.wasAdded()) { - updateParameters(); - } - } - } - }); - - this.matchHandler.observableParameterMatchList.addListener(new ListChangeListener() { - @Override - public void onChanged(Change change) { - while (change.next()) { - if (change.wasAdded()) { - updateParameters(); - } - } - } - }); } // ActionListener for sort-label, search-field and right-click menu @Override public void actionPerformed(ActionEvent actionEvent) { - if (actionEvent.getSource().equals(view.sortByLabel)) { // Change text on click for Ascending/Descending order - if (view.sortByLabel.getText().equals("Desc ↓")) { - view.sortByLabel.setText("Asc ↑"); - filterParameterList(); - } else { - view.sortByLabel.setText("Desc ↓"); - filterParameterList(); - } + if (actionEvent.getSource().equals(view.sortByLabel)) { // change text on click for Ascending/Descending order + sortParameterListAscDescLabelAction(); } else if (actionEvent.getSource().equals(view.filterPicker)) { - JComboBox cb = (JComboBox)actionEvent.getSource(); - model.setSelectedText((String) cb.getSelectedItem()); - filterParameterList(); + sortParameterListFilterPickerAction(actionEvent); } else if (actionEvent.getSource().equals(view.sendToSessionDef)) { - var param = (ParameterContainer) view.parameterJList.getSelectedValue(); - String paramName = param.getName(); - sessionViewController.addToSessionDefList(new SessionDefContainer(paramName, param.getType())); + sendToSessionDefinitionAction(); } else if (actionEvent.getSource().equals(view.hideExcludedParamsCheckBox)) { - this.hideParamsExcludedByNoiseReduction = view.hideExcludedParamsCheckBox.isSelected(); - view.parameterJList.setListData(new Vector<>(filterExcludedParams(containerConverter.parameterToContainer(this.parameterHandler.observableInputParameterList)))); + hideExcludedParametersAction(); } } @@ -144,53 +104,92 @@ public void actionPerformed(ActionEvent actionEvent) { public void valueChanged(ListSelectionEvent listSelectionEvent) { if (!listSelectionEvent.getValueIsAdjusting()) { if (listSelectionEvent.getSource().equals(view.parameterMatchJList)) { - var matchEntity = (ParameterMatchContainer) view.parameterMatchJList.getSelectedValue(); - if (matchEntity != null) { - var url = matchEntity.getUrl(); - var value = matchEntity.getValue(); - var messageId = matchEntity.getMessageId(); - var messageHash = matchEntity.getMessageHash(); - updateSelectedMessageId(String.valueOf(messageId)); - int historyId = getHistoryId(messageHash); - view.httpRequestEditor.setRequest(this.api.proxy().history().get(historyId).finalRequest()); - view.httpResponseEditor.setResponse(this.api.proxy().history().get(historyId).originalResponse()); - renderMatchEntries(model.matchValueEntityList.stream().filter(e -> e.getUrl().equals(url)).filter(e -> e.getValue().equals(value)).distinct().toList()); - } + showDataForSelectedParameterMatch(); } else if (listSelectionEvent.getSource().equals(view.parameterValueJList)) { - var param = (InputParameterContainer) view.parameterValueJList.getSelectedValue(); - if (param != null) { - var messageId = param.getMessageId(); - var messageHash = param.getMessageHash(); - int historyId = getHistoryId(messageHash); - updateSelectedMessageId(String.valueOf(messageId)); - view.httpResponseEditor.setResponse(this.api.proxy().history().get(historyId).originalResponse()); - view.httpRequestEditor.setRequest(this.api.proxy().history().get(historyId).finalRequest()); - } - view.parameterMatchJList.clearSelection(); - clearMatchValueContainer(); + showDataForSelectedParameterValue(); } else if (listSelectionEvent.getSource().equals(view.parameterJList)) { - updateSelectedMessageId(""); - view.parameterValueJList.clearSelection(); - clearParameterMatchContainer(); - view.httpRequestEditor.setRequest(null); - view.httpResponseEditor.setResponse(null); - var param = (ParameterContainer) view.parameterJList.getSelectedValue(); - if (param != null) { - parameterSelectedEvent(); - } + showDataForSelectedParameter(); } } } + private void sortParameterListAscDescLabelAction() { + if (view.sortByLabel.getText().equals("Desc ↓")) { + view.sortByLabel.setText("Asc ↑"); + filterParameterList(); + } else { + view.sortByLabel.setText("Desc ↓"); + filterParameterList(); + } + } + + private void sortParameterListFilterPickerAction(ActionEvent actionEvent) { + JComboBox cb = (JComboBox)actionEvent.getSource(); + model.setSelectedText((String) cb.getSelectedItem()); + filterParameterList(); + } + + private void sendToSessionDefinitionAction() { + var param = (ParameterContainer) view.parameterJList.getSelectedValue(); + String paramName = param.getName(); + sessionViewController.addToSessionDefList(new SessionDefContainer(paramName, param.getType())); + } + + private void hideExcludedParametersAction() { + this.hideParamsExcludedByNoiseReduction = view.hideExcludedParamsCheckBox.isSelected(); + view.parameterJList.setListData(new Vector<>(filterExcludedParams(containerConverter.parameterToContainer(this.parameterHandler.inputParameterStorage.values().stream().toList())))); + } + + private void showDataForSelectedParameter() { + updateSelectedMessageId(""); + view.parameterValueJList.clearSelection(); + clearParameterMatchContainer(); + view.httpRequestEditor.setRequest(null); + view.httpResponseEditor.setResponse(null); + var param = (ParameterContainer) view.parameterJList.getSelectedValue(); + if (param != null) { + parameterSelectedEvent(); + } + } + + private void showDataForSelectedParameterValue() { + var param = (InputParameterContainer) view.parameterValueJList.getSelectedValue(); + if (param != null) { + var messageId = param.getMessageId(); + var messageHash = param.getMessageHash(); + int historyId = getHistoryId(messageHash); + updateSelectedMessageId(String.valueOf(messageId)); + view.httpResponseEditor.setResponse(this.api.proxy().history().get(historyId).originalResponse()); + view.httpRequestEditor.setRequest(this.api.proxy().history().get(historyId).finalRequest()); + } + view.parameterMatchJList.clearSelection(); + clearMatchValueContainer(); + } + + private void showDataForSelectedParameterMatch() { + var matchContainer = (ParameterMatchContainer) view.parameterMatchJList.getSelectedValue(); + if (matchContainer != null) { + var url = matchContainer.getUrl(); + var value = matchContainer.getValue(); + var messageId = matchContainer.getMessageId(); + var messageHash = matchContainer.getMessageHash(); + updateSelectedMessageId(String.valueOf(messageId)); + int historyId = getHistoryId(messageHash); + view.httpRequestEditor.setRequest(this.api.proxy().history().get(historyId).finalRequest()); + view.httpResponseEditor.setResponse(this.api.proxy().history().get(historyId).originalResponse()); + renderMatchEntries(model.matchValueEntityList.stream().filter(e -> e.getUrl().equals(url)).filter(e -> e.getValue().equals(value)).distinct().toList()); + } + } + public void renderMatchEntries(List entriesToDisplay) { view.matchValueJList.setListData(containerConverter.entryOccurrenceToContainer(entriesToDisplay)); view.rightMidPanel.repaint(); view.rightMidPanel.revalidate(); } - private void updateParameters() { + public void updateParameters() { Comparator comparator = getSortSettings(); - List parameters = this.parameterHandler.observableInputParameterList; + List parameters = this.parameterHandler.inputParameterStorage.values().stream().toList(); Vector parameterContainerVector; parameterContainerVector = new Vector<>( containerConverter.parameterToContainer(parameters.stream().toList()) @@ -273,6 +272,7 @@ private void parameterSelectedEvent() { model.loadListData(paramName, paramType); view.parameterValueJList.setListData(containerConverter.parameterOccurrenceToContainer(model.occurrenceEntityList)); + if (!model.parameterMatchEntityList.isEmpty()) { view.parameterMatchJList.setListData(containerConverter.matchOccurrenceToContainer(model.parameterMatchEntityList)); view.parameterMatchJList.addListSelectionListener(this); @@ -321,6 +321,21 @@ private int getHistoryId(String hash) { return messageHashToProxyId.calculateId(hash) - 1; } + public void clearParameterList() { + this.view.parameterJList.setListData(new Vector<>()); + } + + public void clearDataAndView() { + clearParameterList(); + clearMatchValueContainer(); + clearParameterMatchContainer(); + clearParamValueContainer(); + updateSelectedMessageId(""); + model.clearAllData(); + view.cypherQueryField.setText(""); + view.searchField.setText(""); + } + @Override public void onRuleChangeEvent(RuleContainerEvent event) { RuleContainer ruleContainer = event.getRuleContainer(); @@ -328,7 +343,20 @@ public void onRuleChangeEvent(RuleContainerEvent event) { ruleContainer.setActive(false); } this.parameterHandler.updateParameterExclusion(ruleContainer); - DBModel.saveBulkParameters(this.parameterHandler.observableInputParameterList); + List bulkSaveList = new ArrayList<>(this.parameterHandler.inputParameterStorage.values()); + DBModel.saveBulk(bulkSaveList); + List bulkSaveInputValues = new ArrayList<>(this.parameterHandler.inputValueStorage.values()); + DBModel.saveBulk(bulkSaveInputValues); + updateParameters(); + } + + @Override + public void onDeferMatchingFinishedEvent() { + updateParameters(); + } + + @Override + public void onItemsAddedEvent() { updateParameters(); } @@ -342,7 +370,7 @@ public void textValueChanged(TextEvent event) { var newValue = ((TextField)event.getSource()).getText(); if (newValue == null | newValue.isEmpty()) { //Show all as no value entered - view.parameterJList.setListData(new Vector<>(filterExcludedParams(containerConverter.parameterToContainer(view.parameterHandler.observableInputParameterList.stream().toList())))); + view.parameterJList.setListData(new Vector<>(filterExcludedParams(containerConverter.parameterToContainer(view.parameterHandler.inputParameterStorage.values().stream().toList())))); filterParameterList(); view.hideExcludedParamsCheckBox.setEnabled(true); } @@ -359,7 +387,7 @@ else if (newValue.length() > oldValue.length()) { } else { var newParams = new Vector(); - var currentParams = filterExcludedParams(containerConverter.parameterToContainer(view.parameterHandler.observableInputParameterList.stream().toList())); + var currentParams = filterExcludedParams(containerConverter.parameterToContainer(view.parameterHandler.inputParameterStorage.values().stream().toList())); for (var paramContainer : currentParams) { if (paramContainer.getName().contains(newValue)) { newParams.add(paramContainer); @@ -374,18 +402,4 @@ else if (newValue.length() > oldValue.length()) { } } - public void clearParameterList() { - this.view.parameterJList.setListData(new Vector<>()); - } - - public void clearDataAndView() { - clearParameterList(); - clearMatchValueContainer(); - clearParameterMatchContainer(); - clearParamValueContainer(); - updateSelectedMessageId(""); - model.clearAllData(); - view.cypherQueryField.setText(""); - view.searchField.setText(""); - } } diff --git a/src/main/java/controller/SessionViewController.java b/src/main/java/controller/SessionViewController.java index ac33e64..3a4cc91 100644 --- a/src/main/java/controller/SessionViewController.java +++ b/src/main/java/controller/SessionViewController.java @@ -10,7 +10,10 @@ import db.DBModel; import db.MatchHandler; import db.ParameterHandler; -import db.entities.*; +import db.entities.InputValue; +import db.entities.ParameterMatch; +import db.entities.Session; +import db.entities.SessionParameter; import events.DeferMatchingFinishedListener; import gui.SessionView; import gui.container.*; @@ -57,7 +60,7 @@ private void registerEventHandler() { view.saveSessionDefinitionButton.addActionListener(this); view.changeSessionNameButton.addActionListener(this); sessionJList.addListSelectionListener(this); - sessionSpecificParameterJList.addListSelectionListener(this); + sessionSpecificParameterMatchesJList.addListSelectionListener(this); matchJList.addListSelectionListener(this); } @@ -84,35 +87,20 @@ private void loadSessions() { @Override public void actionPerformed(ActionEvent actionEvent) { if (actionEvent.getSource().equals(view.removeFromSessionDefButton)) { - DefaultListModel model = (DefaultListModel) sessionDefJList.getModel(); - model.removeElement(sessionDefJList.getSelectedValue()); + removeFromSessionDefinitionAction(); } else if (actionEvent.getSource().equals(view.saveSessionDefinitionButton)) { // Check if Sessions have been defined previously, if yes delete everything related to them + DefaultListModel listModel = (DefaultListModel) sessionDefJList.getModel(); + if (listModel.isEmpty()) { + clearDataAndView(); + return; + } if (sessionCounter != 0) { - sessionCounter = 0; - ((DefaultListModel) sessionJList.getModel()).clear(); - model.inputValuesFromSessionDefList.clear(); - model.parameterValuesAndNames.clear(); - model.helpers.clear(); - model.deleteSessionsInDB(); - sessionTable.clear(); + clearDataForNewSessionDefinition(); } - sessionCounter++; - getRelevantInformationForSessionDefinition(); - List sessions = model.identifyExistingSessions(); - createSessionParametersFromIdentified(sessions); + createSessionFromDefinition(); } else if (actionEvent.getSource().equals(view.changeSessionNameButton)) { - String newName = view.sessionNameTextField.getText(); - // If the selected session that's name get changed is the last session of the list, change activeSessionName - // in Model, ParameterHandler and MatchHandler - if (model.getSelectedSession().getName().equals(model.getActiveSessionName())) { - model.setActiveSessionName(newName); - parameterHandler.setSessionName(newName); - matchHandler.setSessionName(newName); - model.helpers.clear(); - } - crossSessionAudit.sessionRename(model.getSelectedSession().getName(), newName); - changeSessionName(newName); + changeSessionNameAction(); } } @@ -120,40 +108,79 @@ public void actionPerformed(ActionEvent actionEvent) { public void valueChanged(ListSelectionEvent listSelectionEvent) { if (!listSelectionEvent.getValueIsAdjusting()) { if (listSelectionEvent.getSource().equals(sessionJList)) { - var sessionContainer = (SessionContainer) sessionJList.getSelectedValue(); - if (sessionContainer != null) { - var sessionName = sessionContainer.getName(); - view.sessionNameTextField.setText(sessionName); - Session selected = SessionViewModel.sessionTable.get(sessionName); - model.setSelectedSession(sessionContainer); - model.setSelectedSessionIndex(sessionJList.getSelectedIndex()); - addToParamMonitorContainer(selected); - setSessionSpecificParameters(sessionName); - view.clearMatchLists(); - } - } else if (listSelectionEvent.getSource().equals(SessionView.sessionSpecificParameterJList)) { - var paramContainer = (SessionParameterContainer) SessionView.sessionSpecificParameterJList.getSelectedValue(); - if (paramContainer != null) { - var paramName = paramContainer.getName(); - var type = paramContainer.getType(); - var value = paramContainer.getValue(); - var session = model.getSelectedSession().getName(); - view.clearMatchLists(); - renderMatchEntries(type, value, session); - setCypherQueryTest(paramName); - } + showDataForSelectedSession(); + } else if (listSelectionEvent.getSource().equals(SessionView.sessionSpecificParameterMatchesJList)) { + showDataForSelectedMatchDuringSession(); } else if (listSelectionEvent.getSource().equals(matchJList)) { - var parameterMatchContainer = (ParameterMatchContainer) matchJList.getSelectedValue(); - if (parameterMatchContainer != null) { - var value = parameterMatchContainer.getValue(); - var url = parameterMatchContainer.getUrl(); - var session = model.getSelectedSession().getName(); - renderMatchInfo(value, url, session); - } + showParameterMatchDetails(); } } } + private void removeFromSessionDefinitionAction() { + DefaultListModel model = (DefaultListModel) sessionDefJList.getModel(); + model.removeElement(sessionDefJList.getSelectedValue()); + } + + private void createSessionFromDefinition() { + sessionCounter++; + getRelevantInformationForSessionDefinition(); + List sessions = model.identifyExistingSessions(); + createSessionParametersFromIdentified(sessions); + } + + private void changeSessionNameAction() { + String newName = view.sessionNameTextField.getText(); + // If the selected session that's name get changed is the last session of the list, change activeSessionName + // in Model, ParameterHandler and MatchHandler + if (model.getSelectedSession().getName().equals(model.getActiveSessionName())) { + model.setActiveSessionName(newName); + parameterHandler.setSessionName(newName); + matchHandler.setSessionName(newName); + model.helpers.clear(); + } + crossSessionAudit.sessionRename(model.getSelectedSession().getName(), newName); + changeSessionName(newName); + } + + private void showDataForSelectedSession() { + var sessionContainer = (SessionContainer) sessionJList.getSelectedValue(); + if (sessionContainer != null) { + var sessionName = sessionContainer.getName(); + view.sessionNameTextField.setText(sessionName); + Session selected = SessionViewModel.sessionTable.get(sessionName); + model.setSelectedSession(sessionContainer); + model.setSelectedSessionIndex(sessionJList.getSelectedIndex()); + addToParamMonitorContainer(selected); + setSessionSpecificParameters(sessionName); + view.clearMatchLists(); + } + } + + private void showDataForSelectedMatchDuringSession() { + var paramContainer = (SessionParameterContainer) SessionView.sessionSpecificParameterMatchesJList.getSelectedValue(); + if (paramContainer != null) { + var paramName = paramContainer.getName(); + var type = paramContainer.getType(); + var value = paramContainer.getValue(); + var session = model.getSelectedSession().getName(); + view.clearMatchLists(); + renderMatchEntries(type, value, session); + setCypherQueryTest(paramName); + } + } + + private void showParameterMatchDetails() { + var parameterMatchContainer = (ParameterMatchContainer) matchJList.getSelectedValue(); + if (parameterMatchContainer != null) { + var value = parameterMatchContainer.getValue(); + var url = parameterMatchContainer.getUrl(); + var session = model.getSelectedSession().getName(); + renderMatchInfo(value, url, session); + } + } + + public void addToSessionDefList(SessionDefContainer container) { DefaultListModel model = (DefaultListModel) sessionDefJList.getModel(); model.addElement(container); @@ -248,7 +275,7 @@ public int compare(SessionParameter p1, SessionParameter t1) { model.updateSessionInformation(newSession, sessionName, sessionParameters); model.changeDatabaseEntriesAccordingToSession(sessionName, lowestId, highestId); this.identifyAuditFindings(sessionName); - DBModel.saveSession(newSession); + DBModel.saveEntity(newSession); } } @@ -294,7 +321,7 @@ public void updateIdForLastSession(int id) { sessionContainer.updateRange(id); Session session = sessionTable.get(sessionContainer.getName()); session.setHighestHistoryId(id); - DBModel.saveSession(session); + DBModel.saveEntity(session); view.sessionsPanel.revalidate(); view.sessionsPanel.repaint(); } @@ -329,7 +356,7 @@ public static void updateParameterListInActiveSession() { // Update only if the last Session in the list is selected if (model.getSelectedSessionIndex() + 1 == sessionJList.getModel().getSize()) { // get current list of parameters in sessions - ListModel parameterList = sessionSpecificParameterJList.getModel(); + ListModel parameterList = sessionSpecificParameterMatchesJList.getModel(); Vector parameterContainerVector = new Vector<>( model.containerConverter.parameterToContainerSessionDef(matchHandler.parameterMatchStorage.values().stream().distinct().toList(), model.getActiveSessionName(), getParamNamesFromSessionDefList())); @@ -389,7 +416,7 @@ public static void createSessionFromMonitor(List parameters sessionJList.revalidate(); sessionJList.repaint(); model.updateSessionInformation(newSession, sessionName, newParameters); - DBModel.saveSession(newSession); + DBModel.saveEntity(newSession); } public void setSessionSpecificParameters(String sessionName) { @@ -408,15 +435,10 @@ public void setSessionSpecificParameters(String sessionName) { } public List getSessionSpecificParameters(String sessionName) { - List parameters = new ArrayList<>(); Map values = Collections.singletonMap("sessionName", sessionName); String query = "MATCH (m:ParameterMatch {session: $sessionName}) RETURN m"; Result queryResult = DBModel.query(query, values); - Iterator> resultIterator = queryResult.queryResults().iterator(); - while (resultIterator.hasNext()) { - Map result = resultIterator.next(); - parameters.add((ParameterMatch) result.get("m")); - } + List parameters = CypherQueryHandler.getParameterMatchesFromQueryResult(queryResult); return parameters; } @@ -466,7 +488,7 @@ private void identifyAuditFindings(String sessionName) { List sessionSpecificParams = getSessionSpecificParameters(sessionName); for (ParameterMatch match : sessionSpecificParams) { Map values = Map.of("name", match.getName(), "type", match.getType(), "value", match.getValue()); - String valueQuery = "MATCH (p:InputParameter {name: $name, type: $type})-[OCCURS_WITH_VALUE]-(m:InputValue {type: $type, value: $value}) RETURN m"; + String valueQuery = "MATCH (p:InputParameter {name: $name, type: $type})-[OCCURS_WITH_VALUE]-(o:InputValue {type: $type, value: $value}) RETURN o"; Result result = DBModel.query(valueQuery, values); for (var parameterValue : CypherQueryHandler.getOccurrencesFromQueryResult(result).stream().distinct().toList()) { String sessionEntered = parameterValue.getSession(); @@ -483,7 +505,7 @@ private void identifyAuditFindings(String sessionName) { } private static void setParametersInList(Vector parameters) { - DefaultListModel model = (DefaultListModel) sessionSpecificParameterJList.getModel(); + DefaultListModel model = (DefaultListModel) sessionSpecificParameterMatchesJList.getModel(); model.clear(); model.addAll(parameters); } @@ -503,18 +525,30 @@ public void onDeferMatchingFinishedEvent() { identifySessionSpecificParametersAfterDeferMatching(); } + private void clearDataForNewSessionDefinition() { + sessionCounter = 0; + ((DefaultListModel) sessionJList.getModel()).clear(); + model.inputValuesFromSessionDefList.clear(); + model.parameterValuesAndNames.clear(); + model.helpers.clear(); + model.deleteSessionsInDB(); + sessionTable.clear(); + } + public void clearDataAndView() { model.clearAllData(); view.clearMatchLists(); view.cypherQueryField.setText(""); view.sessionNameTextField.setText(""); + parameterHandler.setSessionName("not set"); + matchHandler.setSessionName("not"); DefaultListModel listModel1 = (DefaultListModel) sessionDefJList.getModel(); listModel1.clear(); DefaultListModel listModel2 = (DefaultListModel) sessionJList.getModel(); listModel2.clear(); DefaultListModel listModel3 = (DefaultListModel) view.sessionParamMonitorJList.getModel(); listModel3.clear(); - DefaultListModel listModel4 = (DefaultListModel) sessionSpecificParameterJList.getModel(); + DefaultListModel listModel4 = (DefaultListModel) sessionSpecificParameterMatchesJList.getModel(); listModel4.clear(); view.revalidate(); view.repaint(); diff --git a/src/main/java/db/CypherQueryHandler.java b/src/main/java/db/CypherQueryHandler.java index b2e7277..ff9393a 100644 --- a/src/main/java/db/CypherQueryHandler.java +++ b/src/main/java/db/CypherQueryHandler.java @@ -1,6 +1,7 @@ package db; import db.entities.InputValue; +import db.entities.MatchValue; import db.entities.ParameterMatch; import org.neo4j.ogm.model.Result; @@ -17,18 +18,32 @@ public static List getOccurrencesFromQueryResult(Result queryResult) List occurrences = new ArrayList<>(); while (resultIterator.hasNext()) { Map result = resultIterator.next(); - occurrences.add((InputValue) result.get("m")); + if (result.get("o") != null) + occurrences.add((InputValue) result.get("o")); } return occurrences; } - public static List getMatchesFromQueryResult(Result queryResult) { + public static List getParameterMatchesFromQueryResult(Result queryResult) { Iterator> resultIterator = queryResult.queryResults().iterator(); - List occurrences = new ArrayList<>(); + List parameterMatches = new ArrayList<>(); while (resultIterator.hasNext()) { Map result = resultIterator.next(); - occurrences.add((ParameterMatch) result.get("m")); + if (result.get("m") != null) + parameterMatches.add((ParameterMatch) result.get("m")); } - return occurrences; + return parameterMatches; + } + + public static List getMatchValuesFromQueryResult(Result queryResult) { + Iterator> resultIterator = queryResult.queryResults().iterator(); + List matchValues = new ArrayList<>(); + while (resultIterator.hasNext()) { + Map result = resultIterator.next(); + if (result.get("mv") != null) + matchValues.add((MatchValue) result.get("mv")); + } + return matchValues; } + } diff --git a/src/main/java/db/DBModel.java b/src/main/java/db/DBModel.java index 4543db9..859c797 100644 --- a/src/main/java/db/DBModel.java +++ b/src/main/java/db/DBModel.java @@ -44,136 +44,6 @@ public DBModel() { } } - public static void saveParameter(InputParameter inputParameter) { - paramSaveCounter += 1; - Throwable txEx = null; - int RETRIES = 20; - int BACKOFF = 3000; - Session session = sessionFactory.openSession(); - for ( int i = 0; i < RETRIES; i++ ) - { - try ( Transaction tx = session.beginTransaction() ) - { - session.save(inputParameter); - tx.commit(); - session.clear(); - return; - } - catch ( Throwable ex ) - { - txEx = ex; - - // Add whatever exceptions to retry on here - if ( !(ex instanceof DeadlockDetectedException || ex instanceof TransientException) ) - { - break; - } - } - - // Wait so that we don't immediately get into the same deadlock - if ( i < RETRIES - 1 ) - { - try - { - Thread.sleep( BACKOFF ); - } - catch ( InterruptedException e ) - { - TransactionFailureException exception = new TransactionFailureException( "Interrupted", e ); - Logger.getInstance().logToError(Arrays.toString(exception.getStackTrace())); - throw exception; - } - } - } - - session.clear(); - - if ( txEx instanceof TransactionFailureException ) - { - Logger.getInstance().logToError(Arrays.toString(txEx.getStackTrace())); - throw ((TransactionFailureException) txEx); - } - else if ( txEx instanceof Error ) - { - Logger.getInstance().logToError(Arrays.toString(txEx.getStackTrace())); - throw ((Error) txEx); - } - else - { - Logger.getInstance().logToError(Arrays.toString(txEx.getStackTrace())); - throw ((RuntimeException) txEx); - } - } - - public static void saveURL(Url urlEntity) { - Throwable txEx = null; - int RETRIES = 20; - int BACKOFF = 3000; - Session session = sessionFactory.openSession(); - for ( int i = 0; i < RETRIES; i++ ) - { - try ( Transaction tx = session.beginTransaction() ) - { - session.save(urlEntity); - tx.commit(); - session.clear(); - return; - } - catch ( Throwable ex ) - { - txEx = ex; - - // Add whatever exceptions to retry on here - if ( !(ex instanceof DeadlockDetectedException || ex instanceof TransientException) ) - { - break; - } - } - - // Wait so that we don't immediately get into the same deadlock - if ( i < RETRIES - 1 ) - { - try - { - Thread.sleep( BACKOFF ); - } - catch ( InterruptedException e ) - { - TransactionFailureException exception = new TransactionFailureException( "Interrupted", e ); - Logger.getInstance().logToError(Arrays.toString(exception.getStackTrace())); - throw exception; - } - } - } - - session.clear(); - - if ( txEx instanceof TransactionFailureException ) - { - Logger.getInstance().logToError(Arrays.toString(txEx.getStackTrace())); - throw ((TransactionFailureException) txEx); - } - else if ( txEx instanceof Error ) - { - Logger.getInstance().logToError(Arrays.toString(txEx.getStackTrace())); - throw ((Error) txEx); - } - else - { - Logger.getInstance().logToError(Arrays.toString(txEx.getStackTrace())); - throw ((RuntimeException) txEx); - } - } - - public static void saveMatchEntity(ParameterMatch parameterMatchEntity) { - Session session = sessionFactory.openSession(); - try (Transaction tx = session.beginTransaction()) { - session.save(parameterMatchEntity); - tx.commit(); - } - session.clear(); - } - public static Collection loadAllParameters() { Session session = sessionFactory.openSession(); Collection collection = session.loadAll(InputParameter.class); @@ -240,25 +110,9 @@ public static void executeCypher(String cypherQuery, Map values) session.clear(); } - public static void saveBulk(List entities) { - Session session = sessionFactory.openSession(); - try (Transaction tx = session.beginTransaction()) { - session.save(entities); - tx.commit(); - } - session.clear(); - } - - public static void saveBulkParameters(List entities) { - Session session = sessionFactory.openSession(); - try (Transaction tx = session.beginTransaction()) { - session.save(entities); - tx.commit(); - } - session.clear(); - } - - public static void saveSession(db.entities.Session entity) { + public static void saveEntity(Object entity) { + if (entity instanceof InputParameter) + paramSaveCounter += 1; Throwable txEx = null; int RETRIES = 20; int BACKOFF = 3000; @@ -318,6 +172,52 @@ else if ( txEx instanceof Error ) } } + public static void saveBulk(List entities) { + Throwable txEx = null; + int RETRIES = 20; + int BACKOFF = 3000; + Session session = sessionFactory.openSession(); + for ( int i = 0; i < RETRIES; i++ ) { + try ( Transaction tx = session.beginTransaction() ) { + session.save(entities); + tx.commit(); + session.clear(); + return; + } catch ( Throwable ex ) { + txEx = ex; + + // Add whatever exceptions to retry on here + if ( !(ex instanceof DeadlockDetectedException || ex instanceof TransientException) ) { + break; + } + } + + // Wait so that we don't immediately get into the same deadlock + if ( i < RETRIES - 1 ) { + try { + Thread.sleep( BACKOFF ); + } catch ( InterruptedException e ) { + TransactionFailureException exception = new TransactionFailureException( "Interrupted", e ); + Logger.getInstance().logToError(Arrays.toString(exception.getStackTrace())); + throw exception; + } + } + } + + session.clear(); + + if ( txEx instanceof TransactionFailureException ) { + Logger.getInstance().logToError(Arrays.toString(txEx.getStackTrace())); + throw ((TransactionFailureException) txEx); + } else if ( txEx instanceof Error ) { + Logger.getInstance().logToError(Arrays.toString(txEx.getStackTrace())); + throw ((Error) txEx); + } else { + Logger.getInstance().logToError(Arrays.toString(txEx.getStackTrace())); + throw ((RuntimeException) txEx); + } + } + public static void purgeDatabase() { Session session = sessionFactory.openSession(); try (Transaction tx = session.beginTransaction()) { diff --git a/src/main/java/db/DeferMatching.java b/src/main/java/db/DeferMatching.java index 72bc07f..c94611e 100644 --- a/src/main/java/db/DeferMatching.java +++ b/src/main/java/db/DeferMatching.java @@ -3,25 +3,21 @@ import burp.BurpExtender; import burp.HttpResponse; import burp.HttpResponseParser; -import burp.RegexMatcher; import burp.api.montoya.MontoyaApi; import burp.api.montoya.proxy.ProxyHttpRequestResponse; import db.entities.*; import events.DeferMatchingFinishedEvent; import events.DeferMatchingFinishedListener; +import gui.ProgressDialog; import model.SessionViewModel; -import net.miginfocom.swing.MigLayout; import utils.Hashing; import utils.Logger; import javax.swing.*; -import java.awt.*; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; import java.beans.PropertyChangeEvent; import java.beans.PropertyChangeListener; -import java.util.*; import java.util.List; +import java.util.*; public class DeferMatching implements PropertyChangeListener { @@ -29,12 +25,9 @@ public class DeferMatching implements PropertyChangeListener { private HttpResponseParser parser; private ParameterHandler pHandler; private MatchHandler matchHandler; - private JProgressBar progressBar; - private JDialog progressDialog; private MatchTask task; - private JTextArea taskOutput; - private JButton closeButton; private int historySize; + private ProgressDialog progressDialog; private List listeners; public DeferMatching(MontoyaApi api, HttpResponseParser parser, ParameterHandler pHandler, MatchHandler mHandler) { @@ -49,36 +42,11 @@ public void init() { SwingUtilities.invokeLater(new Runnable() { @Override public void run() { - historySize = api.proxy().history().size() - BurpExtender.historyStart; - progressDialog = new JDialog((Frame) null, "Matching Parameters...", true); - progressDialog.setResizable(false); - progressDialog.getContentPane().setLayout(new MigLayout("fill")); - progressDialog.setLocationRelativeTo(null); - progressBar = new JProgressBar(JProgressBar.HORIZONTAL); - progressBar.setMinimum(0); - progressBar.setMaximum(100); - progressBar.setSize(280, 20); - progressBar.setStringPainted(true); - taskOutput = new JTextArea(); - taskOutput.setMargin(new Insets(5,5,5,5)); - taskOutput.setEditable(false); - closeButton = new JButton("Close"); - closeButton.addActionListener(new ActionListener() { - @Override - public void actionPerformed(ActionEvent actionEvent) { - progressDialog.setVisible(false); - } - }); - progressDialog.setDefaultCloseOperation(JDialog.DO_NOTHING_ON_CLOSE); - progressDialog.setMinimumSize(new Dimension(300, 150)); - progressDialog.setResizable(false); - progressDialog.add(new JLabel("Progress..."), "north"); - progressDialog.add(new JScrollPane(taskOutput), "dock center, grow"); - progressDialog.add(closeButton, "south"); - progressDialog.add(progressBar,"south"); - progressDialog.setVisible(true); + progressDialog = new ProgressDialog("Matching Parameters..."); + progressDialog.init(); } }); + historySize = api.proxy().history().size() - BurpExtender.historyStart; task = new MatchTask(); task.addPropertyChangeListener(this); task.execute(); @@ -89,8 +57,8 @@ public void propertyChange(PropertyChangeEvent evt) { if ("progress".equals(evt.getPropertyName())) { int progress = (Integer) evt.getNewValue(); double historyProgress = (((double) progress * historySize / 100) - 1); - progressBar.setValue(progress); - taskOutput.append(String.format( + progressDialog.updateProgressBarValue(progress); + progressDialog.appendTaskOutput(String.format( "Completed %d/%d of Burp history.\n", (int) historyProgress, historySize)); } } @@ -110,17 +78,16 @@ private void fireDeferMatchingFinishedEvent() { } } - class MatchTask extends SwingWorker { + class MatchTask extends SwingWorker { @Override protected Void doInBackground() { try { int historySize = api.proxy().history().size(); var proxyList = api.proxy().history().subList(BurpExtender.historyStart, historySize); - List allInputParameters = pHandler.observableInputParameterList; + List allInputParameters = pHandler.allInputParametersList; List allMatches = new ArrayList<>(); Set duplicateIdentifiers = new HashSet<>(); - Set messageHashes = new HashSet<>(); List inputParametersMatchingToHistory = new ArrayList<>(); Set inputParamIdentifiers = new HashSet<>(); @@ -136,24 +103,35 @@ protected Void doInBackground() { setProgress(0); int listSize = proxyList.size(); - List sessions = SessionViewModel.sessionTable.values().stream().toList(); - - String currentSessionName = matchHandler.getSessionName(); - - HashMap hashMap = correctSessionName(sessions, listSize); + HashMap hashMap = new HashMap<>(); + String currentSessionName = "not set"; + if (matchHandler.isSessionActive()) { + List sessions = SessionViewModel.sessionTable.values().stream().toList(); + currentSessionName = matchHandler.getSessionName(); + hashMap = correctSessionName(sessions, listSize); + } for (int i = 0; i < listSize; i++) { - if (!matchHandler.getSessionName().equals(hashMap.get(i))) { - matchHandler.setSessionName(hashMap.get(i)); + + if (matchHandler.isSessionActive()) { + if (!matchHandler.getSessionName().equals(hashMap.get(i)) && hashMap.size() == listSize) { + matchHandler.setSessionName(hashMap.get(i)); + } else { + // correctSessionName method returns hashmap of size = 1 if there is only 1 session defined + matchHandler.setSessionName(hashMap.get(0)); + } } ProxyHttpRequestResponse proxyResponse = proxyList.get(i); progress = (int) ((i + 1) / (double) listSize * 100); setProgress(progress); - HttpResponse response = parser.parseResponse(proxyResponse.originalResponse(), proxyResponse.finalRequest()); + burp.api.montoya.http.message.responses.HttpResponse originalResponse = proxyResponse.originalResponse(); + if (originalResponse == null) { + continue; + } + HttpResponse response = parser.parseResponse(originalResponse, proxyResponse.finalRequest()); String hash = Hashing.sha1(proxyResponse.finalRequest().toByteArray().getBytes()); - messageHashes.add(hash); for (InputParameter parameter : allInputParameters) { InputParameter realParam = new InputParameter(parameter.getName(), parameter.getType(), parameter.getDomain()); @@ -163,7 +141,7 @@ protected Void doInBackground() { } for (InputValue value : parameter.getOccurrenceEntities()) { if (hash.equals(value.getMessageHash())) { - realParam.addOccurence(value); + realParam.addOccurrence(value); } } if (realParam.getOccurrenceEntities().isEmpty()) { @@ -176,7 +154,7 @@ protected Void doInBackground() { } } List respMatch = parser.getMatches(response, inputParametersMatchingToHistory, hash); - List realMatches = matchHandler.addMatches(respMatch); + List realMatches = matchHandler.addMatchesThreaded(respMatch); for (Object realMatch : realMatches) { int identifier = -1; @@ -194,9 +172,10 @@ protected Void doInBackground() { } } } - taskOutput.append("Saving Entities in Database...\n"); + publish("Saving Entities in Database...\n"); DBModel.saveBulk(allMatches); - matchHandler.setSessionName(currentSessionName); + if (matchHandler.isSessionActive()) + matchHandler.setSessionName(currentSessionName); return null; } catch (Exception e) { Logger.getInstance().logToError(Arrays.toString(e.getStackTrace())); @@ -219,11 +198,16 @@ private HashMap correctSessionName(List sessions, int return hashMap; } + @Override + protected void process(List chunks) { + progressDialog.appendTaskOutput("Saving Entities in Database...\n"); + } + @Override protected void done() { - progressDialog.setDefaultCloseOperation(JDialog.HIDE_ON_CLOSE); - progressBar.setValue(100); - taskOutput.setText("Completed!\n"); + progressDialog.updateDialogDefaultCloseOperation(JDialog.HIDE_ON_CLOSE); + progressDialog.updateProgressBarValue(100); + progressDialog.setTaskOutputText("Completed!\n"); fireDeferMatchingFinishedEvent(); } } diff --git a/src/main/java/db/MatchHandler.java b/src/main/java/db/MatchHandler.java index 1d60cf3..8d70f63 100644 --- a/src/main/java/db/MatchHandler.java +++ b/src/main/java/db/MatchHandler.java @@ -7,12 +7,11 @@ import db.entities.Session; import db.entities.Url; import gui.GettingStartedView; -import javafx.collections.FXCollections; -import javafx.collections.ObservableList; import model.SessionViewModel; import utils.Logger; import java.util.*; +import java.util.concurrent.*; // Handles the logic behind new-found matches public class MatchHandler { @@ -20,8 +19,6 @@ public class MatchHandler { private ParameterHandler parameterHandler; public Hashtable matchValueStorage; public Hashtable parameterMatchStorage; - public ObservableList observableParameterMatchList; - public ObservableList observableParameterMatchListSession; private boolean hasActiveSession = false; private String sessionName; private CrossSessionAudit crossSessionAudit; @@ -38,8 +35,6 @@ public MatchHandler(ParameterHandler parameterHandler, CrossSessionAudit crossSe this.matchValueStorage = new Hashtable<>(); this.parameterMatchStorage = new Hashtable<>(); this.parameterHandler = parameterHandler; - this.observableParameterMatchList = FXCollections.observableArrayList(); - this.observableParameterMatchListSession = FXCollections.observableArrayList(); this.crossSessionAudit = crossSessionAudit; this.crossContentTypeAudit = crossContentTypeAudit; this.crossScopeAudit = crossScopeAudit; @@ -54,7 +49,6 @@ private void loadEntities() { List parameterMatchEntityList = DBModel.loadAllMatchEntities().stream().toList(); List identifiersMatchEntities = parameterMatchEntityList.stream().map(ParameterMatch::getIdentifier).toList(); parameterMatchStorage.putAll(combineListsIntoMatchEntityMap(identifiersMatchEntities, parameterMatchEntityList)); - this.observableParameterMatchList.addAll(parameterMatchEntityList); List matchValueEntityList = DBModel.loadAllMatchEntryEntities().stream().toList(); List identifiersMatchEntryEntities = matchValueEntityList.stream().map(MatchValue::getIdentifier).toList(); matchValueStorage.putAll(combineListsIntoMatchEntryEntityMap(identifiersMatchEntryEntities, matchValueEntityList)); @@ -133,13 +127,11 @@ private List addMatchToDB(MatchHelperClass match) { matchingUrlEntity.addFound(newParameterMatchEntity); parameterMatchStorage.put(newParameterMatchEntity.getIdentifier(), newParameterMatchEntity); this.matchValueStorage.put(newMatchValueEntity.getIdentifier(), newMatchValueEntity); - GettingStartedView.numberOfParameterMatches.setText(String.valueOf(parameterMatchStorage.size())); - GettingStartedView.numberOfMatchValues.setText(String.valueOf(matchValueStorage.size())); // If a session is active add entities to the session and save the session and Url entity, otherwise save both ParameterMatch and Url entity if (session != null) { session.addMatch(newParameterMatchEntity); session.addMatchValue(newMatchValueEntity); - returnList.add(session); + returnList.add(newParameterMatchEntity); returnList.add(matchingUrlEntity); SessionViewModel.sessionTable.put(this.sessionName, session); } else { @@ -150,33 +142,73 @@ private List addMatchToDB(MatchHelperClass match) { ParameterMatch relatedParameterMatchEntity = getMatchEntityFromUrlEntity(name, value, url); relatedParameterMatchEntity.addMatchEntryEntity(newMatchValueEntity); this.matchValueStorage.put(newMatchValueEntity.getIdentifier(), newMatchValueEntity); - GettingStartedView.numberOfMatchValues.setText(String.valueOf(matchValueStorage.size())); if (session != null) { session.addMatch(relatedParameterMatchEntity); session.addMatchValue(newMatchValueEntity); - returnList.add(session); + returnList.add(relatedParameterMatchEntity); SessionViewModel.sessionTable.put(this.sessionName, session); } else { returnList.add(relatedParameterMatchEntity); } } - this.observableParameterMatchList.add(newParameterMatchEntity); - if (newParameterMatchEntity.getSession().equals(sessionName)) { - this.observableParameterMatchListSession.add(newParameterMatchEntity); - } } return returnList; } + class AddMatchesCallable implements Callable> { + private MatchHelperClass match; + + AddMatchesCallable(MatchHelperClass match) { + this.match = match; + } + + @Override + public List call() { + return addMatchToDB(match); + } + } + + public List addMatchesThreaded(List matches) { + List>> futures = new ArrayList<>(); + ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + for (MatchHelperClass match : matches) { + Future> future = executor.submit(new AddMatchesCallable(match)); + futures.add(future); + } + List results = new ArrayList<>(); + for (Future> future : futures) { + try { + results.addAll(future.get()); + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + } + + executor.shutdown(); + + GettingStartedView.numberOfParameterMatches.setText(String.valueOf(parameterMatchStorage.size())); + GettingStartedView.numberOfMatchValues.setText(String.valueOf(matchValueStorage.size())); + + this.crossSessionAudit.renderFindings(); + + return results; + } + public List addMatches(List matches) { List returnList = new ArrayList<>(); for (MatchHelperClass match : matches) { returnList.addAll(addMatchToDB(match)); } + + GettingStartedView.numberOfParameterMatches.setText(String.valueOf(parameterMatchStorage.size())); + GettingStartedView.numberOfMatchValues.setText(String.valueOf(matchValueStorage.size())); + this.crossSessionAudit.renderFindings(); + return returnList; } + private boolean matchEntryExistsInDB(MatchValue matchValueEntity) { int identifier = Objects.hash(matchValueEntity.getName(), matchValueEntity.getValue(), matchValueEntity.getMatchProof(), matchValueEntity.getUrl(), "not set"); @@ -192,8 +224,8 @@ public boolean matchEntityExistsInUrlEntity(Url urlEntity, String name, String v if (hasActiveSession) { identifier = Objects.hash(name, value, urlEntity.getUrl(), this.sessionName); } - List parameterMatchEntityList = urlEntity.getFound(); - return parameterMatchEntityList.stream().map(ParameterMatch::getIdentifier).toList().contains(identifier); + List parameterMatchEntityList = Collections.synchronizedList(new ArrayList<>(urlEntity.getFound())); + return parameterMatchEntityList.parallelStream().map(ParameterMatch::getIdentifier).toList().contains(identifier); } private ParameterMatch getMatchEntityFromUrlEntity(String name, String value, String url) { @@ -205,7 +237,7 @@ private ParameterMatch getMatchEntityFromUrlEntity(String name, String value, St } public Url getMatchingUrlEntity(String url) { - Hashtable urlEntityStorage = parameterHandler.getUrlStorage(); + ConcurrentHashMap urlEntityStorage = parameterHandler.getUrlStorage(); int identifier = Objects.hash(url); return urlEntityStorage.get(identifier); } @@ -222,10 +254,12 @@ public String getSessionName() { return sessionName; } + public boolean isSessionActive() { + return this.hasActiveSession; + } + public void clearAllStorages() { matchValueStorage.clear(); parameterMatchStorage.clear(); - observableParameterMatchList.clear(); - observableParameterMatchListSession.clear(); } } diff --git a/src/main/java/db/ParameterHandler.java b/src/main/java/db/ParameterHandler.java index a95b531..ccf83da 100644 --- a/src/main/java/db/ParameterHandler.java +++ b/src/main/java/db/ParameterHandler.java @@ -7,32 +7,29 @@ import db.entities.Url; import gui.GettingStartedView; import gui.container.RuleContainer; -import javafx.collections.FXCollections; -import javafx.collections.ObservableList; import model.SessionViewModel; import utils.Logger; import java.util.*; +import java.util.concurrent.*; import java.util.stream.Collectors; // InputParameter Entities are being saved via the corresponding URL Entity where the parameter appeared // Hence keeping track of the URL Entities is enough because they contain the Parameters public class ParameterHandler { - public Hashtable urlStorage; - public Hashtable parameterStorage; - public Hashtable parameterValueStorage; - public ObservableList observableInputParameterList; - public ObservableList observableInputParameterListSession; - + public ConcurrentHashMap urlStorage; + public ConcurrentHashMap inputParameterStorage; + public ConcurrentHashMap inputValueStorage; + public List allInputParametersList; private boolean hasActiveSession = false; private String sessionName; + public ParameterHandler() { - this.urlStorage = new Hashtable<>(); - this.parameterStorage = new Hashtable<>(); - this.parameterValueStorage = new Hashtable<>(); - this.observableInputParameterList = FXCollections.observableArrayList(); - this.observableInputParameterListSession = FXCollections.observableArrayList(); + this.urlStorage = new ConcurrentHashMap<>(); + this.inputParameterStorage = new ConcurrentHashMap<>(); + this.inputValueStorage = new ConcurrentHashMap<>(); + this.allInputParametersList = Collections.synchronizedList(new ArrayList()); loadUrls(); loadParameters(); loadParameterOccurrences(); @@ -47,14 +44,14 @@ public void loadUrls() { public void loadParameters() { List inputParameterEntityList = DBModel.loadAllParameters().stream().toList(); List identifiers = inputParameterEntityList.stream().map(InputParameter::getIdentifier).toList(); - parameterStorage.putAll(combineListsIntoParameterEntityMap(identifiers, inputParameterEntityList)); - observableInputParameterList.addAll(inputParameterEntityList); + inputParameterStorage.putAll(combineListsIntoParameterEntityMap(identifiers, inputParameterEntityList)); + allInputParametersList.addAll(inputParameterEntityList); } public void loadParameterOccurrences() { List inputValueEntityList = DBModel.loadAllParameterOccurrenceEntities().stream().toList(); List identifiers = inputValueEntityList.stream().map(InputValue::getIdentifier).toList(); - parameterValueStorage.putAll(combineListsIntoParameterOccurrenceMap(identifiers, inputValueEntityList)); + inputValueStorage.putAll(combineListsIntoParameterOccurrenceMap(identifiers, inputValueEntityList)); } Map combineListsIntoParameterEntityMap(List keys, List values) { @@ -101,7 +98,10 @@ Map combineListsIntoParameterOccurrenceMap(List ke * to the DB @param parameterHelper InputParameter to be added */ - private void addParameterToDB(ParameterHelperClass parameterHelper, String messageHash) { + private List addParameterToDB(ParameterHelperClass parameterHelper, String messageHash) { + + List returnList = new ArrayList<>(); + String name = parameterHelper.getName(); String type = parameterHelper.getType().getName(); String domain = parameterHelper.getDomain(); @@ -115,7 +115,6 @@ private void addParameterToDB(ParameterHelperClass parameterHelper, String messa Session session = SessionViewModel.sessionTable.get(this.sessionName); if (session != null) { session.addInputValue(newInputValue); - DBModel.saveSession(session); SessionViewModel.sessionTable.put(this.sessionName, session); } } @@ -125,72 +124,111 @@ private void addParameterToDB(ParameterHelperClass parameterHelper, String messa // Check if the URL Entity already exists in the DB // If not, add it to the list of known Urls and save the Url + InputParameter in the DB if (!urlExistsInDB(newUrlEntity)) { - newInputParameterEntity.addOccurence(newInputValue); - this.parameterValueStorage.put(newInputValue.getIdentifier(), newInputValue); + if (!newInputValue.getValue().isEmpty()) { + newInputParameterEntity.addOccurrence(newInputValue); + this.inputValueStorage.put(newInputValue.getIdentifier(), newInputValue); + } newUrlEntity.addParameterFoundInUrl(newInputParameterEntity); this.urlStorage.put(newUrlEntity.getIdentifier(), newUrlEntity); - parameterStorage.put(newInputParameterEntity.getIdentifier(), newInputParameterEntity); - GettingStartedView.numberOfParameterValues.setText(String.valueOf(parameterValueStorage.size())); - GettingStartedView.numberOfParameters.setText(String.valueOf(parameterStorage.size())); - GettingStartedView.numberOfUrls.setText(String.valueOf(urlStorage.size())); - DBModel.saveParameter(newInputParameterEntity); - DBModel.saveURL(newUrlEntity); - this.observableInputParameterList.add(newInputParameterEntity); - if (newInputValue.getSession().equals(sessionName)) { - this.observableInputParameterListSession.add(newInputParameterEntity); - } + inputParameterStorage.put(newInputParameterEntity.getIdentifier(), newInputParameterEntity); + this.allInputParametersList.add(newInputParameterEntity); + returnList.add(newInputParameterEntity); + returnList.add(newUrlEntity); } else { // If the URL Entity already exists get the correct Url Entity where the InputParameter Entity is to be added Url relatedUrlEntity = getUrlEntityByParameterUrl(parameterHelper.getUrlFound()); // Check if Relationship to InputParameter already exists, this is the case if the URLs get loaded at start if (!parameterExistsInUrlEntity(relatedUrlEntity, newInputParameterEntity)) { - if(!occurrenceAlreadyExists(newInputValue)) { - newInputParameterEntity.addOccurence(newInputValue); - this.parameterValueStorage.put(newInputValue.getIdentifier(), newInputValue); - GettingStartedView.numberOfParameterValues.setText(String.valueOf(parameterValueStorage.size())); - if (newInputValue.getSession().equals(sessionName)) { - this.observableInputParameterListSession.add(newInputParameterEntity); - } + if(!occurrenceAlreadyExists(newInputValue, newInputParameterEntity) && !newInputValue.getValue().isEmpty()) { + newInputParameterEntity.addOccurrence(newInputValue); + this.inputValueStorage.put(newInputValue.getIdentifier(), newInputValue); } relatedUrlEntity.addParameterFoundInUrl(newInputParameterEntity); - parameterStorage.put(newInputParameterEntity.getIdentifier(), newInputParameterEntity); - GettingStartedView.numberOfParameters.setText(String.valueOf(parameterStorage.size())); - DBModel.saveParameter(newInputParameterEntity); - DBModel.saveURL(relatedUrlEntity); - this.observableInputParameterList.add(newInputParameterEntity); + inputParameterStorage.put(newInputParameterEntity.getIdentifier(), newInputParameterEntity); + this.allInputParametersList.add(newInputParameterEntity); + returnList.add(newInputParameterEntity); + returnList.add(relatedUrlEntity); } else { InputParameter existingEntity = getExistingParameter(newInputParameterEntity.getIdentifier()); - if (!occurrenceAlreadyExists(newInputValue)) { - existingEntity.addOccurence(newInputValue); - this.parameterValueStorage.put(newInputValue.getIdentifier(), newInputValue); - GettingStartedView.numberOfParameterValues.setText(String.valueOf(parameterValueStorage.size())); - DBModel.saveParameter(existingEntity); - this.observableInputParameterList.add(newInputParameterEntity); - if (newInputValue.getSession().equals(sessionName)) { - this.observableInputParameterListSession.add(newInputParameterEntity); - } + if (!occurrenceAlreadyExists(newInputValue, existingEntity) && !newInputValue.getValue().isEmpty()) { + existingEntity.addOccurrence(newInputValue); + this.inputValueStorage.put(newInputValue.getIdentifier(), newInputValue); + this.allInputParametersList.add(newInputParameterEntity); + returnList.add(existingEntity); } } } + + return returnList; + } + + class AddParametersCallable implements Callable> { + private ParameterHelperClass parameter; + private String messageHash; + + AddParametersCallable(ParameterHelperClass parameter, String messageHash) { + this.parameter = parameter; + this.messageHash = messageHash; + } + + @Override + public List call() { + return addParameterToDB(parameter, messageHash); + } } - public void addParameters(Collection oldParameterCollection, String messageHash) { + public List addParametersThreaded(Collection oldParameterCollection, String messageHash) { + List>> futures = new ArrayList<>(); + ExecutorService executor = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); for (ParameterHelperClass oldParameter : oldParameterCollection) { - addParameterToDB(oldParameter, messageHash); + Future> future = executor.submit(new AddParametersCallable(oldParameter, messageHash)); + futures.add(future); + } + List results = new ArrayList<>(); + for (Future> future : futures) { + try { + results.addAll(future.get()); + } catch (InterruptedException | ExecutionException e) { + e.printStackTrace(); + } + } + + executor.shutdown(); + + GettingStartedView.numberOfParameters.setText(String.valueOf(inputParameterStorage.size())); + GettingStartedView.numberOfParameterValues.setText(String.valueOf(inputValueStorage.size())); + GettingStartedView.numberOfUrls.setText(String.valueOf(urlStorage.size())); + + return results; + } + + public List addParameters(Collection parameters, String messageHash) { + List returnList = new ArrayList<>(); + for (ParameterHelperClass param : parameters) { + returnList.addAll(addParameterToDB(param, messageHash)); } + + GettingStartedView.numberOfParameters.setText(String.valueOf(inputParameterStorage.size())); + GettingStartedView.numberOfParameterValues.setText(String.valueOf(inputValueStorage.size())); + GettingStartedView.numberOfUrls.setText(String.valueOf(urlStorage.size())); + + return returnList; } private InputParameter getExistingParameter(int identifier) { - return parameterStorage.get(identifier); + return inputParameterStorage.get(identifier); } - private boolean occurrenceAlreadyExists(InputValue newInputValueEntity) { - return this.parameterValueStorage.containsKey(newInputValueEntity.getIdentifier()); + private boolean occurrenceAlreadyExists(InputValue newInputValueEntity, InputParameter newInputParameterEntity) { + return newInputParameterEntity.getOccurrenceEntities().parallelStream().anyMatch(inputValue -> inputValue.getIdentifier() == newInputValueEntity.getIdentifier()); } private boolean parameterExistsInUrlEntity(Url relatedUrlEntity, InputParameter newInputParameterEntity) { int identifier = newInputParameterEntity.getIdentifier(); - return relatedUrlEntity.getFoundInParameterList().stream().map(InputParameter::getIdentifier).toList().contains(identifier); + List list = Collections.synchronizedList(new ArrayList<>()); + list.addAll(relatedUrlEntity.getFoundInParameterList()); + + return list.stream().parallel().map(InputParameter::getIdentifier).collect(Collectors.toList()).contains(identifier); } private boolean urlExistsInDB(Url urlEntity) { @@ -203,12 +241,12 @@ private Url getUrlEntityByParameterUrl(String url) { return urlStorage.get(identifier); } - public Hashtable getUrlStorage() { + public ConcurrentHashMap getUrlStorage() { return urlStorage; } public List getAllParameters(){ - return parameterStorage.values().stream().toList(); + return inputParameterStorage.values().stream().toList(); } public List getRelevant(String domain){ var all = this.getAllParameters(); @@ -231,15 +269,14 @@ public void clearAllMatchRelatedObjectsFromStorage() { } public void clearAllStorages() { - parameterStorage.clear(); - parameterValueStorage.clear(); + inputParameterStorage.clear(); + inputValueStorage.clear(); urlStorage.clear(); - observableInputParameterList.clear(); - observableInputParameterListSession.clear(); + allInputParametersList.clear(); } public void updateParameterExclusion(RuleContainer ruleContainer) { - RegexMatcher.excludeParametersForSingleRule(this.observableInputParameterList, ruleContainer); + RegexMatcher.excludeParametersForSingleRule(this.allInputParametersList, ruleContainer); if (hasActiveSession) { // Update exclusion on InputValues linked to Sessions var keys = SessionViewModel.sessionTable.keys().asIterator(); diff --git a/src/main/java/db/entities/InputParameter.java b/src/main/java/db/entities/InputParameter.java index 7c978da..bdc5703 100644 --- a/src/main/java/db/entities/InputParameter.java +++ b/src/main/java/db/entities/InputParameter.java @@ -2,11 +2,11 @@ import org.neo4j.ogm.annotation.Id; import org.neo4j.ogm.annotation.Relationship; -import org.neo4j.ogm.annotation.Transient; import utils.Logger; import utils.PatternEscape; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Objects; import java.util.regex.Pattern; @@ -23,7 +23,6 @@ public class InputParameter { @Relationship(type = "OCCURS_WITH_VALUE", direction = Relationship.Direction.OUTGOING) private List occurrenceEntities = new ArrayList<>(); - // Empty Constructor needed for neo4J public InputParameter() { @@ -37,38 +36,10 @@ public InputParameter(String name, String type, String domain) { this.excludedByNoiseReduction = false; } - public String getName() { - return name; - } - - public int getIdentifier(){ - return this.identifier; - } - - public String getDomain() { - return domain; - } - - public String getType() { - return this.type; - } - - public void addOccurence(InputValue occurrence) { + public void addOccurrence(InputValue occurrence) { this.occurrenceEntities.add(occurrence); } - public List getOccurrenceEntities() { - return occurrenceEntities; - } - - public boolean isExcludedByNoiseReduction() { - return excludedByNoiseReduction; - } - - public void setExcludedByNoiseReduction(boolean excludedByNoiseReduction) { - this.excludedByNoiseReduction = excludedByNoiseReduction; - } - public Pattern getRegexMatchingValueByIdentifier(int identifier) { var occurrence = this.getOccurrenceByIdentifier(identifier); if(occurrence == null){ @@ -89,6 +60,7 @@ public InputValue getOccurrenceByIdentifier(int identifier) { try { return occurrence.get(); } catch (Exception ex) { + Logger.getInstance().logToError(Arrays.toString(ex.getStackTrace())); return null; } } @@ -107,6 +79,34 @@ public Pattern getRegexForHeaderMatchingValueByIdentifier(int identifier) { return Pattern.compile(regex, Pattern.CASE_INSENSITIVE); } + public String getName() { + return name; + } + + public int getIdentifier(){ + return this.identifier; + } + + public String getDomain() { + return domain; + } + + public String getType() { + return this.type; + } + + public List getOccurrenceEntities() { + return occurrenceEntities; + } + + public boolean isExcludedByNoiseReduction() { + return excludedByNoiseReduction; + } + + public void setExcludedByNoiseReduction(boolean excludedByNoiseReduction) { + this.excludedByNoiseReduction = excludedByNoiseReduction; + } + @Override public String toString() { diff --git a/src/main/java/db/entities/InputValue.java b/src/main/java/db/entities/InputValue.java index 8838464..d1ae4da 100644 --- a/src/main/java/db/entities/InputValue.java +++ b/src/main/java/db/entities/InputValue.java @@ -1,7 +1,6 @@ package db.entities; import org.neo4j.ogm.annotation.Id; -import org.neo4j.ogm.annotation.Transient; import java.util.Objects; @@ -70,14 +69,14 @@ public String getType() { return type; } - public void setSession(String session) { - this.session = session; - } - public boolean isExcludedByNoiseReduction() { return excludedByNoiseReduction; } + public void setSession(String session) { + this.session = session; + } + public void setExcludedByNoiseReduction(boolean excludedByNoiseReduction) { this.excludedByNoiseReduction = excludedByNoiseReduction; } diff --git a/src/main/java/db/entities/ParameterMatch.java b/src/main/java/db/entities/ParameterMatch.java index a13146d..a44177c 100644 --- a/src/main/java/db/entities/ParameterMatch.java +++ b/src/main/java/db/entities/ParameterMatch.java @@ -88,14 +88,14 @@ public String getMessageHash() { return messageHash; } - public void setSession(String session) { - this.session = session; - } - public InputValue getInputValue(){ return this.valueMatched; } + public void setSession(String session) { + this.session = session; + } + @Override public String toString() { return "ParameterMatch{" + diff --git a/src/main/java/db/entities/Session.java b/src/main/java/db/entities/Session.java index 66d4905..745ae62 100644 --- a/src/main/java/db/entities/Session.java +++ b/src/main/java/db/entities/Session.java @@ -49,12 +49,20 @@ public List getSessionParameter() { return sessionParameter; } - public String getName() { - return name; + public void addMatch(ParameterMatch match) { + this.parameterMatchesRelatedToSession.add(match); } - public void setName(String name) { - this.name = name; + public void addMatchValue(MatchValue value) { + this.matchValuesRelatedToSession.add(value); + } + + public void addInputValue(InputValue inputValue) { + this.inputValuesRelatedToSession.add(inputValue); + } + + public String getName() { + return name; } public int getLowestHistoryId() { @@ -73,10 +81,6 @@ public int getIdentifier() { return identifier; } - public void setSessionParameter(List sessionParameter) { - this.sessionParameter = sessionParameter; - } - public List getInputValuesRelatedToSession() { return inputValuesRelatedToSession; } @@ -85,40 +89,24 @@ public void setInputValuesRelatedToSession(List inputValuesRelatedTo this.inputValuesRelatedToSession = inputValuesRelatedToSession; } - public List getParameterMatchesRelatedToSession() { - return parameterMatchesRelatedToSession; - } - public void setParameterMatchesRelatedToSession(List parameterMatchesRelatedToSession) { this.parameterMatchesRelatedToSession = parameterMatchesRelatedToSession; } - public List getMatchValuesRelatedToSession() { - return matchValuesRelatedToSession; - } - public void setMatchValuesRelatedToSession(List matchValuesRelatedToSession) { this.matchValuesRelatedToSession = matchValuesRelatedToSession; } - public void setLowestHistoryId(int lowestHistoryId) { - this.lowestHistoryId = lowestHistoryId; - } - - public void setHighestHistoryId(int highestHistoryId) { - this.highestHistoryId = highestHistoryId; - } - - public void addMatch(ParameterMatch match) { - this.parameterMatchesRelatedToSession.add(match); + public void setSessionParameter(List sessionParameter) { + this.sessionParameter = sessionParameter; } - public void addMatchValue(MatchValue value) { - this.matchValuesRelatedToSession.add(value); + public void setName(String name) { + this.name = name; } - public void addInputValue(InputValue inputValue) { - this.inputValuesRelatedToSession.add(inputValue); + public void setHighestHistoryId(int highestHistoryId) { + this.highestHistoryId = highestHistoryId; } @Override diff --git a/src/main/java/db/entities/SessionParameter.java b/src/main/java/db/entities/SessionParameter.java index fb9a9fe..d3d6b69 100644 --- a/src/main/java/db/entities/SessionParameter.java +++ b/src/main/java/db/entities/SessionParameter.java @@ -2,6 +2,7 @@ import org.neo4j.ogm.annotation.Id; import org.neo4j.ogm.annotation.NodeEntity; + import java.util.Objects; @NodeEntity diff --git a/src/main/java/db/entities/Url.java b/src/main/java/db/entities/Url.java index 90fe5f4..22886be 100644 --- a/src/main/java/db/entities/Url.java +++ b/src/main/java/db/entities/Url.java @@ -44,6 +44,13 @@ public Url(String url) { this.identifier = Objects.hash(url); } + public void addParameterFoundInUrl(InputParameter inputParameterEntity) { + this.foundInInputParameterList.add(inputParameterEntity); + } + public void addFound(ParameterMatch foundParameterMatchEntity) { + found.add(foundParameterMatchEntity); + } + public List getFound() { return found; } @@ -52,13 +59,6 @@ public List getFoundInParameterList() { return foundInInputParameterList; } - public void addParameterFoundInUrl(InputParameter inputParameterEntity) { - this.foundInInputParameterList.add(inputParameterEntity); - } - public void addFound(ParameterMatch foundParameterMatchEntity) { - found.add(foundParameterMatchEntity); - } - public int getIdentifier(){ return identifier; } diff --git a/src/main/java/events/ItemsAddedEvent.java b/src/main/java/events/ItemsAddedEvent.java new file mode 100644 index 0000000..edb10c4 --- /dev/null +++ b/src/main/java/events/ItemsAddedEvent.java @@ -0,0 +1,9 @@ +package events; + +import java.util.EventObject; + +public class ItemsAddedEvent extends EventObject { + public ItemsAddedEvent(Object source) { + super(source); + } +} diff --git a/src/main/java/events/ItemsAddedListener.java b/src/main/java/events/ItemsAddedListener.java new file mode 100644 index 0000000..f42d6a4 --- /dev/null +++ b/src/main/java/events/ItemsAddedListener.java @@ -0,0 +1,8 @@ +package events; + +import java.util.EventListener; + +public interface ItemsAddedListener extends EventListener { + + void onItemsAddedEvent(); +} diff --git a/src/main/java/events/ListChangeEvent.java b/src/main/java/events/ListChangeEvent.java new file mode 100644 index 0000000..9495b53 --- /dev/null +++ b/src/main/java/events/ListChangeEvent.java @@ -0,0 +1,43 @@ +package events; + +import utils.ChangeType; + +import java.util.ArrayList; +import java.util.List; + +public class ListChangeEvent { + private final List list; + private final List> listSubChangeEvents = new ArrayList<>(); + private int subChangeIndex = -1; + + public ListChangeEvent(List list) { + this.list = list; + } + + public void addSubChange(ListSubChangeEvent listSubChangeEvent) { + listSubChangeEvents.add(listSubChangeEvent); + } + + public boolean next() { + if (subChangeIndex < listSubChangeEvents.size() - 1) { + subChangeIndex++; + return true; + } + return false; + } + + public boolean wasAdded() { + return getCurrentSubChange().getType() == ChangeType.ADD; + } + + public boolean wasRemoved() { + return getCurrentSubChange().getType() == ChangeType.REMOVE; + } + + private ListSubChangeEvent getCurrentSubChange() { + if (subChangeIndex >= 0 && subChangeIndex < listSubChangeEvents.size()) { + return listSubChangeEvents.get(subChangeIndex); + } + throw new IllegalStateException("No current sub-change"); + } +} diff --git a/src/main/java/events/ListChangeListener.java b/src/main/java/events/ListChangeListener.java new file mode 100644 index 0000000..215bc24 --- /dev/null +++ b/src/main/java/events/ListChangeListener.java @@ -0,0 +1,5 @@ +package events; + +public interface ListChangeListener { + void onChanged(ListChangeEvent change); +} \ No newline at end of file diff --git a/src/main/java/events/ListSubChangeEvent.java b/src/main/java/events/ListSubChangeEvent.java new file mode 100644 index 0000000..bbbb04d --- /dev/null +++ b/src/main/java/events/ListSubChangeEvent.java @@ -0,0 +1,27 @@ +package events; + +import utils.ChangeType; + +public class ListSubChangeEvent { + private final ChangeType type; + private final E element; + private final int index; + + public ListSubChangeEvent(ChangeType type, E element, int index) { + this.type = type; + this.element = element; + this.index = index; + } + + public ChangeType getType() { + return type; + } + + public E getElement() { + return element; + } + + public int getIndex() { + return index; + } +} \ No newline at end of file diff --git a/src/main/java/gui/AuditFindingView.java b/src/main/java/gui/AuditFindingView.java index 92b2c6f..ac170f7 100644 --- a/src/main/java/gui/AuditFindingView.java +++ b/src/main/java/gui/AuditFindingView.java @@ -21,7 +21,7 @@ public class AuditFindingView extends JScrollPane { private PropertiesHandler propertiesHandler; private JPanel panel; private JList listOfFindings; - private JEditorPane desccriptionPane; + private JEditorPane descriptionPane; private JScrollPane findingsOverviewScrollPane; private JScrollPane findingDescriptionScrollPane; private Vector findings; @@ -47,10 +47,10 @@ private void initialize(){ this.findingsOverviewScrollPane.setMinimumSize(new Dimension(700, 800)); this.findingsOverviewScrollPane.setMaximumSize(new Dimension(700, 800)); - this.desccriptionPane = new JEditorPane(); - this.desccriptionPane.setEditable(false); - this.desccriptionPane.setContentType("text/html"); - this.findingDescriptionScrollPane = new JScrollPane(desccriptionPane); + this.descriptionPane = new JEditorPane(); + this.descriptionPane.setEditable(false); + this.descriptionPane.setContentType("text/html"); + this.findingDescriptionScrollPane = new JScrollPane(descriptionPane); this.findingDescriptionScrollPane.setMinimumSize(new Dimension(500, 800)); this.findingDescriptionScrollPane.setMaximumSize(new Dimension(500, 800)); @@ -65,7 +65,7 @@ public void valueChanged(ListSelectionEvent listSelectionEvent) { if (!listSelectionEvent.getValueIsAdjusting()) { var finding = listOfFindings.getSelectedValue(); if (finding != null) - desccriptionPane.setText(finding.getLongDescription()); + descriptionPane.setText(finding.getLongDescription()); } } }); @@ -115,7 +115,7 @@ public void loadAuditFindings() { } private void clearDescriptionPane() { - this.desccriptionPane.setText(""); + this.descriptionPane.setText(""); this.panel.revalidate(); this.panel.repaint(); } diff --git a/src/main/java/gui/GettingStartedView.java b/src/main/java/gui/GettingStartedView.java index 482f072..a8da64e 100644 --- a/src/main/java/gui/GettingStartedView.java +++ b/src/main/java/gui/GettingStartedView.java @@ -3,6 +3,7 @@ import burp.BurpExtender; import burp.HttpListener; import burp.PropertiesHandler; +import burp.RetroactiveParser; import burp.api.montoya.MontoyaApi; import controller.NoiseReductionController; import controller.QueryViewController; @@ -14,11 +15,11 @@ import net.miginfocom.swing.MigLayout; import javax.swing.*; +import java.awt.*; import java.awt.event.ActionEvent; import java.awt.event.ActionListener; import java.awt.event.ItemEvent; import java.awt.event.ItemListener; -import java.util.ArrayList; public class GettingStartedView extends JScrollPane { @@ -31,6 +32,7 @@ public class GettingStartedView extends JScrollPane { private QueryViewController queryViewController; private AuditFindingView auditFindingView; private SessionViewController sessionViewController; + private RetroactiveParser retroactiveParser; private JEditorPane correctStateHeadline; public static JLabel numberOfParameters = new JLabel(); public static JLabel numberOfParameterValues = new JLabel(); @@ -38,14 +40,17 @@ public class GettingStartedView extends JScrollPane { public static JLabel numberOfMatchValues = new JLabel(); public static JLabel numberOfUrls = new JLabel(); public static JCheckBox detectionActiveCheckbox; - private JButton matchButton; - private JButton purgeButton; + private JEditorPane startValueSetInfoPane; + private JEditorPane endValueSetInfoPane; + private int startValue; + private int endValue; public GettingStartedView(MontoyaApi api, PropertiesHandler propertiesHandler, DeferMatching deferMatching, ParameterHandler parameterHandler, MatchHandler matchHandler, NoiseReductionController noiseReductionController, - QueryViewController queryViewController, AuditFindingView auditFindingView, SessionViewController sessionViewController) { + QueryViewController queryViewController, AuditFindingView auditFindingView, SessionViewController sessionViewController, + RetroactiveParser retroactiveParser) { this.api = api; this.propertiesHandler = propertiesHandler; this.deferMatching = deferMatching; @@ -55,6 +60,9 @@ public GettingStartedView(MontoyaApi api, PropertiesHandler propertiesHandler, D this.queryViewController = queryViewController; this.auditFindingView = auditFindingView; this.sessionViewController = sessionViewController; + this.retroactiveParser = retroactiveParser; + this.startValue = -1; + this.endValue = -1; JPanel mainPanel = new JPanel(new MigLayout()); mainPanel.add(renderHowToUse(), "wrap"); mainPanel.add(renderGettingStarted()); @@ -130,7 +138,6 @@ public void actionPerformed(ActionEvent actionEvent) { if (this.propertiesHandler.isScopeSet()) { scopeSet.setSelected(true); detectionActiveCheckbox.setEnabled(true); - detectionActiveCheckbox.setSelected(true); } else { scopeSet.setSelected(false); detectionActiveCheckbox.setSelected(false); @@ -157,7 +164,17 @@ public void actionPerformed(ActionEvent actionEvent) { infoLabel.setEditable(false); infoLabel.setOpaque(true); infoLabel.setContentType("text/html"); - infoLabel.setText("Click the \"Match Now\" Button to identify Matches"); + infoLabel.setText(""" + + Click the "Match Now" Button to identify Matches +
+ Note: The starting point of the matching is either the burp history entries from the point where FlowMate was loaded, +
+ or after a "Purge Database" reset. +
+ If retroactive parsing was performed, the starting point of the matching is the start point set in retroactive parsing. + + """); matchButton = new JButton("Match Now"); matchButton.addActionListener(new ActionListener() { @Override @@ -177,6 +194,8 @@ public void actionPerformed(ActionEvent actionEvent) { panel.add(infoLabel, "gaptop 5"); panel.add(matchButton, "gapleft 5"); + panel.add(renderRetroactiveParsePane(), "gaptop 5"); + panel.add(renderStatistics(), "wrap"); panel.add(renderDbPurgePane()); @@ -245,6 +264,83 @@ public void itemStateChanged(ItemEvent itemEvent) { return togglePanel; } + public JPanel renderRetroactiveParsePane() { + JPanel panel = new JPanel(new MigLayout()); + + JEditorPane header = new JEditorPane(); + header.setEditable(false); + header.setOpaque(true); + header.setContentType("text/html"); + header.setText("

Retroactive Parameter Parsing

"); + + JEditorPane infoPane = new JEditorPane(); + infoPane.setEditable(false); + infoPane.setOpaque(true); + infoPane.setContentType("text/html"); + infoPane.setText(""" + + If FlowMate was started after already browsing the web application, Retroactive Parsing allows the parsing of parameters from the Burp history.
+ Note: If payloads have already been submitted in the web application, they will be parsed and extracted as well.” +

+ Start Point: Set the start point via the context menu in your Burp history
+ End Point: Set the end point via the context menu in your Burp history
+ Note: Select show all items in the history filter settings to view all entries in the history. + + """); + + + JLabel startValueLabel = new JLabel("Start point:"); + JLabel endValueLabel = new JLabel("End point:"); + startValueSetInfoPane = new JEditorPane(); + startValueSetInfoPane.setEditable(false); + startValueSetInfoPane.setOpaque(true); + startValueSetInfoPane.setContentType("text/html"); + startValueSetInfoPane.setText(""" + +

is NOT set

+ + """); + + endValueSetInfoPane = new JEditorPane(); + endValueSetInfoPane.setEditable(false); + endValueSetInfoPane.setOpaque(true); + endValueSetInfoPane.setContentType("text/html"); + endValueSetInfoPane.setText(""" + +

is NOT set

+ + """); + + + startValueSetInfoPane.setMinimumSize(new Dimension(100,20)); + endValueSetInfoPane.setMinimumSize(new Dimension(100,20)); + + JButton startParsingButton = new JButton("Start Retroactive Parsing"); + + startParsingButton.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent actionEvent) { + if (startValue != -1 && endValue != -1) { + BurpExtender.historyStart = startValue; + propertiesHandler.saveHistoryStartValueInState(startValue); + retroactiveParser.init(startValue, endValue); + } else { + JOptionPane.showMessageDialog(panel, "Set Start and End point from Context menu in Burp history.", "Warning", JOptionPane.WARNING_MESSAGE); + } + } + }); + + panel.add(header, "wrap"); + panel.add(infoPane, "wrap"); + panel.add(startValueLabel, "split 2"); + panel.add(startValueSetInfoPane, "wrap"); + panel.add(endValueLabel, "split 2"); + panel.add(endValueSetInfoPane, "wrap"); + panel.add(startParsingButton, "wrap"); + + return panel; + } + public JPanel renderDbPurgePane() { JPanel panel = new JPanel(new MigLayout()); @@ -290,6 +386,7 @@ public void actionPerformed(ActionEvent actionEvent) { queryViewController.clearDataAndView(); HttpListener.monitoredParameter.clear(); HttpListener.hasActiveSession = false; + resetStartEndValues(); } } }); @@ -302,6 +399,40 @@ public void actionPerformed(ActionEvent actionEvent) { return panel; } + public void setStartValue(int startValue) { + this.startValue = startValue; + startValueSetInfoPane.setText(""" + +

is set

+ + """); + } + + public void setEndValue(int endValue) { + this.endValue = endValue; + endValueSetInfoPane.setText(""" + +

is set

+ + """); + } + + private void resetStartEndValues() { + startValue = -1; + endValue = -1; + + startValueSetInfoPane.setText(""" + +

is NOT set

+ + """); + endValueSetInfoPane.setText(""" + +

is NOT set

+ + """); + } + private void resetStatistics() { numberOfParameters.setText("0"); numberOfParameterValues.setText("0"); diff --git a/src/main/java/gui/ProgressDialog.java b/src/main/java/gui/ProgressDialog.java new file mode 100644 index 0000000..d7c33de --- /dev/null +++ b/src/main/java/gui/ProgressDialog.java @@ -0,0 +1,66 @@ +package gui; + +import net.miginfocom.swing.MigLayout; +import javax.swing.*; +import java.awt.*; +import java.awt.event.ActionEvent; +import java.awt.event.ActionListener; + +public class ProgressDialog { + + private JProgressBar progressBar; + private JDialog progressDialog; + private JTextArea taskOutput; + private JButton closeButton; + private String title; + + public ProgressDialog(String title) { + this.title = title; + } + + public void init() { + progressDialog = new JDialog((Frame) null, title, true); + progressDialog.setResizable(false); + progressDialog.getContentPane().setLayout(new MigLayout("fill")); + progressDialog.setLocationRelativeTo(null); + progressBar = new JProgressBar(JProgressBar.HORIZONTAL); + progressBar.setMinimum(0); + progressBar.setMaximum(100); + progressBar.setSize(280, 20); + progressBar.setStringPainted(true); + taskOutput = new JTextArea(); + taskOutput.setMargin(new Insets(5,5,5,5)); + taskOutput.setEditable(false); + closeButton = new JButton("Close"); + closeButton.addActionListener(new ActionListener() { + @Override + public void actionPerformed(ActionEvent actionEvent) { + progressDialog.setVisible(false); + } + }); + progressDialog.setDefaultCloseOperation(JDialog.DO_NOTHING_ON_CLOSE); + progressDialog.setMinimumSize(new Dimension(300, 150)); + progressDialog.setResizable(false); + progressDialog.add(new JLabel("Progress..."), "north"); + progressDialog.add(new JScrollPane(taskOutput), "dock center, grow"); + progressDialog.add(closeButton, "south"); + progressDialog.add(progressBar,"south"); + progressDialog.setVisible(true); + } + + public void updateProgressBarValue(int value) { + this.progressBar.setValue(value); + } + + public void appendTaskOutput(String str) { + this.taskOutput.append(str); + } + + public void setTaskOutputText(String text) { + this.taskOutput.setText(text); + } + + public void updateDialogDefaultCloseOperation(int operation) { + this.progressDialog.setDefaultCloseOperation(operation); + } +} diff --git a/src/main/java/gui/QueryView.java b/src/main/java/gui/QueryView.java index 1760173..469c2c6 100644 --- a/src/main/java/gui/QueryView.java +++ b/src/main/java/gui/QueryView.java @@ -17,6 +17,7 @@ import net.miginfocom.swing.MigLayout; import javax.swing.*; +import javax.swing.text.DefaultCaret; import java.awt.*; public class QueryView extends JScrollPane { @@ -128,7 +129,7 @@ private void initComponents() { private JPanel initParameterList() { JPanel parameterListPanel = new JPanel(new MigLayout()); JLabel parameterListLabel = new JLabel("List of Parameters:"); - this.parameterJList = new JList<>(containerConverter.parameterToContainer(this.parameterHandler.observableInputParameterList.stream().toList())); + this.parameterJList = new JList<>(containerConverter.parameterToContainer(this.parameterHandler.inputParameterStorage.values().stream().toList())); this.parameterJList.setCellRenderer(new ParameterListCellRenderer()); this.parameterScrollPane = new JScrollPane(parameterJList); @@ -191,6 +192,8 @@ private JPanel initEditors() { // JEditorPane because JLabels are somehow unable to render HTML private JEditorPane initSelectedMessageIdLabel() { selectedMessageId = new JEditorPane(); + DefaultCaret caret = (DefaultCaret)selectedMessageId.getCaret(); + caret.setUpdatePolicy(DefaultCaret.NEVER_UPDATE); selectedMessageId.setContentType("text/html"); selectedMessageId.setEditable(false); selectedMessageId.setOpaque(true); diff --git a/src/main/java/gui/SessionView.java b/src/main/java/gui/SessionView.java index 3781324..3cce18b 100644 --- a/src/main/java/gui/SessionView.java +++ b/src/main/java/gui/SessionView.java @@ -37,7 +37,7 @@ public class SessionView extends JScrollPane { public static JList matchInfoJList; public static JList matchJList; private JScrollPane parameterScrollPane; - public static JList sessionSpecificParameterJList; + public static JList sessionSpecificParameterMatchesJList; public JButton removeFromSessionDefButton; public JButton saveSessionDefinitionButton; @@ -170,11 +170,11 @@ private void initSessionInformationPanelComponents() { parameterListPanel = new JPanel(new MigLayout()); JLabel parameterListLabel = new JLabel("List of Matches during Session:"); DefaultListModel model2 = new DefaultListModel<>(); - sessionSpecificParameterJList = new JList<>(); - sessionSpecificParameterJList.setModel(model2); - sessionSpecificParameterJList.setCellRenderer(new SessionParameterListCellRenderer()); + sessionSpecificParameterMatchesJList = new JList<>(); + sessionSpecificParameterMatchesJList.setModel(model2); + sessionSpecificParameterMatchesJList.setCellRenderer(new SessionParameterListCellRenderer()); - this.parameterScrollPane = new JScrollPane(sessionSpecificParameterJList); + this.parameterScrollPane = new JScrollPane(sessionSpecificParameterMatchesJList); this.parameterScrollPane.setMinimumSize(new Dimension(390, 450)); this.parameterScrollPane.setMaximumSize(new Dimension(390, 450)); diff --git a/src/main/java/model/QueryViewModel.java b/src/main/java/model/QueryViewModel.java index 7527b15..3adccdd 100644 --- a/src/main/java/model/QueryViewModel.java +++ b/src/main/java/model/QueryViewModel.java @@ -1,4 +1,5 @@ package model; +import db.CypherQueryHandler; import db.DBModel; import db.entities.InputValue; import db.entities.MatchValue; @@ -38,20 +39,15 @@ public void loadListData(String paramName, String paramType) { this.occurrenceEntityList.clear(); this.parameterMatchEntityList.clear(); String query = "MATCH (n:InputParameter {name: $param, type: \"%s\"})-[OCCURS_WITH_VALUE]->(o:InputValue)".formatted(paramType) + - "OPTIONAL MATCH (m:ParameterMatch {name: $param, type: \"%s\"})-[MATCH]->(e:MatchValue)".formatted(paramType) + - "RETURN o, m, e"; + "OPTIONAL MATCH (m:ParameterMatch {name: $param, type: \"%s\"})-[MATCH]->(mv:MatchValue)".formatted(paramType) + + "RETURN o, m, mv"; Map params = Map.of("param", paramName); Result queryResult = DBModel.query(query, params); Iterator> results = queryResult.queryResults().iterator(); - while (results.hasNext()) { - Map result = results.next(); - occurrenceEntityList.add((InputValue) result.get("o")); - if (result.get("m") != null) { - parameterMatchEntityList.add((ParameterMatch) result.get("m")); - matchValueEntityList.add((MatchValue) result.get("e")); - } - } + occurrenceEntityList.addAll(CypherQueryHandler.getOccurrencesFromQueryResult(queryResult)); + parameterMatchEntityList.addAll(CypherQueryHandler.getParameterMatchesFromQueryResult(queryResult)); + matchValueEntityList.addAll(CypherQueryHandler.getMatchValuesFromQueryResult(queryResult)); } public void clearAllData() { diff --git a/src/main/java/model/SessionViewModel.java b/src/main/java/model/SessionViewModel.java index b0a6819..d908521 100644 --- a/src/main/java/model/SessionViewModel.java +++ b/src/main/java/model/SessionViewModel.java @@ -2,8 +2,8 @@ import burp.ContainerConverter; import burp.HttpListener; -import burp.RegexMatcher; import burp.api.montoya.MontoyaApi; +import db.CypherQueryHandler; import db.DBModel; import db.MatchHandler; import db.ParameterHandler; @@ -11,9 +11,7 @@ import gui.container.SessionContainer; import org.neo4j.ogm.model.Result; import session.IdentifiedSession; -import db.entities.Session; import session.SessionHelper; -import db.entities.SessionParameter; import utils.Hashing; import java.util.*; @@ -119,7 +117,7 @@ public void changeDatabaseEntriesAccordingToSession(String sessionName, int lowe "RETURN n"; DBModel.executeCypher(query, values); Session sessionToSave = connectEntitiesToSessionNode(sessionName, inputValues, parameterMatches, matchValues); - DBModel.saveSession(sessionToSave); + DBModel.saveEntity(sessionToSave); changeInStorages(nodesToChange, sessionName); } } @@ -129,11 +127,11 @@ public void renameSession(String oldName, String newName) { sessionTable.remove(oldName); session.setName(newName); sessionTable.put(newName, session); - DBModel.saveSession(session); + DBModel.saveEntity(session); } private void changeInStorages(List nodesToChange, String sessionName) { - for (InputParameter param : parameterHandler.parameterStorage.values()) { + for (InputParameter param : parameterHandler.inputParameterStorage.values()) { for (InputValue value : param.getOccurrenceEntities()) { for (Integer id : nodesToChange) { if (value.getIdentifier() == id) { @@ -143,7 +141,7 @@ private void changeInStorages(List nodesToChange, String sessionName) { } } - for (InputValue value : parameterHandler.parameterValueStorage.values()) { + for (InputValue value : parameterHandler.inputValueStorage.values()) { for (Integer id : nodesToChange) { if (value.getIdentifier() == id) { value.setSession(sessionName); @@ -181,66 +179,54 @@ private void changeInStorages(List nodesToChange, String sessionName) { } private List getInputValuesForSessionRange(String sessionName, int lowestHistoryId, int highestHistoryId) { - List inputValues = new ArrayList<>(); - String messageHashesArray = identifyMessageHashes(lowestHistoryId, highestHistoryId); if (messageHashesArray.isEmpty()) { return null; } String query = "UNWIND %s as X ".formatted(messageHashesArray) + - "Match (n:InputValue {messageHash: X}) " + - "RETURN n"; + "Match (o:InputValue {messageHash: X}) " + + "RETURN o"; Result queryResultParameterValues = DBModel.query(query, Map.of()); - Iterator> resultIterator = queryResultParameterValues.queryResults().iterator(); - while (resultIterator.hasNext()) { - Map result = resultIterator.next(); - InputValue inputValue = (InputValue) result.get("n"); + + List inputValues = CypherQueryHandler.getOccurrencesFromQueryResult(queryResultParameterValues); + for (InputValue inputValue : inputValues) { inputValue.setSession(sessionName); - inputValues.add(inputValue); } return inputValues; } private List getParameterMatchesForSessionRange(String sessionName, int lowestHistoryId, int highestHistoryId) { - List parameterMatches = new ArrayList<>(); - String messageHashesArray = identifyMessageHashes(lowestHistoryId, highestHistoryId); if (messageHashesArray.isEmpty()) { return null; } String query = "UNWIND %s as X ".formatted(messageHashesArray) + - "Match (n:ParameterMatch {messageHash: X}) " + - "RETURN n"; - Result queryResultParameterValues = DBModel.query(query, Map.of()); - Iterator> resultIterator = queryResultParameterValues.queryResults().iterator(); - while (resultIterator.hasNext()) { - Map result = resultIterator.next(); - ParameterMatch parameterMatch = (ParameterMatch) result.get("n"); + "Match (m:ParameterMatch {messageHash: X}) " + + "RETURN m"; + Result queryResultParameterMatches = DBModel.query(query, Map.of()); + + List parameterMatches = CypherQueryHandler.getParameterMatchesFromQueryResult(queryResultParameterMatches); + for (ParameterMatch parameterMatch : parameterMatches) { parameterMatch.setSession(sessionName); - parameterMatches.add(parameterMatch); } return parameterMatches; } private List getMatchValuesForSessionRange(String sessionName, int lowestHistoryId, int highestHistoryId) { - List matchValues = new ArrayList<>(); - String messageHashesArray = identifyMessageHashes(lowestHistoryId, highestHistoryId); if (messageHashesArray.isEmpty()) { return null; } String query = "UNWIND %s as X ".formatted(messageHashesArray) + - "Match (n:MatchValue {messageHash: X}) " + - "RETURN n"; + "Match (mv:MatchValue {messageHash: X}) " + + "RETURN mv"; Result queryResultMatchValues = DBModel.query(query, Map.of()); - Iterator> resultIterator = queryResultMatchValues.queryResults().iterator(); - while (resultIterator.hasNext()) { - Map result = resultIterator.next(); - MatchValue matchValue = (MatchValue) result.get("n"); + + List matchValues = CypherQueryHandler.getMatchValuesFromQueryResult(queryResultMatchValues); + for (MatchValue matchValue : matchValues) { matchValue.setSession(sessionName); - matchValues.add(matchValue); } return matchValues; @@ -328,6 +314,8 @@ public void clearAllData() { sessionCounter = 0; activeSessionName = "not set"; sessionTable.clear(); + matchHandler.setHasActiveSession(false); + HttpListener.setHasActiveSession(false); } public void deleteSessionsInDB() { diff --git a/src/main/java/utils/ChangeType.java b/src/main/java/utils/ChangeType.java new file mode 100644 index 0000000..e8c7409 --- /dev/null +++ b/src/main/java/utils/ChangeType.java @@ -0,0 +1,6 @@ +package utils; + +public enum ChangeType { + ADD, REMOVE, SET, CLEAR, BULK_ADD, BULK_REMOVE +} + diff --git a/src/main/java/utils/Logger.java b/src/main/java/utils/Logger.java index 550d9bd..9723bf4 100644 --- a/src/main/java/utils/Logger.java +++ b/src/main/java/utils/Logger.java @@ -1,6 +1,7 @@ package utils; import burp.api.montoya.logging.Logging; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.StandardOpenOption;