From ab6e95c20e37fa8471dab409a2a6ab82871c3f74 Mon Sep 17 00:00:00 2001 From: thc202 Date: Mon, 6 Jan 2025 13:08:27 +0000 Subject: [PATCH] client: flag spidered URLs accessed indirectly Flag the URLs that were accessed by the browser as side effect of the spidering process (e.g. loaded scripts, images, submitted forms). Signed-off-by: thc202 --- .../client/ExtensionClientIntegration.java | 9 +++++ .../addon/client/internal/ClientMap.java | 31 +++++++++++++++--- .../client/internal/ClientSideComponent.java | 4 ++- .../client/internal/ClientSideDetails.java | 31 +++++------------- .../addon/client/spider/ClientSpider.java | 12 +++++-- .../client/ui/ClientMapTreeCellRenderer.java | 7 +++- .../resources/overlay-content-loaded.png | Bin 0 -> 734 bytes 7 files changed, 64 insertions(+), 30 deletions(-) create mode 100644 addOns/client/src/main/resources/org/zaproxy/addon/client/resources/overlay-content-loaded.png diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java b/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java index c545a0c9cb1..e9db3aeaf6f 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java @@ -504,6 +504,15 @@ public boolean setVisited(String url) { return false; } + public boolean setContentLoaded(String url) { + ClientNode node = clientTree.setContentLoaded(url); + if (node != null) { + clientNodeChanged(node); + return true; + } + return false; + } + public void deleteNodes(List nodes) { this.clientTree.deleteNodes(nodes); if (View.isInitialised()) { diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientMap.java b/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientMap.java index 3b25f535b05..a5b139aa540 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientMap.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientMap.java @@ -59,16 +59,16 @@ public ClientNode getRoot() { public ClientNode getOrAddNode(String url, boolean visited, boolean storage) { LOGGER.debug("getOrAddNode {}", url); - return this.getNode(url, visited, storage, true); + return this.getNode(url, visited, storage, true, true); } public ClientNode getNode(String url, boolean visited, boolean storage) { LOGGER.debug("getNode {}", url); - return this.getNode(url, visited, storage, false); + return this.getNode(url, visited, storage, false, false); } private synchronized ClientNode getNode( - String url, boolean visited, boolean storage, boolean add) { + String url, boolean visited, boolean storage, boolean add, boolean publishEvent) { if (url == null) { throw new IllegalArgumentException("The url parameter should not be null"); } @@ -91,7 +91,7 @@ private synchronized ClientNode getNode( new ClientNode( new ClientSideDetails(nodeName, url, visited, storage), storage); - if (!storage) { + if (!storage && publishEvent) { Map map = new HashMap<>(); map.put(URL_KEY, url); // Note we haven't added the child to the parent yet @@ -208,6 +208,29 @@ public ClientNode setVisited(String url) { LOGGER.debug("setVisited, no node for URL or already visited {}", url); return null; } + + public ClientNode setContentLoaded(String url) { + ClientNode node = getNode(url, false, false, true, false); + if (node.getUserObject().isVisited()) { + return null; + } + + node.getUserObject().setContentLoaded(true); + node.getUserObject() + .addComponent( + new ClientSideComponent( + Map.of(), + ClientSideComponent.CONTENT_LOADED, + null, + null, + null, + ClientSideComponent.CONTENT_LOADED, + null, + null, + -1)); + + return node; + } } /** diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideComponent.java b/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideComponent.java index 59097ac8422..fa9b66df809 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideComponent.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideComponent.java @@ -32,7 +32,9 @@ @AllArgsConstructor public class ClientSideComponent { - public static String REDIRECT = "Redirect"; + public static final String REDIRECT = "Redirect"; + + public static final String CONTENT_LOADED = "ContentLoaded"; private final Map data; diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideDetails.java b/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideDetails.java index 9427917bc82..c11b29f0496 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideDetails.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideDetails.java @@ -21,11 +21,14 @@ import java.util.HashSet; import java.util.Set; +import lombok.Getter; +@Getter public class ClientSideDetails { - private String name; - private String url; + private final String name; + private final String url; private boolean visited; + private boolean contentLoaded; private boolean storage; private boolean redirect; @@ -42,18 +45,6 @@ public ClientSideDetails(String name, String url) { this(name, url, false, false); } - public String getName() { - return name; - } - - public String getUrl() { - return url; - } - - public boolean isVisited() { - return visited; - } - public Set getComponents() { return components; } @@ -62,22 +53,18 @@ protected void setVisited(boolean visited) { this.visited = visited; } - protected boolean addComponent(ClientSideComponent component) { - return this.components.add(component); + protected void setContentLoaded(boolean contentLoaded) { + this.contentLoaded = contentLoaded; } - public boolean isStorage() { - return storage; + protected boolean addComponent(ClientSideComponent component) { + return this.components.add(component); } protected void setStorage(boolean storage) { this.storage = storage; } - public boolean isRedirect() { - return redirect; - } - public void setRedirect(boolean redirect) { this.redirect = redirect; } diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java index e1af505a2ac..a9a8e2dfd9b 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java @@ -60,6 +60,7 @@ import org.zaproxy.addon.client.ExtensionClientIntegration; import org.zaproxy.addon.client.internal.ClientMap; import org.zaproxy.addon.client.internal.ClientNode; +import org.zaproxy.addon.client.internal.ClientSideDetails; import org.zaproxy.addon.client.spider.actions.ClickElement; import org.zaproxy.addon.client.spider.actions.OpenUrl; import org.zaproxy.addon.client.spider.actions.SubmitForm; @@ -274,8 +275,12 @@ private List getUnvisitedUrls() { } private void getUnvisitedUrls(ClientNode node, List urls) { - String nodeUrl = node.getUserObject().getUrl(); - if (!node.isStorage() && !node.getUserObject().isVisited() && isUrlInScope(nodeUrl)) { + ClientSideDetails details = node.getUserObject(); + String nodeUrl = details.getUrl(); + if (!node.isStorage() + && !details.isVisited() + && !details.isContentLoaded() + && isUrlInScope(nodeUrl)) { urls.add(nodeUrl); } for (int i = 0; i < node.getChildCount(); i++) { @@ -575,6 +580,9 @@ private void finished() { clear(webDriverPool); clear(webDriverActive); } + + crawledUrls.forEach(extClient::setContentLoaded); + if (listener != null) { listener.scanFinshed(scanId, displayName); } diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/ui/ClientMapTreeCellRenderer.java b/addOns/client/src/main/java/org/zaproxy/addon/client/ui/ClientMapTreeCellRenderer.java index 13481438f94..780a2ad086f 100644 --- a/addOns/client/src/main/java/org/zaproxy/addon/client/ui/ClientMapTreeCellRenderer.java +++ b/addOns/client/src/main/java/org/zaproxy/addon/client/ui/ClientMapTreeCellRenderer.java @@ -48,6 +48,8 @@ public class ClientMapTreeCellRenderer extends DefaultTreeCellRenderer { ExtensionClientIntegration.getIcon("overlay-minus.png"); private static final ImageIcon REDIRECT_OVERLAY = ExtensionClientIntegration.getIcon("overlay-redirect.png"); + private static final ImageIcon CONTENT_LOADED_OVERLAY = + ExtensionClientIntegration.getIcon("overlay-content-loaded.png"); private static final ImageIcon DATABASE_ICON = ExtensionClientIntegration.getIcon("database.png"); @@ -97,7 +99,10 @@ public Component getTreeCellRendererComponent( icon = new OverlayIcon(LEAF_ICON); } if (!csd.isVisited()) { - icon.add(NOT_VISITED_OVERLAY); + icon.add( + csd.isContentLoaded() + ? CONTENT_LOADED_OVERLAY + : NOT_VISITED_OVERLAY); } else if (csd.isRedirect()) { icon.add(REDIRECT_OVERLAY); } diff --git a/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/overlay-content-loaded.png b/addOns/client/src/main/resources/org/zaproxy/addon/client/resources/overlay-content-loaded.png new file mode 100644 index 0000000000000000000000000000000000000000..30f9e8298b1949d01def34612bb4e9cef1721c99 GIT binary patch literal 734 zcmV<40wMj0P)EX>4Tx04R}tkv&MmP!xqvTeYH<4t5Z6$WWc^qN0wq3Pq?8YK2xEOkVm2O&XFE z7e~Rh;NZ_<)xpJCR|i)?5c~mg7n~Gbq{RD@LW>wLJl@B7_Z;545AZiCOf|dWfT~$W zI++l%xm6+f3LkpVj~Rp|X6kdIn1W}0-BUN!U5saW_x)J|O5S9EPauvn-LQx^h^IF# zo%23%gq0+P_?&p$pbHW|a$R=$jdRgqKhKO9ne;qygjgtcu-w6{WT?bb#8E}nDBquT zS>e3JS*_Gq>z@3Dp}e-T%yn8LNMaF75FtQD4P{hdAx67KiitFxCp`Saj$b5~Os+B* zITlcb3d!+<|H1FsnuW;;Hz||=x?gPjV-yJP0?oQ@e;?a+^8^Sy16NwxUu^)hpQP8@ zTJ#7AZvz+CZB5<-E_Z;TCtWfmM+(sL7Ye}p8GTa@7`z4g*4(+Z&T;wxWN22Y8{ps& z7%Nity2rbFJLmRqPiuZZd+>6q{#J@#00006VoOIv0RI600RN!9r;`8x010qNS#tmY z3ljhU3ljkVnw%H_008buL_t(I%hi%WPQm~XMIY5FX{SweB^1#^cn(kKRlI-)5$lQ? zpxAW&xF9hB7A0Y$v;4__dHEUm_cuBWUS8iKLRe<3H!Y9T2f9wUi-_R)soLvY9s=gG z8B!1-%AzDqF97H|aMzK+5#YUEV41OO8X|;T^t~95Gghn3FkoI)NI_W?zb5ja?{`Ln zfW>ExyCVgb8_Q)wVTN3+H|;RszM3Kha_vm4*xTfU4QedUX__zG9m@?VXxlAUX*vv; zO(sMHrR|Ev^%w~v0x3?}-~$C%mb3ldQr=Z0Ng$0#lE3=Sx95}-;N~2_4@UfnEDbcQ Q5dZ)H07*qoM6N<$f~tc&O#lD@ literal 0 HcmV?d00001