Merge pull request #6063 from thc202/client/spider/content

client: flag spidered URLs accessed indirectly
zaproxy · Jan 6, 2025 · 63b5b74 · 63b5b74
2 parents 036607c + ab6e95c
commit 63b5b74
Show file tree

Hide file tree

Showing 7 changed files with 64 additions and 30 deletions.
diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java b/addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java
@@ -504,6 +504,15 @@ public boolean setVisited(String url) {
         return false;
     }
 
+    public boolean setContentLoaded(String url) {
+        ClientNode node = clientTree.setContentLoaded(url);
+        if (node != null) {
+            clientNodeChanged(node);
+            return true;
+        }
+        return false;
+    }
+
     public void deleteNodes(List<ClientNode> nodes) {
         this.clientTree.deleteNodes(nodes);
         if (View.isInitialised()) {

diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientMap.java b/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientMap.java
@@ -59,16 +59,16 @@ public ClientNode getRoot() {
 
     public ClientNode getOrAddNode(String url, boolean visited, boolean storage) {
         LOGGER.debug("getOrAddNode {}", url);
-        return this.getNode(url, visited, storage, true);
+        return this.getNode(url, visited, storage, true, true);
     }
 
     public ClientNode getNode(String url, boolean visited, boolean storage) {
         LOGGER.debug("getNode {}", url);
-        return this.getNode(url, visited, storage, false);
+        return this.getNode(url, visited, storage, false, false);
     }
 
     private synchronized ClientNode getNode(
-            String url, boolean visited, boolean storage, boolean add) {
+            String url, boolean visited, boolean storage, boolean add, boolean publishEvent) {
         if (url == null) {
             throw new IllegalArgumentException("The url parameter should not be null");
         }
@@ -91,7 +91,7 @@ private synchronized ClientNode getNode(
                             new ClientNode(
                                     new ClientSideDetails(nodeName, url, visited, storage),
                                     storage);
-                    if (!storage) {
+                    if (!storage && publishEvent) {
                         Map<String, String> map = new HashMap<>();
                         map.put(URL_KEY, url);
                         // Note we haven't added the child to the parent yet
@@ -208,6 +208,29 @@ public ClientNode setVisited(String url) {
         LOGGER.debug("setVisited, no node for URL or already visited {}", url);
         return null;
     }
+
+    public ClientNode setContentLoaded(String url) {
+        ClientNode node = getNode(url, false, false, true, false);
+        if (node.getUserObject().isVisited()) {
+            return null;
+        }
+
+        node.getUserObject().setContentLoaded(true);
+        node.getUserObject()
+                .addComponent(
+                        new ClientSideComponent(
+                                Map.of(),
+                                ClientSideComponent.CONTENT_LOADED,
+                                null,
+                                null,
+                                null,
+                                ClientSideComponent.CONTENT_LOADED,
+                                null,
+                                null,
+                                -1));
+
+        return node;
+    }
 }
 
 /**

diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideComponent.java b/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideComponent.java
@@ -32,7 +32,9 @@
 @AllArgsConstructor
 public class ClientSideComponent {
 
-    public static String REDIRECT = "Redirect";
+    public static final String REDIRECT = "Redirect";
+
+    public static final String CONTENT_LOADED = "ContentLoaded";
 
     private final Map<String, String> data;
 

diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideDetails.java b/addOns/client/src/main/java/org/zaproxy/addon/client/internal/ClientSideDetails.java
@@ -21,11 +21,14 @@
 
 import java.util.HashSet;
 import java.util.Set;
+import lombok.Getter;
 
+@Getter
 public class ClientSideDetails {
-    private String name;
-    private String url;
+    private final String name;
+    private final String url;
     private boolean visited;
+    private boolean contentLoaded;
     private boolean storage;
     private boolean redirect;
 
@@ -42,18 +45,6 @@ public ClientSideDetails(String name, String url) {
         this(name, url, false, false);
     }
 
-    public String getName() {
-        return name;
-    }
-
-    public String getUrl() {
-        return url;
-    }
-
-    public boolean isVisited() {
-        return visited;
-    }
-
     public Set<ClientSideComponent> getComponents() {
         return components;
     }
@@ -62,22 +53,18 @@ protected void setVisited(boolean visited) {
         this.visited = visited;
     }
 
-    protected boolean addComponent(ClientSideComponent component) {
-        return this.components.add(component);
+    protected void setContentLoaded(boolean contentLoaded) {
+        this.contentLoaded = contentLoaded;
     }
 
-    public boolean isStorage() {
-        return storage;
+    protected boolean addComponent(ClientSideComponent component) {
+        return this.components.add(component);
     }
 
     protected void setStorage(boolean storage) {
         this.storage = storage;
     }
 
-    public boolean isRedirect() {
-        return redirect;
-    }
-
     public void setRedirect(boolean redirect) {
         this.redirect = redirect;
     }

diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java b/addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java
@@ -60,6 +60,7 @@
 import org.zaproxy.addon.client.ExtensionClientIntegration;
 import org.zaproxy.addon.client.internal.ClientMap;
 import org.zaproxy.addon.client.internal.ClientNode;
+import org.zaproxy.addon.client.internal.ClientSideDetails;
 import org.zaproxy.addon.client.spider.actions.ClickElement;
 import org.zaproxy.addon.client.spider.actions.OpenUrl;
 import org.zaproxy.addon.client.spider.actions.SubmitForm;
@@ -274,8 +275,12 @@ private List<String> getUnvisitedUrls() {
     }
 
     private void getUnvisitedUrls(ClientNode node, List<String> urls) {
-        String nodeUrl = node.getUserObject().getUrl();
-        if (!node.isStorage() && !node.getUserObject().isVisited() && isUrlInScope(nodeUrl)) {
+        ClientSideDetails details = node.getUserObject();
+        String nodeUrl = details.getUrl();
+        if (!node.isStorage()
+                && !details.isVisited()
+                && !details.isContentLoaded()
+                && isUrlInScope(nodeUrl)) {
             urls.add(nodeUrl);
         }
         for (int i = 0; i < node.getChildCount(); i++) {
@@ -580,6 +585,9 @@ private void finished() {
             clear(webDriverPool);
             clear(webDriverActive);
         }
+
+        crawledUrls.forEach(extClient::setContentLoaded);
+
         if (listener != null) {
             listener.scanFinshed(scanId, displayName);
         }

diff --git a/addOns/client/src/main/java/org/zaproxy/addon/client/ui/ClientMapTreeCellRenderer.java b/addOns/client/src/main/java/org/zaproxy/addon/client/ui/ClientMapTreeCellRenderer.java
@@ -48,6 +48,8 @@ public class ClientMapTreeCellRenderer extends DefaultTreeCellRenderer {
             ExtensionClientIntegration.getIcon("overlay-minus.png");
     private static final ImageIcon REDIRECT_OVERLAY =
             ExtensionClientIntegration.getIcon("overlay-redirect.png");
+    private static final ImageIcon CONTENT_LOADED_OVERLAY =
+            ExtensionClientIntegration.getIcon("overlay-content-loaded.png");
     private static final ImageIcon DATABASE_ICON =
             ExtensionClientIntegration.getIcon("database.png");
 
@@ -97,7 +99,10 @@ public Component getTreeCellRendererComponent(
                         icon = new OverlayIcon(LEAF_ICON);
                     }
                     if (!csd.isVisited()) {
-                        icon.add(NOT_VISITED_OVERLAY);
+                        icon.add(
+                                csd.isContentLoaded()
+                                        ? CONTENT_LOADED_OVERLAY
+                                        : NOT_VISITED_OVERLAY);
                     } else if (csd.isRedirect()) {
                         icon.add(REDIRECT_OVERLAY);
                     }

diff --git a/...rc/main/resources/org/zaproxy/addon/client/resources/overlay-content-loaded.png b/...rc/main/resources/org/zaproxy/addon/client/resources/overlay-content-loaded.png