Merge pull request #6064 from psiinon/client/spider-enable

Client: enable spider + help

addOns/client/CHANGELOG.md

@@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ### Added
 - Added support for Browser Based Authentication when installed in conjunction with the Auth Helper add-on.
+- Client spider, along with Automation Framework support.
 
 ## [0.9.0] - 2024-11-29
 ### Changed

addOns/client/src/main/java/org/zaproxy/addon/client/ExtensionClientIntegration.java

@@ -196,16 +196,13 @@ public void hook(ExtensionHook extensionHook) {
             extensionHook.getHookView().addWorkPanel(getClientDetailsPanel());
             extensionHook.getHookView().addStatusPanel(getClientHistoryPanel());
 
-            if (Constant.isDevMode()) {
-                // Not for release .. yet ;)
-                extensionHook.getHookMenu().addToolsMenuItem(getMenuItemCustomScan());
-                extensionHook
-                        .getHookMenu()
-                        .addPopupMenuItem(
-                                new PopupMenuSpider(
-                                        Constant.messages.getString("client.attack.spider"), this));
-                extensionHook.getHookView().addStatusPanel(getClientSpiderPanel());
-            }
+            extensionHook.getHookMenu().addToolsMenuItem(getMenuItemCustomScan());
+            extensionHook
+                    .getHookMenu()
+                    .addPopupMenuItem(
+                            new PopupMenuSpider(
+                                    Constant.messages.getString("client.attack.spider"), this));
+            extensionHook.getHookView().addStatusPanel(getClientSpiderPanel());
 
             // Client Map menu items
             extensionHook
@@ -815,10 +812,6 @@ public void resumeAllScans() {
     }
 
     private void addScanToUi(final ClientSpider scan) {
-        if (!Constant.isDevMode()) {
-            return;
-        }
-
         if (!EventQueue.isDispatchThread()) {
             SwingUtilities.invokeLater(() -> addScanToUi(scan));
             return;
@@ -837,9 +830,7 @@ public void sessionAboutToChange(Session session) {
             spiderScanController.reset();
 
             if (hasView()) {
-                if (Constant.isDevMode()) {
-                    getClientSpiderPanel().reset();
-                }
+                getClientSpiderPanel().reset();
                 if (spiderDialog != null) {
                     spiderDialog.reset();
                 }
@@ -848,14 +839,14 @@ public void sessionAboutToChange(Session session) {
 
         @Override
         public void sessionChanged(final Session session) {
-            if (hasView() && Constant.isDevMode()) {
+            if (hasView()) {
                 ThreadUtils.invokeAndWaitHandled(getClientSpiderPanel()::reset);
             }
         }
 
         @Override
         public void sessionScopeChanged(Session session) {
-            if (hasView() && Constant.isDevMode()) {
+            if (hasView()) {
                 getClientSpiderPanel().sessionScopeChanged(session);
             }
         }
@@ -867,9 +858,7 @@ public void sessionModeChanged(Mode mode) {
             }
 
             if (hasView()) {
-                if (Constant.isDevMode()) {
-                    getClientSpiderPanel().sessionModeChanged(mode);
-                }
+                getClientSpiderPanel().sessionModeChanged(mode);
                 getMenuItemCustomScan().setEnabled(!Mode.safe.equals(mode));
             }
         }

addOns/client/src/main/java/org/zaproxy/addon/client/spider/ClientSpider.java

@@ -89,7 +89,6 @@ public class ClientSpider implements EventConsumer, GenericScanner2 {
      *
      * TODO The following features will need to be implemented before the first release:
      * 		Support for modes
-     * 		Help pages
      *
      * The following features should be implemented in future releases:
      * 		Clicking on likely navigation elements

addOns/client/src/main/java/org/zaproxy/addon/client/ui/PopupMenuClientAttack.java

@@ -43,12 +43,9 @@ public boolean isEnableForComponent(Component invoker) {
             JTree tree = (JTree) invoker;
             if (ClientMapPanel.CLIENT_TREE_NAME.equals(tree.getName())) {
                 removeAll();
-                if (Constant.isDevMode()) {
-                    // Not for release .. yet ;)
-                    add(new PopupClientSpider(clientMapPanel));
-                    List<ClientNode> nodes = clientMapPanel.getSelectedNodes();
-                    this.setEnabled(nodes.size() == 1 && !nodes.get(0).isRoot());
-                }
+                add(new PopupClientSpider(clientMapPanel));
+                List<ClientNode> nodes = clientMapPanel.getSelectedNodes();
+                this.setEnabled(nodes.size() == 1 && !nodes.get(0).isRoot());
                 return true;
             }
         }

addOns/client/src/main/javahelp/org/zaproxy/addon/client/resources/help/contents/ajax-scan.html

@@ -18,6 +18,8 @@ <H1>Client Side Integration - AJAX Spider Enhancement</H1>
 <p>
 If it finds any 'missed' URLs that were part of the AJAX Spider scan scope then it makes direct requests to these URLs.
 You will be able to see these requests in the History, Sites Tree, and Output tabs.
+<p>
+This add-on also adds a <a href="spider.html">Client Spider</a> which is designed to explore modern web apps more effectively.
 
 </BODY>
 </HTML>

addOns/client/src/main/javahelp/org/zaproxy/addon/client/resources/help/contents/automation.html

@@ -0,0 +1,35 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<HTML>
+<HEAD>
+<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=utf-8">
+<TITLE>
+Client Side Integration - Automation Framework Support
+</TITLE>
+</HEAD>
+<BODY>
+<H1>Client Side Integration - Automation Framework Support</H1>
+This add-on supports the Automation Framework.
+
+<H2>Job: spiderClient</H2>
+The spiderClient job allows you to run the <a href="spider.html">Client Spider</a>, which is designed to explore modern web apps more effectively.
+<p>
+This job supports monitor tests.
+
+<pre>
+  - type: spiderClient                 # The client spider - a spider which explores modern web apps more effectively
+    parameters:
+      context:                         # String: Name of the context to spider, default: first context
+      user:                            # String: An optional user to use for authentication, must be defined in the env
+      url:                             # String: URL to start spidering from, default: first context URL
+      maxDuration:                     # Int: The max time in minutes the spider will be allowed to run for, default: 0 unlimited
+      maxCrawlDepth:                   # Int: The maximum tree depth to explore, default 5
+      maxChildren:                     # Int: The maximum number of children to add to each node in the tree
+      numberOfBrowsers:                # Int: The number of browsers the spider will use, more will be faster but will use up more memory, default 2 x number of cores
+      browserId:                       # String: Browser ID to use, default: firefox-headless
+      initialLoadTime:                 # Int: The time in seconds to wait after the initial URL is loaded, default: 5
+      pageLoadTime:                    # Int: The time in seconds to wait after a new URL is loaded, default: 1
+      shutdownTime:                    # Int: The time in seconds to wait after no activity before shutting down, default: 5</pre>
+</pre>
+
+</BODY>
+</HTML>

addOns/client/src/main/javahelp/org/zaproxy/addon/client/resources/help/contents/client.html

@@ -8,6 +8,14 @@
 <BODY>
 <H1>Client Side Integration</H1>
 
+This add-on adds the following features to ZAP.
+
+<h2>Client Spider</h2>
+
+This add-on adds a <a href="spider.html">Client Spider</a> which is designed to explore modern web apps more effectively.
+
+<h2>Browser Extensions</h2>
+
 This add-on adds a ZAP browser extension to both Firefox and Chrome when they are launched from ZAP.
 <p>
 The extensions stream significant browser based events back to ZAP, giving ZAP an insight into what is going on in the browser.
@@ -16,7 +24,7 @@ <H1>Client Side Integration</H1>
 
 <ul>
 <li>Firefox <a href="https://addons.mozilla.org/en-GB/firefox/addon/zap-browser-extension">ZAP Browser Extension</a>
-<li>Chrome <a href="https://chrome.google.com/webstore/detail/zap-browser-extension/cnmficmodhagepcmhogkbdakncebckho">ZAP Browser Extension</a>
+<li>Chrome <a href="https://chromewebstore.google.com/detail/zap-by-checkmarx-browser/hkjompcghnoikggknlpadkiphjbffilc">ZAP Browser Extension</a>
 </ul>
 
 Note that you may need to configure the extensions via their options pages to update the host and API key.
@@ -38,7 +46,32 @@ <h3>Client Map</h3>
 This means that the Client Map looks very different to the Sites tree for modern web apps, and may give you a better understanding of the
 client side structure of the sites.
 <p>
-Any leaf nodes in the Map with a small red 'minus' sign represent URLs which have been found in the DOM but which have not been directly accessed by ZAP.
+The following icons are used for leaf nodes so that you can easily tell their status:
+
+<table>
+	<tr>
+		<td><img src="images/node-standard.png" align="bottom" width="16" height="16" /></td>
+		<td>A URL which will be present in both the Site Tree and the Client Map</td>
+	</tr>
+	<tr>
+		<td><img src="images/node-fragment.png" align="bottom" width="16" height="16" /></td>
+		<td>A URL which will only be present in Client Map as the URL contains a fragment</td>
+	</tr>
+	<tr>
+		<td><img src="images/node-minus.png" align="bottom" width="16" height="16" /></td>
+		<td>A URL which was found in the DOM but which has not been accessed yet</td>
+	</tr>
+	<tr>
+		<td><img src="images/node-redirect.png" align="bottom" width="16" height="16" /></td>
+		<td>A URL which redirects to another URL</td>
+	</tr>
+	<tr>
+		<td><img src="images/node-content-loaded.png" align="bottom" width="16" height="16" /></td>
+		<td>A URL which was loaded as content by the browser (e.g. JavaScript files) rather than as HTML</td>
+	</tr>
+</table>
+
+
 <p>
 Selecting a node that has been visited by ZAP will display details about that node in the Client Details tab.
 <p>

addOns/client/src/main/javahelp/org/zaproxy/addon/client/resources/help/contents/spider.html

@@ -0,0 +1,33 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<HTML>
+<HEAD>
+<TITLE>
+Client Side Integration - Client Spider
+</TITLE>
+</HEAD>
+<BODY>
+<H1>Client Side Integration - Client Spider</H1>
+
+This <a href="client.html">add-on</a> adds a Client Spider which is designed to explore modern web apps more effectively.
+<p>
+The Client Spider works in a similar way to the AJAX Spider but it has access to the DOM via the ZAP Browser Extension
+which means that it can find content which the AJAX Spider cannot find.
+<p>
+While it is still at an early stage we believe it is a more effective approach than the AJAX Spider.
+<br>
+We will be focussing on improving the Client Spider and the current plan is for it to supersede the AJAX Spider
+as the recommended way of crawling modern web apps.
+<br>
+We would appreciate feedback via the <a href="https://groups.google.com/group/zaproxy-users">ZAP User Group</a>
+to let us know how effectively it works for you, especially in comparison with the AJAX Spider.
+
+<p>
+The spider can be invoked via:
+<ul>
+<li>Context specific "Attack" menu
+<li>"Tools / Client Spider" menu item
+<li>Automation Framework <a href="automation.html">spiderClient</a> job
+</ul>
+
+</BODY>
+</HTML>

addOns/client/src/main/javahelp/org/zaproxy/addon/client/resources/help/index.xml

@@ -6,6 +6,8 @@
 <index version="2.0">
     <!-- index entries are merged (sorted) into core index -->
 	<indexitem text="client" target="addon.client" />
+	<indexitem text="client automation" target="addon.client.auto" />
+	<indexitem text="client spider" target="addon.client.spider" />
 	<indexitem text="client passive scanning" target="addon.client.pscan" />
 	<indexitem text="client ajax spider enhancement" target="addon.client.ajaxscan" />
 	<indexitem text="client firefox profile" target="addon.client.fxprofile" />

addOns/client/src/main/javahelp/org/zaproxy/addon/client/resources/help/map.jhm

@@ -6,6 +6,8 @@
 <map version="1.0">
     <mapID target="addon.client-icon" url="contents/images/application-browser.png" />
     <mapID target="addon.client" url="contents/client.html" />
+    <mapID target="addon.client.auto" url="contents/automation.html" />
+    <mapID target="addon.client.spider" url="contents/spider.html" />
     <mapID target="addon.client.pscan" url="contents/pscan.html" />
     <mapID target="addon.client.ajaxscan" url="contents/ajax-scan.html" />
     <mapID target="addon.client.fxprofile" url="contents/firefox-profile.html" />

addOns/client/src/main/javahelp/org/zaproxy/addon/client/resources/help/toc.xml

@@ -7,6 +7,8 @@
 	<tocitem text="ZAP User Guide" tocid="toplevelitem">
 		<tocitem text="Add Ons" tocid="addons">
 			<tocitem text="Client Side Integration" image="client-icon" target="addon.client">
+				<tocitem text="Automation" target="addon.client.auto"/>
+				<tocitem text="Spider" target="addon.client.spider"/>
 				<tocitem text="Passive Scanning" target="addon.client.pscan"/>
 				<tocitem text="AJAX Spider Enhancement" target="addon.client.ajaxscan"/>
 				<tocitem text="Firefox Profile" target="addon.client.fxprofile"/>