Merge pull request #9 from wiejakp/v0.4.2

V0.4.2

README.md

@@ -1,7 +1,7 @@
 # Python Sitemap Generator
 
-- Version: 0.4.1
-- Update: 2021/12/30
+- Version: 0.4.2
+- Update: 2022/12/26
 - Author: Przemek Wiejak @ [email protected]
 
 Python Site Map Generator uses python multi-threaded approach to read all links accessible through the Web site and generate proper sitemap for SEO purposes. 

changelog.txt

@@ -1,3 +1,11 @@
+Date: 2022/12/26
+Version: 0.4.2
+
+- Add MIT license (thank you https://github.com/kylebarney)
+- Fixed timeout issues throwing URLError and handling exceptions properly
+
+~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
+
 Date: 2021/12/30
 Version: 0.4.1
 

python-sitemap-generator.py

@@ -11,6 +11,7 @@
 import sys
 from urllib.request import urlopen
 from urllib.request import Request
+from urllib.error import URLError
 from urllib.request import HTTPError
 from urllib.parse import urljoin
 from urllib.parse import urlparse
@@ -36,9 +37,9 @@
 
 # adjust to your liking
 # keep values low to prevent firewalls blocking you for flooding
-MaxThreads = 5
+MaxThreads = 20
 
-# DWFINE YOUR URL - CUSTOM URL!
+# DEFINE YOUR URL - CUSTOM URL!
 InitialURL = 'HTTPS://SOME_URL.TEST/'
 
 InitialURLInfo = urlparse(InitialURL)
@@ -231,9 +232,13 @@ def run(self):
                         #var_dump(temp_content)
                         pass
 
-
+        except URLError as e:
+            print ('URLError: ', self.obj['url'])
+            temp_status = 000
+            pass
 
         except HTTPError as e:
+            print ('HTTPError: ', self.obj['url'])
             temp_status = e.code
             pass