Updated TrickyHttpClient for better hosts 'tricking'

+ readme updated
+ minor defect fixes

PriceChecker.Core/Services/TrickyHttpClient.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Concurrent;
+using System.Net;
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
@@ -16,7 +17,7 @@ public class TrickyHttpClient : ITrickyHttpClient
     {
         private readonly ConcurrentDictionary<string, SemaphoreSlim> _lockers = new();
         private readonly ILogger<TrickyHttpClient> _logger;
-        private static object _locker = new object();
+        private static object _locker = new();
 
         private const int DELAY_MS = 500;
 
@@ -44,12 +45,18 @@ public async Task<string> DownloadContent(string url)
 
         private async Task<string> DownloadInternal(string url)
         {
-            var httpClient = new HttpClient();
+            var handler = new HttpClientHandler()
+            {
+                AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate
+            };
+            var httpClient = new HttpClient(handler);
 
             // To confuse the hosts
             httpClient.DefaultRequestHeaders.Add("X-Cookies-Accepted", "1");
-            httpClient.DefaultRequestHeaders.Add("accept", "text/html");
-            httpClient.DefaultRequestHeaders.Add("user-agent", "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; WOW64; Trident/6.0)");
+            httpClient.DefaultRequestHeaders.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9");
+            httpClient.DefaultRequestHeaders.Add("Accept-Language", "en-US,en;q=0.9");
+            httpClient.DefaultRequestHeaders.Add("Accept-Encoding", "gzip, deflate");
+            httpClient.DefaultRequestHeaders.Add("User-Agent", CreateRandomUserAgent());
 
             var response = await httpClient.GetAsync(url);
             if (!response.IsSuccessStatusCode)
@@ -60,5 +67,53 @@ private async Task<string> DownloadInternal(string url)
             }
             return await response.Content.ReadAsStringAsync();
         }
+
+        private string CreateRandomUserAgent()
+        {
+            var platform = new [] { "Machintosh", "Windows", "X11" }.TakeRandom();
+            var os = (platform switch {
+                "Machintosh" => new [] { "68K", "PPC" },
+                "Windows" => new [] { "Win3.11", "WinNT3.51", "WinNT4.0", "Windows NT 5.0", "Windows NT 5.1", "Windows NT 5.2", "Windows NT 6.0", "Windows NT 6.1", "Windows NT 6.2", "Win95", "Win98", "Win 9x 4.90", "WindowsCE" },
+                "X11" => new [] { "Linux i686", "Linux x86_64" },
+                _ => new string[] {}
+            }).TakeRandom();
+            var browser = new [] { "Chrome", "Firefox", "IE" }.TakeRandom();
+
+            if (browser == "Chrome")
+            {
+			    var webkit = Utils.RandomInt(500, 599).ToString();
+			    var version = $"{Utils.RandomInt(0, 24)}.0{Utils.RandomInt(0, 1500)}.{Utils.RandomInt(0, 999)}";
+
+			    return $"Mozilla/5.0 ({os}) AppleWebKit{webkit}.0 (KHTML, live Gecko) Chrome/{version} Safari/{webkit}";
+            }
+            if (browser == "Firefox")
+            {
+                var year = Utils.RandomInt(2000, 2021);
+                var month = Utils.RandomInt(1, 12);
+                var day = Utils.RandomInt(1, 28);
+                var gecko = $"{year}{month:00}{day:00}";
+                var version = $"{Utils.RandomInt(1, 15)}.0";
+
+                return $"Mozillia/5.0 ({os}; rv:{version}) Gecko/{gecko} Firefox/{version}";
+            }
+            if (browser == "IE")
+            {
+                var version = $"{Utils.RandomInt(1, 10)}.0";
+                var engine = $"{Utils.RandomInt(1, 5)}.0";
+                var option = Utils.RandomBool();
+                string token;
+                if (option)
+                {
+                    var v = new [] { ".NET CLR", "SV1", "Tablet PC", "Win64; IA64", "Win64; x64", "WOW64" }.TakeRandom();
+                    token = $"{v};";
+                }
+                else
+                    token = "";
+
+                return $"Mozilla/5.0 (compatible; MSIE {version}; {os}; {token}Trident/{engine})";
+            }
+
+            throw new NotImplementedException();
+        }
     }
-}
+}

PriceChecker.Core/Utils.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+
+namespace Genius.PriceChecker.Core
+{
+    public static class Utils
+    {
+        private static Random _rnd = new();
+
+        public static bool RandomBool()
+            => _rnd.NextDouble() >= 0.5;
+
+        public static int RandomInt(int from, int to)
+            => _rnd.Next(from, to);
+
+        public static T TakeRandom<T>(this IList<T> list)
+        {
+            if (list.Count == 0)
+                return default (T);
+            return list[_rnd.Next(0, list.Count - 1)];
+        }
+    }
+}

PriceChecker.UI/ViewModels/TrackerProductViewModel.cs

@@ -163,7 +163,9 @@ private void CommitProduct()
 
         private TrackerProductSourceViewModel CreateSourceViewModel(ProductSource productSource)
         {
-            var lastPrice = _product?.Recent?.FirstOrDefault(x => x.ProductSourceId == productSource.Id)?.Price;
+            var lastPrice = productSource == null || _product?.Recent == null
+                ? null
+                : _product.Recent.FirstOrDefault(x => x.ProductSourceId == productSource.Id)?.Price;
             var vm = new TrackerProductSourceViewModel(_ui, productSource, lastPrice);
             vm.DeleteCommand.Executed += (_, __) => {
                 Sources.Remove(vm);

README.md

@@ -1,2 +1,11 @@
 # price-checker
 A simple price checker for the selected products to catch the cheapest deal
+
+### Main window:
+![Main window](docs/assets/MainWindow.png)
+
+### Product edit:
+![Product edit](docs/assets/ProductEdit.png)
+
+### Product with updated lowest price:
+![Product with updated lowest price](docs/assets/ProductWithUpdatedLowestPrice.png)

agents.json

@@ -40,11 +40,5 @@
     "Url": "https://www.viofo.nl/en/collections/viofo-dashcams/products/{0}",
     "PricePattern": "\u0022og:price:amount\u0022 content=\u0022(?\u003Cprice\u003E[\\d,]\u002B)\u0022\u003E",
     "DecimalDelimiter": ","
-  },
-  {
-    "Id": "t1",
-    "Url": "t2",
-    "PricePattern": "t3",
-    "DecimalDelimiter": "."
   }
 ]