From a2f21af78e41f24510c93c021dbe4a08d090b21e Mon Sep 17 00:00:00 2001 From: Hedon-dev <158850208+Hedon-dev@users.noreply.github.com> Date: Mon, 30 Sep 2024 00:05:26 +0200 Subject: [PATCH] Add warning print for unexpected null values --- lib/backend/universal_formats.dart | 49 ++++++++++++++++++++++ lib/plugins/official_plugins/pornhub.dart | 23 +++++++--- lib/plugins/official_plugins/xhamster.dart | 25 +++++++++-- 3 files changed, 88 insertions(+), 9 deletions(-) diff --git a/lib/backend/universal_formats.dart b/lib/backend/universal_formats.dart index 42ee549d..850f7a21 100644 --- a/lib/backend/universal_formats.dart +++ b/lib/backend/universal_formats.dart @@ -174,6 +174,23 @@ class UniversalSearchResult { // convert all dynamics to strings, as logger only accepts strings logger.d(result.map((key, value) => MapEntry(key, value.toString()))); } + + /// Print values that are null, but the plugin didn't expect to be null + // TODO: Set up automatic/user prompted reporting + void printNullKeys(String pluginCodeName, List exceptions) { + Map objectAsMap = convertToMap(); + List nullKeys = []; + // Check whether key is not in exception list and whether value is null + objectAsMap.forEach((key, value) { + if (!exceptions.contains(key) && value == null) { + nullKeys.add(key); + } + }); + if (nullKeys.isNotEmpty) { + logger.w( + "$pluginCodeName: UniversalSearchResult ($videoID): Failed to scrape keys: $nullKeys"); + } + } } class UniversalVideoMetadata { @@ -249,6 +266,23 @@ class UniversalVideoMetadata { void printAllAttributes() { logger.d(convertToMap()); } + + /// Print values that are null, but the plugin didn't expect to be null + // TODO: Set up automatic/user prompted reporting + void printNullKeys(String pluginCodeName, List exceptions) { + Map objectAsMap = convertToMap(); + List nullKeys = []; + // Check whether key is not in exception list and whether value is null + objectAsMap.forEach((key, value) { + if (!exceptions.contains(key) && value == null) { + nullKeys.add(key); + } + }); + if (nullKeys.isNotEmpty) { + logger.w( + "$pluginCodeName: UniversalVideoMetadata ($videoID): Failed to scrape keys: $nullKeys"); + } + } } class UniversalComment { @@ -314,4 +348,19 @@ class UniversalComment { void printAllAttributes() { logger.d(convertToMap()); } + + /// Print values that are null, but the plugin didn't expect to be null + // TODO: Set up automatic/user prompted reporting + void printNullKeys(String pluginCodeName, List exceptions) { + Map objectAsMap = convertToMap(); + List nullKeys = []; + // Check whether key is not in exception list and whether value is null + objectAsMap.forEach((key, value) { + if (!exceptions.contains(key) && value == null) { + nullKeys.add(key); + } + }); + logger.d( + "$pluginCodeName: UniversalComment ($commentID): Failed to scrape keys: $nullKeys"); + } } diff --git a/lib/plugins/official_plugins/pornhub.dart b/lib/plugins/official_plugins/pornhub.dart index 32a88ce4..f4478d5c 100644 --- a/lib/plugins/official_plugins/pornhub.dart +++ b/lib/plugins/official_plugins/pornhub.dart @@ -249,13 +249,10 @@ class PornhubPlugin extends PluginBase implements PluginInterface { // TODO: determine video resolution // pornhub only offers up to 1080p - results.add(UniversalSearchResult( + UniversalSearchResult uniResult = UniversalSearchResult( videoID: iD ?? "-", title: title ?? "-", plugin: this, - author: author ?? "-", - // All authors on pornhub are verified - verifiedAuthor: true, thumbnail: thumbnail, videoPreview: videoPreview != null ? Uri.parse(videoPreview) : null, duration: duration, @@ -263,7 +260,16 @@ class PornhubPlugin extends PluginBase implements PluginInterface { ratingsPositivePercent: ratings, maxQuality: null, virtualReality: virtualReality, - )); + author: author, + // All authors on pornhub are verified + verifiedAuthor: true, + ); + + // print warnings if some data is missing + uniResult.printNullKeys(codeName, + ["thumbnailBinary", "lastWatched", "firstWatched", "maxQuality"]); + + results.add(uniResult); } catch (e) { displayError("Failed to scrape video result: $e"); } @@ -397,7 +403,7 @@ class PornhubPlugin extends PluginBase implements PluginInterface { } } - return UniversalVideoMetadata( + UniversalVideoMetadata metadata = UniversalVideoMetadata( videoID: videoId, m3u8Uris: m3u8Map, title: jscriptMap["video_title"] ?? "-", @@ -417,6 +423,11 @@ class PornhubPlugin extends PluginBase implements PluginInterface { virtualReality: jscriptMap["isVR"] == 1, chapters: null, rawHtml: rawHtml); + + // print warnings if some data is missing + metadata.printNullKeys(codeName, ["tags", "uploadDate", "chapters"]); + + return metadata; } @override diff --git a/lib/plugins/official_plugins/xhamster.dart b/lib/plugins/official_plugins/xhamster.dart index a638b28d..c9578f0e 100644 --- a/lib/plugins/official_plugins/xhamster.dart +++ b/lib/plugins/official_plugins/xhamster.dart @@ -199,7 +199,7 @@ class XHamsterPlugin extends PluginBase implements PluginInterface { } } - results.add(UniversalSearchResult( + UniversalSearchResult uniResult = UniversalSearchResult( videoID: iD ?? "-", title: title ?? "-", plugin: this, @@ -211,7 +211,20 @@ class XHamsterPlugin extends PluginBase implements PluginInterface { ratingsPositivePercent: null, maxQuality: resolution, virtualReality: virtualReality, - )); + author: author, + // Set to false if null or if the author is "Unknown amateur author" + verifiedAuthor: author != null && author != "Unknown amateur author", + ); + + // print warnings if some data is missing + uniResult.printNullKeys(codeName, [ + "thumbnailBinary", + "lastWatched", + "firstWatched", + "ratingsPositivePercent" + ]); + + results.add(uniResult); } } catch (e) { displayError("Failed to scrape video result: $e"); @@ -331,7 +344,8 @@ class XHamsterPlugin extends PluginBase implements PluginInterface { // convert master m3u8 to list of media m3u8 Map m3u8Map = await parseM3U8(Uri.parse(videoM3u8.attributes["href"]!)); - return UniversalVideoMetadata( + + UniversalVideoMetadata metadata = UniversalVideoMetadata( videoID: videoId, m3u8Uris: m3u8Map, title: videoTitle.text, @@ -350,6 +364,11 @@ class XHamsterPlugin extends PluginBase implements PluginInterface { virtualReality: false, chapters: null, rawHtml: rawHtml); + + // print warnings if some data is missing + metadata.printNullKeys(codeName, ["tags", "chapters"]); + + return metadata; } }