diff --git a/.gitignore b/.gitignore index c1f6e7a..7eb4194 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ **-workspace *.pytest** -**/test_cases/* **/.idea/* \ No newline at end of file diff --git a/4chanScrapper.py b/4chanScrapper.py deleted file mode 100644 index aec12fc..0000000 --- a/4chanScrapper.py +++ /dev/null @@ -1,6 +0,0 @@ -from scrapper.scrapper_methods import download_files_from_thread - -download_files_from_thread( - thread_link="https://boards.4chan.org/wg/thread/7955071", - download_path=r"C:\Users\balbi\Desktop\teste-scrapper" -) \ No newline at end of file diff --git a/FchanScrapper.py b/FchanScrapper.py new file mode 100644 index 0000000..2155539 --- /dev/null +++ b/FchanScrapper.py @@ -0,0 +1,136 @@ +import sys + +from scrapper.scrapper_methods import download_files_from_thread + +MINIMUM_ARGUMENTS_THRESHOLD = 1 + + +def get_arguments_from_command_line(): + """ + Gets arguments from command line and returns them as a list. + """ + return sys.argv + + +def is_manual_mode_trigger(argument): + return argument.startswith("-m") + + +def has_minimum_argument_thresold(arguments): + return len(arguments) >= MINIMUM_ARGUMENTS_THRESHOLD + + +def manual_resolution_setting_simple(arguments): + """ + Checks if resolution is set in manual mode - simple. + """ + return "--resolution" in arguments + + +def manual_resolution_setting_full_config(arguments): + """ + Checks if resolution is set in manual mode - full config and not in simple. + """ + return ("--max-res" in arguments or "--min-res" in arguments) and not manual_resolution_setting_simple(arguments) + + +def get_resolution_from_arguments_standart(res_arguments, resolution_trigger_str): + """ + Abstract method for getting resolution from arguments. + """ + resolution_trigger_index = res_arguments.index(resolution_trigger_str) + x_resolution = int(res_arguments[resolution_trigger_index + 1]) + y_resolution = int(res_arguments[resolution_trigger_index + 2]) + return x_resolution, y_resolution + + +def get_resolution_from_arguments_simple(res_arguments): + """ + Gets resolution from arguments. + """ + return get_resolution_from_arguments_standart(res_arguments, "--resolution") + + +def define_resolution_full_config(res_args, resolution_trigger_str): + # Obtain minimum resolution + if resolution_trigger_str in res_args: + x, y = get_resolution_from_arguments_standart(res_args, resolution_trigger_str) + else: + x, y = None, None + return x, y + + +def get_resolution_from_arguments_full_config(res_arguments): + """ + Gets resolution from arguments. + """ + + # Obtain minimum resolution + min_x_res, min_y_res = define_resolution_full_config(res_arguments, "--min-res") + + # Obtain maximum resolution + max_x_res, max_y_res = define_resolution_full_config(res_arguments, "--max-res") + + return min_x_res, min_y_res, max_x_res, max_y_res + + +def manual_mode_download(arguments): + """ + Manual mode is triggered when -m is passed as argument. + + Expected arguments: + # Full command example: + $ python FchanScrapper.py -m thread_link download_path --resolution 1920 1080 + [0] [1] [2] [3] [4] [5] [6] + + $ python FchanScrapper.py -m thread_link download_path --min-res 1920 1080 --max-res 1920 1080 + [0] [1] [2] [3] [4] [5] [6] [7] [8] [9] + + """ + + manual_mode_arguments = arguments[2:] + if manual_resolution_setting_simple(manual_mode_arguments): + x_resolution, y_resolution = get_resolution_from_arguments_simple(manual_mode_arguments) + print("Resolution limits set to: " + str(x_resolution) + "x" + str(y_resolution)) + + return download_files_from_thread(manual_mode_arguments[0], manual_mode_arguments[1], + min_x_res=x_resolution, min_y_res=y_resolution, + max_x_res=None, max_y_res=None + ) + if manual_resolution_setting_full_config(manual_mode_arguments): + [min_x, min_y, max_x, max_y] = get_resolution_from_arguments_full_config(manual_mode_arguments) + print("minimum accepted resolution: " + str(min_x) + "x" + str(min_y)) + print("maximum accepted resolution: " + str(max_x) + "x" + str(max_y)) + + return download_files_from_thread(manual_mode_arguments[0], manual_mode_arguments[1], + min_x_res=min_x, min_y_res=min_y, + max_x_res=max_x, max_y_res=max_y + ) + else: + raise Exception("Resolution not set. Please set resolution with --resolution or " + "type -h for help.") + + +if __name__ == '__main__': + command_line_arguments = get_arguments_from_command_line() + try: + if is_manual_mode_trigger(command_line_arguments[1]): + manual_mode_download(command_line_arguments) + elif has_minimum_argument_thresold(command_line_arguments): + thread_link = command_line_arguments[1] + + if len(command_line_arguments) == 2: + download_path = None + else: + download_path = command_line_arguments[2] + download_files_from_thread(thread_link, download_path) + else: + raise Exception("Not enough arguments.") + sys.exit(0) + except IndexError as e: + print("Please insert arguments correctly while on manual mode.") + + except Exception as e: + print(e) + + sys.exit(1) diff --git a/scrapper/scrapper_methods.py b/scrapper/scrapper_methods.py index 95fe714..57f35d5 100644 --- a/scrapper/scrapper_methods.py +++ b/scrapper/scrapper_methods.py @@ -56,11 +56,17 @@ def get_filename(link): print(f"{number_of_ended_tasks} of {number_of_links_to_download} downloaded!") -async def download_image_task(img_link): # WATCH MY ASS +async def download_image_task(img_link): """ - Function responsible for creating the task of downloading a single image. + Creates the task of downloading a single image. + Args: + img_link: Link of the image, the standart is a internal link without the protocol included. + + Returns: + """ link = HTTP_PROTOCOL_SYMBOL + ":" + img_link + print(f"Downloading {link}") download(link) @@ -68,20 +74,68 @@ def get_thread_data_from_web(thread_link): return requests.get(thread_link).content +def get_resolution_from_div_text(text): + """ + Gets resolution from div text. + Used "File: filename.jpg(491 KB, 1164x705)" as example. + Args: + text: Text from div. + + Returns: Resolution as tuple. + """ + resolution = text[:-1].strip().split(",")[-1] + x_res, y_res = resolution.split("x") + return int(x_res), int(y_res) + + def get_file_links_from_thread_divs(list_of_divs, **filters): - def get_div_link(image_raw_div): + def check_minimum_resolution(img_x, img_y, min_x, min_y): + if min_x: + x_validation = img_x >= min_x + else: + x_validation = True + if min_y: + y_validation = img_y >= min_y + else: + y_validation = True + + return x_validation and y_validation + + def check_maximum_resolution(img_x, img_y, max_x, max_y): + if max_x: + x_validation = img_x <= max_x + else: + x_validation = True + if max_y: + y_validation = img_y <= max_y + else: + y_validation = True + + return x_validation and y_validation + + def check_resolution(img_x, img_y, min_x, min_y, max_x, max_y): + # VALIDATE AS NONE IS ENTERING MIN AND MAX VARS + return check_minimum_resolution(img_x, img_y, min_x, min_y) and check_maximum_resolution(img_x, img_y, max_x, + max_y) + + def get_div_link(image_raw_div, **filters): div_soup = make_soup(image_raw_div, SELECTED_PARSER) - - return div_soup.find('a')['href'] + div_link = div_soup.find('a')['href'] + if filters.values(): + x_resolution, y_resolution = get_resolution_from_div_text(div_soup.text) + if not check_resolution(x_resolution, y_resolution, *filters.values()): + div_link = None + return div_link # filter this after primary tests - file_links = [get_div_link(div_html) for div_html in list_of_divs] + file_links = [get_div_link(div_html, **filters) for div_html in list_of_divs] # - return file_links + return [link for link in file_links if type(link) is str] -def get_all_divs_with_classname(soup, classname): - return list(map(str, soup.findAll('div', class_=classname))) + +def get_all_divs_with_class_name(soup, class_name): + return list(map(str, soup.findAll('div', class_=class_name))) def get_thread_name(soup): @@ -92,7 +146,6 @@ def get_thread_name(soup): return soup.find('span', class_=THREAD_CLASS_STD_NAME).text - async def download_tasks(download_path, links_from_threads_files): # Organizing download tasks thread_download_tasks = [] @@ -106,9 +159,9 @@ async def download_tasks(download_path, links_from_threads_files): await asyncio.gather(*thread_download_tasks) -def get_local_html_file_string(file_path): +def get_html_file_as_string(file_path): """ - Gets html file as string. + Returns html content as string. Args: file_path: File path. @@ -119,10 +172,25 @@ def get_local_html_file_string(file_path): def get_thread_data(thread_link): - if "https://boards.4chan.org" in thread_link: + if "boards.4chan.org" in thread_link: return get_thread_data_from_web(thread_link) else: - return get_local_html_file_string(thread_link) + return get_html_file_as_string(thread_link) + + +def has_empty_arguments(t, d): + return '' in [t, d] + + +def create_local_download_path(new_dir_name): + """ + Creates a local download path. + Args: + new_dir_name: Thread name for new dir. + + Returns: Path as string. + """ + return f"{STD_PATH_DOWNLOAD}/{new_dir_name}" def download_files_from_thread(thread_link, download_path, **filters): @@ -137,10 +205,17 @@ def download_files_from_thread(thread_link, download_path, **filters): """ global number_of_links_to_download + # Analyse arguments + if has_empty_arguments(thread_link, download_path): + raise Exception("Invalid number of arguments. Please, pass at least 2 arguments") + # Gathering files links thread_html = get_thread_data(thread_link) thread_soup = make_soup(thread_html, SELECTED_PARSER) - list_of_thread_files_divs_html = get_all_divs_with_classname(thread_soup, DIV_CLASS_STD_NAME) + thread_name_ = get_thread_name(thread_soup) + if not download_path: + create_local_download_path(thread_name_) + list_of_thread_files_divs_html = get_all_divs_with_class_name(thread_soup, DIV_CLASS_STD_NAME) links_from_threads_files = get_file_links_from_thread_divs(list_of_thread_files_divs_html, **filters) number_of_links_to_download = len(links_from_threads_files) diff --git a/tests/conftest.py b/tests/conftest.py index fb37690..576ba52 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,5 @@ +import shutil + import pytest import sys, os from shutil import rmtree @@ -5,27 +7,49 @@ projects_test_directory = sys.path[0] projects_root = projects_test_directory + "/.." temp_dir_name = "$temp" +temp_dir_path = projects_test_directory + "/test_cases/" + temp_dir_name + + +def remove_recursive(path): + if os.path.isdir(path) and not os.path.islink(path): + shutil.rmtree(path) + elif os.path.exists(path): + os.remove(path) @pytest.fixture() def setup_and_teardown_at_temp_dir(): # setup - os.chdir(projects_test_directory + "/test_cases") - os.mkdir(temp_dir_name) - os.chdir(temp_dir_name) + os.mkdir(temp_dir_path) + os.chdir(temp_dir_path) # test yield # teardown os.chdir("..") - rmtree(f"{temp_dir_name}") + remove_recursive(f"{temp_dir_path}") @pytest.fixture() -def setup_at_battleship_thread_dir(): +def setup_for_thread_name_at_battleship_thread_dir(): # setup os.chdir(projects_test_directory + "/test_cases/battleship") # test yield + + +@pytest.fixture() +def setup_and_teardown_at_battleship_thread_dir(): + # setup + os.chdir(projects_root) + # create temp directory + os.mkdir(temp_dir_path) + + # test + yield + + # teardown + os.chdir("..") + remove_recursive(temp_dir_path) diff --git a/tests/test_cases/avatar/avatar.html b/tests/test_cases/avatar/avatar.html new file mode 100644 index 0000000..103b3fa --- /dev/null +++ b/tests/test_cases/avatar/avatar.html @@ -0,0 +1,1022 @@ + + +/wg/ - Vehicles of Avatar. Regardless of your thoughts on - Wallpapers/General - 4chan
[a / b / c / d / e / f / g / gif / h / hr / k / m / o / p / r / s / t / u / v / vg / vm / vmg / vr / vrpg / vst / w / wg] [i / ic] [r9k / s4s / vip / qa] [cm / hm / lgbt / y] [3 / aco / adv / an / bant / biz / cgl / ck / co / diy / fa / fit / gd / hc / his / int / jp / lit / mlp / mu / n / news / out / po / pol / pw / qst / sci / soc / sp / tg / toy / trv / tv / vp / vt / wsg / wsr / x / xs] [Edit][Settings] [Search] [Mobile] [Home]
Board
▼ Settings Mobile Home
4chan
/wg/ - Wallpapers/General
Return Catalog Bottom Refresh
Name
Options
Comment
Verification
4chan Pass users can bypass this verification. [Learn More] [Login]
File
  • Please read the Rules and FAQ before posting.
  • Maximum file size allowed is 6144 KB.
  • Images smaller than 480x600 pixels are not allowed.
  • There are 4 posters in this thread.
08/21/20New boards added: /vrpg/, /vmg/, /vst/ and /vm/
05/04/17New trial board added: /bant/ - International/Random
10/04/16New board for 4chan Pass users: /vip/ - Very Important Posts
[Hide] [Show All]
[Advertise on 4chan]
[Return] [Catalog] [Bottom]
12 / 5 / 4 / 7
[Update] []
Anonymous
12/18/22(Sun)21:00:07 No.7952860
File: wp5459610.png (3.64 MB, 1920x1200)
3.64 MB
3.64 MB PNG
Anonymous 12/18/22(Sun)23:00:07 No.7952860▶
Vehicles of Avatar. Regardless of your thoughts on the films, the hardware is pure James Cameron military goodness.
>>
Anonymous
12/18/22(Sun)21:01:15 No.7952861
Anonymous 12/18/22(Sun)23:01:15 No.7952861▶
File: wp7793003.jpg (113 KB, 1600x900)
113 KB
113 KB JPG
>>
Anonymous
12/18/22(Sun)21:01:46 No.7952862
Anonymous 12/18/22(Sun)23:01:46 No.7952862▶
File: wp7803554.jpg (253 KB, 1920x1080)
253 KB
253 KB JPG
>>
Anonymous
12/18/22(Sun)21:02:32 No.7952863
Anonymous 12/18/22(Sun)23:02:32 No.7952863▶
File: wp4921780.png (1.43 MB, 1920x1080)
1.43 MB
1.43 MB PNG
>>
Anonymous
12/18/22(Sun)21:03:43 No.7952864
Anonymous 12/18/22(Sun)23:03:43 No.7952864▶
File: wp4921762.jpg (79 KB, 1366x768)
79 KB
79 KB JPG
>>
Anonymous
12/18/22(Sun)21:04:18 No.7952865
Anonymous 12/18/22(Sun)23:04:18 No.7952865▶
File: wp7156075.jpg (255 KB, 1920x1200)
255 KB
255 KB JPG
And that's all I've got.
>>
Anonymous
12/22/22(Thu)06:09:46 No.7953864
Anonymous 12/22/22(Thu)08:09:46 No.7953864▶
Anyone else got any more?
>>
Anonymous
12/25/22(Sun)04:59:22 No.7954466
Anonymous 12/25/22(Sun)06:59:22 No.7954466▶
>>7953864
Look for artstation stuff for the 2nd movie. The crab mech looks sick
>>
Anonymous
12/25/22(Sun)05:11:17 No.7954468
Anonymous 12/25/22(Sun)07:11:17 No.7954468▶
>>7952860 (OP)
I wonder if having these types of props would actually be feasable in real life.
>>
Anonymous
12/25/22(Sun)13:27:58 No.7954499
Anonymous 12/25/22(Sun)15:27:58 No.7954499▶
>>7952860 (OP)
how many more helicopter pilots must be impaled with arrows before humans realize they can just use drones?
>>
Anonymous
12/26/22(Mon)01:33:13 No.7954652
Anonymous 12/26/22(Mon)03:33:13 No.7954652▶
>>7954468
Most Yes actually. The only issue is power capacity for the larger technology
>>
Anonymous
12/26/22(Mon)05:38:11 No.7954669
Anonymous 12/26/22(Mon)07:38:11 No.7954669▶
>>7954499
They can't use drones in most places around Pandora (especially where the Na'vi live) because of the Flux Vortex. The only way to pilot is by sight.
>>
Anonymous
12/26/22(Mon)05:39:12 No.7954670
Anonymous 12/26/22(Mon)07:39:12 No.7954670▶
>>7954466
>The crab mech looks sick
It really does, definitely gonna pick up at least the smaller figure from McFarlane.
[Post a Reply]
[Return] [Catalog] [Top]
12 / 5 / 4 / 7
[Update] []
Post a Reply
Return Catalog Top Refresh
[Advertise on 4chan]
Delete Post: [File Only] Style:
[a / b / c / d / e / f / g / gif / h / hr / k / m / o / p / r / s / t / u / v / vg / vm / vmg / vr / vrpg / vst / w / wg] [i / ic] [r9k / s4s / vip / qa] [cm / hm / lgbt / y] [3 / aco / adv / an / bant / biz / cgl / ck / co / diy / fa / fit / gd / hc / his / int / jp / lit / mlp / mu / n / news / out / po / pol / pw / qst / sci / soc / sp / tg / toy / trv / tv / vp / vt / wsg / wsr / x / xs] [Edit][Settings] [Search] [Mobile] [Home]
[Disable Mobile View / Use Desktop Site]

[Enable Mobile View / Use Mobile Site]

All trademarks and copyrights on this page are owned by their respective parties. Images uploaded are the responsibility of the Poster. Comments are owned by the Poster.
\ No newline at end of file diff --git a/tests/test_cases/battleship/battleship.html b/tests/test_cases/battleship/battleship.html new file mode 100644 index 0000000..353f9b6 --- /dev/null +++ b/tests/test_cases/battleship/battleship.html @@ -0,0 +1,10 @@ +/wg/ - Battleship Bread (Real Ones) - Wallpapers/General - 4chan
[a / b / c / d / e / f / g / gif / h / hr / k / m / o / p / r / s / t / u / v / vg / vm / vmg / vr / vrpg / vst / w / wg] [i / ic] [r9k / s4s / vip / qa] [cm / hm / lgbt / y] [3 / aco / adv / an / bant / biz / cgl / ck / co / diy / fa / fit / gd / hc / his / int / jp / lit / mlp / mu / n / news / out / po / pol / pw / qst / sci / soc / sp / tg / toy / trv / tv / vp / vt / wsg / wsr / x / xs] [Edit][Settings] [Search] [Mobile] [Home]
Board
▼ Settings Mobile Home
4chan
/wg/ - Wallpapers/General
Return Catalog Bottom Refresh
Name
Options
Comment
Verification
4chan Pass users can bypass this verification. [Learn More] [Login]
File
  • Please read the Rules and FAQ before posting.
  • Maximum file size allowed is 6144 KB.
  • Images smaller than 480x600 pixels are not allowed.
  • There are 8 posters in this thread.
08/21/20New boards added: /vrpg/, /vmg/, /vst/ and /vm/
05/04/17New trial board added: /bant/ - International/Random
10/04/16New board for 4chan Pass users: /vip/ - Very Important Posts
[Hide] [Show All]
[Advertise on 4chan]
[Return] [Catalog] [Bottom]
25 / 15 / 8 / 5
[Update] []
Anonymous
Battleship Bread (Real Ones) 12/06/22(Tue)18:30:41 No.7950511
File: NEWJERSEYbb62.jpg (491 KB, 1164x705)
491 KB
491 KB JPG
Battleship Bread (Real Ones) Anonymous 12/06/22(Tue)20:30:41 No.7950511▶
From the earliest of boats with guns to the dreadnaught era, WW2, or modern. Let's see em boys. December 7 is near.
>>
Anonymous
12/06/22(Tue)18:31:29 No.7950512
Anonymous 12/06/22(Tue)20:31:29 No.7950512▶
File: IOWA.jpg (456 KB, 1920x1080)
456 KB
456 KB JPG
>>7950511 (OP)
>>
Anonymous
12/06/22(Tue)18:32:05 No.7950513
Anonymous 12/06/22(Tue)20:32:05 No.7950513▶
File: 1592084476566.jpg (1.17 MB, 2816x1872)
1.17 MB
1.17 MB JPG
>>7950512
>>
Anonymous
12/06/22(Tue)18:37:53 No.7950515
Anonymous 12/06/22(Tue)20:37:53 No.7950515▶
File: 1591673878146.jpg (965 KB, 2560x1440)
965 KB
965 KB JPG
>>
MTGfan
12/07/22(Wed)12:42:45 No.7950632
MTGfan 12/07/22(Wed)14:42:45 No.7950632▶
HMS Duke of York fires her 14 inch main battery at the German battleship Scharnhorst during the battle of the North Cape in 1943
>>
MTGfan
12/07/22(Wed)12:43:52 No.7950634
MTGfan 12/07/22(Wed)14:43:52 No.7950634▶
File: Scharnhorst.jpg (2.11 MB, 3840x2160)
2.11 MB
2.11 MB JPG
>>
MTGfan
12/07/22(Wed)12:44:47 No.7950635
MTGfan 12/07/22(Wed)14:44:47 No.7950635▶
File: KM Bismarck.jpg (1.46 MB, 2560x1440)
1.46 MB
1.46 MB JPG
>>
MTGfan
12/07/22(Wed)12:47:21 No.7950636
MTGfan 12/07/22(Wed)14:47:21 No.7950636▶
File: USS Arizona.jpg (3.16 MB, 5728x3222)
3.16 MB
3.16 MB JPG
>>
MTGfan
12/07/22(Wed)12:48:10 No.7950637
MTGfan 12/07/22(Wed)14:48:10 No.7950637▶
File: USS Missouri (1).jpg (1.72 MB, 2880x1620)
1.72 MB
1.72 MB JPG
>>
MTGfan
12/07/22(Wed)12:49:24 No.7950639
MTGfan 12/07/22(Wed)14:49:24 No.7950639▶
File: USS Missouri (2).jpg (1.93 MB, 3200x1800)
1.93 MB
1.93 MB JPG
>>
MTGfan
12/07/22(Wed)12:50:55 No.7950640
MTGfan 12/07/22(Wed)14:50:55 No.7950640▶
File: USS Wisconsin.jpg (2.68 MB, 2894x1627)
2.68 MB
2.68 MB JPG
>>
MTGfan
12/07/22(Wed)12:53:29 No.7950641
MTGfan 12/07/22(Wed)14:53:29 No.7950641▶
File: 1601001514683.jpg (1.59 MB, 2880x1620)
1.59 MB
1.59 MB JPG
unknown ship 1
>>
MTGfan
12/07/22(Wed)12:54:21 No.7950642
MTGfan 12/07/22(Wed)14:54:21 No.7950642▶
File: 1601519490841.jpg (825 KB, 3200x1800)
825 KB
825 KB JPG
unknown ship 2
>>
MTGfan
12/07/22(Wed)12:55:12 No.7950643
MTGfan 12/07/22(Wed)14:55:12 No.7950643▶
File: 1601826686851.jpg (735 KB, 2560x1440)
735 KB
735 KB JPG
unknown ship 3
>>
MTGfan
12/07/22(Wed)12:56:05 No.7950644
MTGfan 12/07/22(Wed)14:56:05 No.7950644▶
File: 1602955126045.jpg (1.05 MB, 2560x1440)
1.05 MB
1.05 MB JPG
unknown ship 4
>>
Anonymous
12/07/22(Wed)14:44:13 No.7950668
Anonymous 12/07/22(Wed)16:44:13 No.7950668▶
>>7950643
appears to say D-185

if true -

"PNS Tippu Sultan, a Tariq-class destroyer, served in the Pakistan Navy after it was acquired in 1994. Her design was based on the British Type 21 frigate, and previously served in the Royal Navy as HMS Avenger as a general purpose frigate."
>>
Anonymous
12/07/22(Wed)14:45:21 No.7950669
Anonymous 12/07/22(Wed)16:45:21 No.7950669▶
>>7950643
>>7950668

if D186 (sorry eyesight is booty)

"D186 Mölders was one of three Lütjens-class guided-missile destroyers, a modified version of the American Charles F. Adams class, built for the Bundesmarine during the 1960s."

I think it looks more like this.
>>
Anonymous
12/09/22(Fri)15:48:10 No.7950991
Anonymous 12/09/22(Fri)17:48:10 No.7950991▶
File: MH 370.jpg (333 KB, 1200x800)
333 KB
333 KB JPG
Japanese Battleship "Yamato" over 70 years after her sinking.
>>
Anonymous
12/17/22(Sat)04:38:12 No.7952367
Anonymous 12/17/22(Sat)06:38:12 No.7952367▶
>>7950641
This is New Jersey. New Jersey was the only Iowa recommissioned for service during the Vietnam War, and this left her with a uniquely shaped superstructure top to support her electronic countermeasures (ECM) suite. On New Jersey this is two rectangular projections to the beams, all other Iowas received a wrap-around structure.
>>
MTGfan
12/17/22(Sat)20:56:01 No.7952543
MTGfan 12/17/22(Sat)22:56:01 No.7952543▶
>>7950669
>>7952367
Thank you for the info. I appreciate it.
>>
Anonymous
12/17/22(Sat)21:45:08 No.7952550
Anonymous 12/17/22(Sat)23:45:08 No.7952550▶
>>7950642

Yamato
>>
MTGfan
12/18/22(Sun)01:31:44 No.7952567
MTGfan 12/18/22(Sun)03:31:44 No.7952567▶
>>7952550
Yamato or Musashi?
>>
Anonymous
12/23/22(Fri)09:38:25 No.7954121
Anonymous 12/23/22(Fri)11:38:25 No.7954121▶
>>7950644
Looks like New York class BB
>>
Anonymous
12/25/22(Sun)13:14:42 No.7954496
Anonymous 12/25/22(Sun)15:14:42 No.7954496▶
>>7952567
Yamato. During sea trials near Bungo Strait in October 20th 1941.
>>
MTGfan
12/27/22(Tue)11:36:50 No.7954837
MTGfan 12/27/22(Tue)13:36:50 No.7954837▶
>>7954496
Thank you
>>
Anonymous
12/29/22(Thu)09:58:48 No.7955354
Anonymous 12/29/22(Thu)11:58:48 No.7955354▶
>>7950991
kek
[Post a Reply]
[Return] [Catalog] [Top]
25 / 15 / 8 / 5
[Update] []
Post a Reply
Return Catalog Top Refresh
[Advertise on 4chan]
Delete Post: [File Only] Style:
[a / b / c / d / e / f / g / gif / h / hr / k / m / o / p / r / s / t / u / v / vg / vm / vmg / vr / vrpg / vst / w / wg] [i / ic] [r9k / s4s / vip / qa] [cm / hm / lgbt / y] [3 / aco / adv / an / bant / biz / cgl / ck / co / diy / fa / fit / gd / hc / his / int / jp / lit / mlp / mu / n / news / out / po / pol / pw / qst / sci / soc / sp / tg / toy / trv / tv / vp / vt / wsg / wsr / x / xs] [Edit][Settings] [Search] [Mobile] [Home]
[Disable Mobile View / Use Desktop Site]

[Enable Mobile View / Use Mobile Site]

All trademarks and copyrights on this page are owned by their respective parties. Images uploaded are the responsibility of the Poster. Comments are owned by the Poster.
\ No newline at end of file diff --git a/tests/test_scrapper.py b/tests/test_scrapper.py index 2aeb5d5..62f3ce0 100644 --- a/tests/test_scrapper.py +++ b/tests/test_scrapper.py @@ -1,23 +1,29 @@ -from conftest import temp_dir_name, projects_test_directory +import os + +from conftest import temp_dir_path, temp_dir_name, projects_test_directory import sys # Path adjustment to add scrapper methods sys.path.append(projects_test_directory) from scrapper.scrapper_methods import * +from FchanScrapper import manual_mode_download + + # class Test_Scrapper: BATTLESHIP_THREAD_PATH = projects_test_directory + "/test_cases/battleship/battleship.html" BATTLESHIP_THREAD_NAME = "Battleship Bread (Real Ones)" + WINDOWS_ERROR_CODE = 1 - def test_download_thread(self, setup_and_teardown_at_temp_dir): + def test_download_thread_directly(self, setup_and_teardown_at_temp_dir): """ Tests main functionality by creating temporary directory, downloading files found in locally installed thread. Deletes temporary directory after conclusion. Args: - create_and_cleanup_temp_dir: Pytest fixture with setup and teardown functions. + setup_and_teardown_at_temp_dir: Pytest fixture with setup and teardown functions. """ selected_thread_link = "https://boards.4chan.org/wg/thread/7950511" @@ -27,7 +33,57 @@ def test_download_thread(self, setup_and_teardown_at_temp_dir): assert os.getcwd().endswith(temp_dir_name) assert len(os.listdir()) == number_of_links - def test_get_battleship_thread_name(self, setup_at_battleship_thread_dir): + def test_download_thread_from_manual_mode(self, setup_and_teardown_at_temp_dir): + """ + Tests program's response when correct arguments are passed. Expects OS success code 0. + Args: + setup_and_teardown_at_temp_dir: Pytest fixture. + + Returns: + + """ + selected_thread_link = "https://boards.4chan.org/wg/thread/7950511" + selected_download_path_link = "." # current directory is $temp here + + arguments = [ + "FchanScrapper.py", + "-m", + selected_thread_link, + selected_download_path_link, + "--resolution", + "1920", + "1080" + ] + number_of_links = manual_mode_download(arguments) + assert os.getcwd().endswith(temp_dir_name) + assert len(os.listdir()) == number_of_links + + def test_download_thread_from_manual_mode_only_min_res(self, setup_and_teardown_at_temp_dir): + """ + Tests program's response when correct arguments are passed. Expects OS success code 0. + Args: + setup_and_teardown_at_temp_dir: Pytest fixture. + + Returns: + + """ + selected_thread_link = "https://boards.4chan.org/wg/thread/7950511" + selected_download_path_link = "." # current directory is $temp here + + arguments = [ + "FchanScrapper.py", + "-m", + selected_thread_link, + selected_download_path_link, + "--min-res", + "3200", + "1800" + ] + number_of_links = manual_mode_download(arguments) + assert os.getcwd().endswith(temp_dir_name) + assert len(os.listdir()) == number_of_links + + def test_get_battleship_thread_name(self, setup_for_thread_name_at_battleship_thread_dir): """ Test local files at ./test_cases to check if name validations are working. In threads, 2 different patterns are found, nameless threads with big commentaries and threads with names. @@ -38,3 +94,157 @@ def test_get_battleship_thread_name(self, setup_at_battleship_thread_dir): name = get_thread_name(soup) assert self.BATTLESHIP_THREAD_NAME.startswith(name) + + def test_download_thread_from_command_line_with_no_arguments(self, setup_and_teardown_at_battleship_thread_dir): + """ + Tests program's response when no arguments are passed. Expects OS error code 1. + Meaning "ERROR_SUCCESS" (https://learn.microsoft.com/en-us/windows/win32/debug/system-error-codes--0-499-). + """ + execution_code = os.system("python FchanScrapper.py") + assert execution_code == self.WINDOWS_ERROR_CODE + + def test_download_thread_from_command_line_auto_mode(self, setup_and_teardown_at_battleship_thread_dir): + """ + Tests program's response when correct arguments are passed. Expects OS success code 0. + Args: + setup_and_teardown_at_battleship_thread_dir: Pytest fixture. + """ + # test_vars + battleship_thread_file_full_path = projects_test_directory + "/test_cases/battleship/battleship.html" + download_path = temp_dir_path + number_of_images_in_thread = 16 + + # test + execution_code = os.system(f"python FchanScrapper.py {battleship_thread_file_full_path} {download_path}") + os.chdir(temp_dir_path) + + # assertions + assert execution_code == 0 + assert len(os.listdir()) == number_of_images_in_thread + + def test_download_thread_from_command_line_manual_mode_simple(self, setup_and_teardown_at_battleship_thread_dir): + # test_vars + battleship_thread_file_full_path = projects_test_directory + "/test_cases/battleship/battleship.html" + download_path = temp_dir_path + number_of_images_above_minimum_resolution_in_thread = 14 + + # test + command = f"python FchanScrapper.py -m {battleship_thread_file_full_path} {download_path} " \ + "--resolution 1920 1080" + execution_code = os.system(command) + os.chdir(temp_dir_path) + + # assertions + assert execution_code == 0 + assert len(os.listdir()) == number_of_images_above_minimum_resolution_in_thread + + def test_error_missing_arg_download_thread_cmd_line_manual_mode(self, setup_and_teardown_at_battleship_thread_dir): + # test_vars + battleship_thread_file_full_path = projects_test_directory + "/test_cases/battleship/battleship.html" + download_path = temp_dir_path + + # test + command = f"python FchanScrapper.py -m {battleship_thread_file_full_path} {download_path} " \ + "--resolution 1080" + execution_code = os.system(command) + os.chdir(temp_dir_path) + + # assertions + assert execution_code == self.WINDOWS_ERROR_CODE + + def test_download_thread_from_cmd_line_manual_mode_full_config(self, setup_and_teardown_at_battleship_thread_dir): + """ + Tests program's response when all arguments are passed (correctly). Expects OS success code 0. + Args: + setup_and_teardown_at_battleship_thread_dir: Pytest fixture. + """ + # test_vars + battleship_thread_file_full_path = projects_test_directory + "/test_cases/battleship/battleship.html" + download_path = temp_dir_path + number_of_images_within_chosen_resolution_in_thread = 3 # Modify accordingly + min_x_resolution = 1000 + min_y_resolution = 700 + max_x_resolution = 1921 + max_y_resolution = 1081 + + # test + command = f"python FchanScrapper.py -m {battleship_thread_file_full_path} {download_path} " \ + f"--min-res {min_x_resolution} {min_y_resolution} " \ + f"--max-res {max_x_resolution} {max_y_resolution}" + execution_code = os.system(command) + os.chdir(temp_dir_path) + + # assertions + assert execution_code == 0 + assert len(os.listdir()) == number_of_images_within_chosen_resolution_in_thread + + def test_error_download_from_cmd_line_manual_full_config(self, setup_and_teardown_at_battleship_thread_dir): + """ + Tests program's response when no arguments are passed. Expects OS success code 1. + Args: + setup_and_teardown_at_battleship_thread_dir: Pytest fixture. + """ + # test_vars + battleship_thread_file_full_path = projects_test_directory + "/test_cases/battleship/battleship.html" + download_path = temp_dir_path + max_x_resolution = 1921 + max_y_resolution = 1081 + + # test + command = f"python FchanScrapper.py -m {battleship_thread_file_full_path} {download_path} " \ + f"--min-res" \ + f"--max-res {max_x_resolution} {max_y_resolution}" + execution_code = os.system(command) + os.chdir(temp_dir_path) + + # assertions + assert execution_code == self.WINDOWS_ERROR_CODE + + def test_download_thread_from_cmd_line_manual_mode_only_max(self, setup_and_teardown_at_battleship_thread_dir): + """ + Tests program's response when half the arguments are passed (correctly). Expects OS success code 0. + Args: + setup_and_teardown_at_battleship_thread_dir: Pytest fixture. + """ + # test_vars + battleship_thread_file_full_path = projects_test_directory + "/test_cases/battleship/battleship.html" + download_path = temp_dir_path + number_of_images_within_chosen_resolution_in_thread = 2 # Modify accordingly + max_x_resolution = 1200 + max_y_resolution = 800 + + # test + command = f"python FchanScrapper.py -m {battleship_thread_file_full_path} {download_path} " \ + f"--max-res {max_x_resolution} {max_y_resolution}" + execution_code = os.system(command) + os.chdir(temp_dir_path) + + # assertions + assert execution_code == 0 + assert len(os.listdir()) == number_of_images_within_chosen_resolution_in_thread + + def test_download_thread_from_cmd_line_manual_mode_only_min(self, setup_and_teardown_at_battleship_thread_dir): + """ + Tests program's response when all arguments are passed (correctly). Expects OS success code 0. + Args: + setup_and_teardown_at_battleship_thread_dir: Pytest fixture. + """ + # test_vars + battleship_thread_file_full_path = projects_test_directory + "/test_cases/battleship/battleship.html" + download_path = temp_dir_path + number_of_images_within_chosen_resolution_in_thread = 4 # Modify accordingly + min_x_resolution = 3200 + min_y_resolution = 1800 + + # test + command = f"python FchanScrapper.py -m {battleship_thread_file_full_path} {download_path} " \ + f"--min-res {min_x_resolution} {min_y_resolution}" + execution_code = os.system(command) + os.chdir(temp_dir_path) + + # assertions + assert execution_code == 0 + assert len(os.listdir()) == number_of_images_within_chosen_resolution_in_thread + + def test_automatic_directory_creation(self): # TODO: Implement + pass \ No newline at end of file