From 539b75cb095d979b08622f730e614fa0a17a5a90 Mon Sep 17 00:00:00 2001 From: EndMove Date: Mon, 5 Sep 2022 16:14:05 +0200 Subject: [PATCH] The thread system not being sufficient remodulation of it --- controller/HomeController.py | 38 +++++-- model/WebPicDownloader.py | 201 +++++++++++++++++------------------ 2 files changed, 124 insertions(+), 115 deletions(-) diff --git a/controller/HomeController.py b/controller/HomeController.py index c9fac8b..f7b0456 100644 --- a/controller/HomeController.py +++ b/controller/HomeController.py @@ -1,4 +1,6 @@ +import time from controller.MainController import MainController +from model.WebPicDownloader import WebPicDownloader from util.AsyncTask import AsyncTask @@ -16,7 +18,7 @@ class HomeController: # Variables __main_controller = None __view = None - __webpic = None + __webpic: WebPicDownloader = None __download_task = None # Constructor @@ -43,7 +45,9 @@ class HomeController: * :view: -> The view that this controller manage. """ self.__view = view - self.__webpic.set_messenger_callback(view.add_log) + self.__webpic.set_messenger_callback(self.on_webpic_messenger) + self.__webpic.set_success_callback(self.on_webpic_success) + self.__webpic.set_failure_callback(self.on_webpic_failure) # END View method # START View events @@ -65,29 +69,41 @@ class HomeController: * :name: -> The name of the folder in which put pictures. """ if url.strip() and name.strip(): - self.__download_task = AsyncTask( - rcallback=self.__async_task_start, - rargs=(url, name), - qcallback=self.__async_task_stop - ) - self.__download_task.start() + self.__webpic.start_downloading(url, name) else: self.__view.show_error_message("Opss, the url or folder name are not valid!") # END View events + # START Webpic events + def on_webpic_messenger(self, message: str) -> None: + """ + """ + self.__view.add_log(message) + + def on_webpic_success(self) -> None: + """ + """ + self.__view.show_success_message("The download has been successfully completed.") + + def on_webpic_failure(self) -> None: + """ + """ + self.__view.show_error_message("A critical error preventing the download occurred, check the logs.") + # END Webpic events + # START Controller methods def on_quit(self) -> bool: """ [event function for controller] => Call this event when a request to exit is thrown. """ - if self.__download_task and self.__download_task.is_alive(): + if self.__webpic.is_alive(): if self.__main_controller.show_question_dialog( "Are you sure?", "Do you really want to quit while the download is running?\nThis will stop the download." ): - self.__download_task.stop() - self.__download_task.join() + self.__webpic.stop_downloading() + time.sleep(4) return False else: return True diff --git a/model/WebPicDownloader.py b/model/WebPicDownloader.py index 62e91c2..77cc698 100644 --- a/model/WebPicDownloader.py +++ b/model/WebPicDownloader.py @@ -1,11 +1,11 @@ import os -import threading +from threading import Semaphore, Thread from urllib import request from urllib.error import HTTPError, URLError from bs4 import BeautifulSoup, Tag, ResultSet -class WebPicDownloader(threading.Thread): +class WebPicDownloader(Thread): """ WebPicDownloader @@ -21,18 +21,16 @@ class WebPicDownloader(threading.Thread): @version 1.2.0 """ # Variables - path: str = None # Main working folder directory - headers: dict = None # Header parameters - dl_setting: dict = None # Parameter dictionary - - callbacks: dict = None # Callback dictionary - - use_thread: bool = None # Indicates if the script must launch the download in a separate thread - async_run: bool = None # Idicate if the task can still run (for use in a thread) + __callbacks: dict = None # Callback dictionary + __settings: dict = None # + __dl_infos: dict = None # + __sem: Semaphore = None # + + _exit: bool = None # When set to True quit the thread # Constructor - def __init__(self, path: str = None, headers: dict = None, asynchrone: bool = False, - messenger = None, success = None, failure = None) -> None: + def __init__(self, path: str = None, headers: dict = None, messenger = None, + success = None, failure = None) -> None: """ Constructor => It is important to initialize the WebPicDownloader object properly. The callback @@ -46,24 +44,30 @@ class WebPicDownloader(threading.Thread): * :success: -> Callback function success (see setter). * :failure: -> Callback function failure (see setter). """ - super().__init__() - self.path = path if path else os.getcwd() - self.headers = headers if headers else { - 'User-Agent': "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36" - } - self.dl_setting = { - 'url': None, - 'name': None - } - - self.callbacks = { + super().__init__(daemon=True) + self.__callbacks = { 'messenger': messenger if messenger else lambda msg: print(msg), 'success': success if success else lambda: print("Success!"), 'failure': failure if failure else lambda: print("failure!") } - - self.use_thread = asynchrone - self.async_run = True + self.__settings = { + 'root_path': path if path else os.getcwd(), + 'headers': headers if headers else { + 'User-Agent': "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36" + } + } + self.__dl_infos = { + 'website_url': 'url', + 'download_name': 'name', + 'download_path': 'full_path', + 'tot_image_count': 0, + 'dl_image_count': 0, + 'running': False + } + self.__sem = Semaphore(0) + self.__exit = False + + self.start() # start deamon # Internal functions def __get_html(self, url: str) -> str: @@ -74,7 +78,7 @@ class WebPicDownloader(threading.Thread): * :url: -> The url of the site for which we want to get the content of the HTML page. * RETURN -> Web page content. """ - req = request.Request(url, headers=self.headers) + req = request.Request(url, headers=self.__settings.get('headers')) response = request.urlopen(req) return response.read().decode('utf-8') @@ -132,7 +136,7 @@ class WebPicDownloader(threading.Thread): * :url: -> Image url on the web. * :filename: -> Full path with name of the future image. """ - req = request.Request(url, headers=self.headers) + req = request.Request(url, headers=self.__settings.get('headers')) raw_img = request.urlopen(req).read() with open(filename, 'wb') as img: img.write(raw_img) @@ -155,88 +159,16 @@ class WebPicDownloader(threading.Thread): Internal Function #do-not-use# => Use the messenger callback to send a message. """ - self.callbacks.get('messenger')(message) + self.__callbacks.get('messenger')(message) # Public functions - def start_downloading(self, url: str, folder_name: str) -> None: - """ - Start downloading all pictures of a website. - Depending on the configuration the download will either be started in the - main process or in a separate thread. - - * :url: -> The url of the website to annalyse. - * :folder_name: -> The name of the folder in which to upload the photos. - """ - # Updating settings - self.dl_setting['url'] = url - self.dl_setting['name'] = folder_name - - # Start processing - if self.use_thread: - self.async_run = True - self.start() - else: - self.run() - - def stop_downloading(self) -> None: - """ - Stops the download after the current item is processed and exit the downloading thread. - - this function is available only if asynchronous option is set to True in constructor. - """ - if self.use_thread: - self.async_run = False - else: - raise Exception("Can not stop a no multi thread execution") - - def run(self) -> None: - """ - - This function must not be used! It is an internal function called automatically - when the download task starts. - - """ - try: - # init variables - download_count = 0 # count to 0 - download_folder = f"{self.path}/{self.dl_setting.get('name')}/" # format path - website_html = self.__get_html(self.dl_setting.get('url')) # website html - images = self.__find_all_img(website_html) # find all img balises ing html - - self.__initialize_folder(download_folder) # initialize download folder - self.__msg(f"WebPicDownloader found {len(images)} images on the website.") - - # process pictures - for i, img in enumerate(images): - if self.use_thread and not self.async_run: - return - try: - self.__msg(f"Start downloading image {i}.") - img_link = self.__find_img_link(img) - self.__download_img(img_link, f"{download_folder}image-{i}.{self.__find_image_type(img_link)}") - self.__msg(f"Download of image {i}, done!") - download_count += 1 - except Exception as err: - self.__msg(f"ERROR: Unable to process image {i} -> err[{err}].") - self.__msg(f"WebPicDownloader has processed {download_count} images out of {len(images)}.") - self.callbacks.get('success')() # success, launch callback and return - return - - except HTTPError as err: - self.__msg(f"ERROR: An http error occured -> err[{err}].") - except (ValueError, URLError) as err: - self.__msg(f"ERROT: An error occured with the url -> err[{err}].") - except Exception as err: - self.__msg(f"ERROR: An unknown error occured -> err[{err}]") - self.callbacks.get('failure')() # failure, launch callback and exit - def set_success_callback(self, callback) -> None: """ Setter to define the callback function when the download succeeded. * :callback: -> the callback function to call when the download is a success. """ - self.callbacks['success'] = callback + self.__callbacks['success'] = callback def set_failure_callback(self, callback) -> None: """ @@ -244,7 +176,7 @@ class WebPicDownloader(threading.Thread): * :callback: -> the callback function to call when the download is a failure. """ - self.callbacks['failure'] = callback + self.__callbacks['failure'] = callback def set_messenger_callback(self, callback) -> None: """ @@ -252,15 +184,76 @@ class WebPicDownloader(threading.Thread): * :callback: -> the callback function to call when a message event is emited. """ - self.callbacks['messenger'] = callback + self.__callbacks['messenger'] = callback + + def start_downloading(self, url: str, name: str) -> None: + """ + TODO desc + """ + if self.__dl_infos.get('running'): + print("bussy") + else: + self.__dl_infos['website_url'] = url + self.__dl_infos['download_name'] = name + self.__sem.release() + + def stop_downloading(self, block=False) -> None: + """ + TODO DESC + """ + self.__exit = True + self.__sem.release() + if block: + self.join() + + # Thread corp function + def run(self) -> None: + while True: + self.__sem.acquire() + + if self.__exit: + return + + self.__dl_infos['running'] = True # reserv run + try: + html = self.__get_html(self.__dl_infos.get('website_url')) # website html + images = self.__find_all_img(html) # find all img balises ing html + + self.__dl_infos['tot_image_count'] = len(images) # count total image + self.__dl_infos['dl_image_count'] = 0 # set download count to 0 + self.__dl_infos['download_path'] = f"{self.__settings.get('root_path')}/{self.__dl_infos.get('download_name')}/" # format path + + self.__initialize_folder(self.__dl_infos.get('download_path')) # Init download folder + self.__msg(f"WebPicDownloader found {self.__dl_infos.get('tot_image_count')} images on the website.") + + # process pictures + for i, img in enumerate(images): + try: + self.__msg(f"Start downloading image {i}.") + img_link = self.__find_img_link(img) # find image link + self.__download_img(img_link, f"{self.__dl_infos.get('download_path')}image-{i}.{self.__find_image_type(img_link)}") # download the image + self.__msg(f"Download of image {i}, done!") + self.__dl_infos['dl_image_count'] += 1 # increment download counter + except Exception as err: + self.__msg(f"ERROR: Unable to process image {i} -> err[{err}].") + self.__msg(f"WebPicDownloader has processed {self.__dl_infos.get('dl_image_count')} images out of {self.__dl_infos.get('tot_image_count')}.") + self.__callbacks.get('success')() # success, launch callback + except Exception as err: + self.__msg(f"ERROR: An error occured -> err[{err}]") + self.__callbacks.get('failure')() # error, launch callback + self.__dl_infos['running'] = False # free run if __name__ == "__main__": # Internal entry point for testing and consol use. wpd = WebPicDownloader() + def lol(msg): + pass + wpd.set_messenger_callback(lol) while True: url = input("Website URL ? ") name = input("Folder name ? ") wpd.start_downloading(url, name) if "n" == input("Do you want to continue [Y/n] ? ").lower(): + wpd.stop_downloading() break print("Good bye !") \ No newline at end of file