The thread system not being sufficient remodulation of it

This commit is contained in:
Jérémi N ‘EndMove’ 2022-09-05 16:14:05 +02:00
parent 63e67772a3
commit 539b75cb09
Signed by: EndMove
GPG Key ID: 65C4A02E1F5371A4
2 changed files with 124 additions and 115 deletions

View File

@ -1,4 +1,6 @@
import time
from controller.MainController import MainController from controller.MainController import MainController
from model.WebPicDownloader import WebPicDownloader
from util.AsyncTask import AsyncTask from util.AsyncTask import AsyncTask
@ -16,7 +18,7 @@ class HomeController:
# Variables # Variables
__main_controller = None __main_controller = None
__view = None __view = None
__webpic = None __webpic: WebPicDownloader = None
__download_task = None __download_task = None
# Constructor # Constructor
@ -43,7 +45,9 @@ class HomeController:
* :view: -> The view that this controller manage. * :view: -> The view that this controller manage.
""" """
self.__view = view self.__view = view
self.__webpic.set_messenger_callback(view.add_log) self.__webpic.set_messenger_callback(self.on_webpic_messenger)
self.__webpic.set_success_callback(self.on_webpic_success)
self.__webpic.set_failure_callback(self.on_webpic_failure)
# END View method # END View method
# START View events # START View events
@ -65,29 +69,41 @@ class HomeController:
* :name: -> The name of the folder in which put pictures. * :name: -> The name of the folder in which put pictures.
""" """
if url.strip() and name.strip(): if url.strip() and name.strip():
self.__download_task = AsyncTask( self.__webpic.start_downloading(url, name)
rcallback=self.__async_task_start,
rargs=(url, name),
qcallback=self.__async_task_stop
)
self.__download_task.start()
else: else:
self.__view.show_error_message("Opss, the url or folder name are not valid!") self.__view.show_error_message("Opss, the url or folder name are not valid!")
# END View events # END View events
# START Webpic events
def on_webpic_messenger(self, message: str) -> None:
"""
"""
self.__view.add_log(message)
def on_webpic_success(self) -> None:
"""
"""
self.__view.show_success_message("The download has been successfully completed.")
def on_webpic_failure(self) -> None:
"""
"""
self.__view.show_error_message("A critical error preventing the download occurred, check the logs.")
# END Webpic events
# START Controller methods # START Controller methods
def on_quit(self) -> bool: def on_quit(self) -> bool:
""" """
[event function for controller] [event function for controller]
=> Call this event when a request to exit is thrown. => Call this event when a request to exit is thrown.
""" """
if self.__download_task and self.__download_task.is_alive(): if self.__webpic.is_alive():
if self.__main_controller.show_question_dialog( if self.__main_controller.show_question_dialog(
"Are you sure?", "Are you sure?",
"Do you really want to quit while the download is running?\nThis will stop the download." "Do you really want to quit while the download is running?\nThis will stop the download."
): ):
self.__download_task.stop() self.__webpic.stop_downloading()
self.__download_task.join() time.sleep(4)
return False return False
else: else:
return True return True

View File

@ -1,11 +1,11 @@
import os import os
import threading from threading import Semaphore, Thread
from urllib import request from urllib import request
from urllib.error import HTTPError, URLError from urllib.error import HTTPError, URLError
from bs4 import BeautifulSoup, Tag, ResultSet from bs4 import BeautifulSoup, Tag, ResultSet
class WebPicDownloader(threading.Thread): class WebPicDownloader(Thread):
""" """
WebPicDownloader WebPicDownloader
@ -21,18 +21,16 @@ class WebPicDownloader(threading.Thread):
@version 1.2.0 @version 1.2.0
""" """
# Variables # Variables
path: str = None # Main working folder directory __callbacks: dict = None # Callback dictionary
headers: dict = None # Header parameters __settings: dict = None #
dl_setting: dict = None # Parameter dictionary __dl_infos: dict = None #
__sem: Semaphore = None #
callbacks: dict = None # Callback dictionary
_exit: bool = None # When set to True quit the thread
use_thread: bool = None # Indicates if the script must launch the download in a separate thread
async_run: bool = None # Idicate if the task can still run (for use in a thread)
# Constructor # Constructor
def __init__(self, path: str = None, headers: dict = None, asynchrone: bool = False, def __init__(self, path: str = None, headers: dict = None, messenger = None,
messenger = None, success = None, failure = None) -> None: success = None, failure = None) -> None:
""" """
Constructor Constructor
=> It is important to initialize the WebPicDownloader object properly. The callback => It is important to initialize the WebPicDownloader object properly. The callback
@ -46,24 +44,30 @@ class WebPicDownloader(threading.Thread):
* :success: -> Callback function success (see setter). * :success: -> Callback function success (see setter).
* :failure: -> Callback function failure (see setter). * :failure: -> Callback function failure (see setter).
""" """
super().__init__() super().__init__(daemon=True)
self.path = path if path else os.getcwd() self.__callbacks = {
self.headers = headers if headers else {
'User-Agent': "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
}
self.dl_setting = {
'url': None,
'name': None
}
self.callbacks = {
'messenger': messenger if messenger else lambda msg: print(msg), 'messenger': messenger if messenger else lambda msg: print(msg),
'success': success if success else lambda: print("Success!"), 'success': success if success else lambda: print("Success!"),
'failure': failure if failure else lambda: print("failure!") 'failure': failure if failure else lambda: print("failure!")
} }
self.__settings = {
self.use_thread = asynchrone 'root_path': path if path else os.getcwd(),
self.async_run = True 'headers': headers if headers else {
'User-Agent': "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
}
}
self.__dl_infos = {
'website_url': 'url',
'download_name': 'name',
'download_path': 'full_path',
'tot_image_count': 0,
'dl_image_count': 0,
'running': False
}
self.__sem = Semaphore(0)
self.__exit = False
self.start() # start deamon
# Internal functions # Internal functions
def __get_html(self, url: str) -> str: def __get_html(self, url: str) -> str:
@ -74,7 +78,7 @@ class WebPicDownloader(threading.Thread):
* :url: -> The url of the site for which we want to get the content of the HTML page. * :url: -> The url of the site for which we want to get the content of the HTML page.
* RETURN -> Web page content. * RETURN -> Web page content.
""" """
req = request.Request(url, headers=self.headers) req = request.Request(url, headers=self.__settings.get('headers'))
response = request.urlopen(req) response = request.urlopen(req)
return response.read().decode('utf-8') return response.read().decode('utf-8')
@ -132,7 +136,7 @@ class WebPicDownloader(threading.Thread):
* :url: -> Image url on the web. * :url: -> Image url on the web.
* :filename: -> Full path with name of the future image. * :filename: -> Full path with name of the future image.
""" """
req = request.Request(url, headers=self.headers) req = request.Request(url, headers=self.__settings.get('headers'))
raw_img = request.urlopen(req).read() raw_img = request.urlopen(req).read()
with open(filename, 'wb') as img: with open(filename, 'wb') as img:
img.write(raw_img) img.write(raw_img)
@ -155,88 +159,16 @@ class WebPicDownloader(threading.Thread):
Internal Function #do-not-use# Internal Function #do-not-use#
=> Use the messenger callback to send a message. => Use the messenger callback to send a message.
""" """
self.callbacks.get('messenger')(message) self.__callbacks.get('messenger')(message)
# Public functions # Public functions
def start_downloading(self, url: str, folder_name: str) -> None:
"""
Start downloading all pictures of a website.
Depending on the configuration the download will either be started in the
main process or in a separate thread.
* :url: -> The url of the website to annalyse.
* :folder_name: -> The name of the folder in which to upload the photos.
"""
# Updating settings
self.dl_setting['url'] = url
self.dl_setting['name'] = folder_name
# Start processing
if self.use_thread:
self.async_run = True
self.start()
else:
self.run()
def stop_downloading(self) -> None:
"""
Stops the download after the current item is processed and exit the downloading thread.
<!> this function is available only if asynchronous option is set to True in constructor. <!>
"""
if self.use_thread:
self.async_run = False
else:
raise Exception("Can not stop a no multi thread execution")
def run(self) -> None:
"""
<!>
This function must not be used! It is an internal function called automatically
when the download task starts.
<!>
"""
try:
# init variables
download_count = 0 # count to 0
download_folder = f"{self.path}/{self.dl_setting.get('name')}/" # format path
website_html = self.__get_html(self.dl_setting.get('url')) # website html
images = self.__find_all_img(website_html) # find all img balises ing html
self.__initialize_folder(download_folder) # initialize download folder
self.__msg(f"WebPicDownloader found {len(images)} images on the website.")
# process pictures
for i, img in enumerate(images):
if self.use_thread and not self.async_run:
return
try:
self.__msg(f"Start downloading image {i}.")
img_link = self.__find_img_link(img)
self.__download_img(img_link, f"{download_folder}image-{i}.{self.__find_image_type(img_link)}")
self.__msg(f"Download of image {i}, done!")
download_count += 1
except Exception as err:
self.__msg(f"ERROR: Unable to process image {i} -> err[{err}].")
self.__msg(f"WebPicDownloader has processed {download_count} images out of {len(images)}.")
self.callbacks.get('success')() # success, launch callback and return
return
except HTTPError as err:
self.__msg(f"ERROR: An http error occured -> err[{err}].")
except (ValueError, URLError) as err:
self.__msg(f"ERROT: An error occured with the url -> err[{err}].")
except Exception as err:
self.__msg(f"ERROR: An unknown error occured -> err[{err}]")
self.callbacks.get('failure')() # failure, launch callback and exit
def set_success_callback(self, callback) -> None: def set_success_callback(self, callback) -> None:
""" """
Setter to define the callback function when the download succeeded. Setter to define the callback function when the download succeeded.
* :callback: -> the callback function to call when the download is a success. * :callback: -> the callback function to call when the download is a success.
""" """
self.callbacks['success'] = callback self.__callbacks['success'] = callback
def set_failure_callback(self, callback) -> None: def set_failure_callback(self, callback) -> None:
""" """
@ -244,7 +176,7 @@ class WebPicDownloader(threading.Thread):
* :callback: -> the callback function to call when the download is a failure. * :callback: -> the callback function to call when the download is a failure.
""" """
self.callbacks['failure'] = callback self.__callbacks['failure'] = callback
def set_messenger_callback(self, callback) -> None: def set_messenger_callback(self, callback) -> None:
""" """
@ -252,15 +184,76 @@ class WebPicDownloader(threading.Thread):
* :callback: -> the callback function to call when a message event is emited. * :callback: -> the callback function to call when a message event is emited.
""" """
self.callbacks['messenger'] = callback self.__callbacks['messenger'] = callback
def start_downloading(self, url: str, name: str) -> None:
"""
TODO desc
"""
if self.__dl_infos.get('running'):
print("bussy")
else:
self.__dl_infos['website_url'] = url
self.__dl_infos['download_name'] = name
self.__sem.release()
def stop_downloading(self, block=False) -> None:
"""
TODO DESC
"""
self.__exit = True
self.__sem.release()
if block:
self.join()
# Thread corp function
def run(self) -> None:
while True:
self.__sem.acquire()
if self.__exit:
return
self.__dl_infos['running'] = True # reserv run
try:
html = self.__get_html(self.__dl_infos.get('website_url')) # website html
images = self.__find_all_img(html) # find all img balises ing html
self.__dl_infos['tot_image_count'] = len(images) # count total image
self.__dl_infos['dl_image_count'] = 0 # set download count to 0
self.__dl_infos['download_path'] = f"{self.__settings.get('root_path')}/{self.__dl_infos.get('download_name')}/" # format path
self.__initialize_folder(self.__dl_infos.get('download_path')) # Init download folder
self.__msg(f"WebPicDownloader found {self.__dl_infos.get('tot_image_count')} images on the website.")
# process pictures
for i, img in enumerate(images):
try:
self.__msg(f"Start downloading image {i}.")
img_link = self.__find_img_link(img) # find image link
self.__download_img(img_link, f"{self.__dl_infos.get('download_path')}image-{i}.{self.__find_image_type(img_link)}") # download the image
self.__msg(f"Download of image {i}, done!")
self.__dl_infos['dl_image_count'] += 1 # increment download counter
except Exception as err:
self.__msg(f"ERROR: Unable to process image {i} -> err[{err}].")
self.__msg(f"WebPicDownloader has processed {self.__dl_infos.get('dl_image_count')} images out of {self.__dl_infos.get('tot_image_count')}.")
self.__callbacks.get('success')() # success, launch callback
except Exception as err:
self.__msg(f"ERROR: An error occured -> err[{err}]")
self.__callbacks.get('failure')() # error, launch callback
self.__dl_infos['running'] = False # free run
if __name__ == "__main__": if __name__ == "__main__":
# Internal entry point for testing and consol use. # Internal entry point for testing and consol use.
wpd = WebPicDownloader() wpd = WebPicDownloader()
def lol(msg):
pass
wpd.set_messenger_callback(lol)
while True: while True:
url = input("Website URL ? ") url = input("Website URL ? ")
name = input("Folder name ? ") name = input("Folder name ? ")
wpd.start_downloading(url, name) wpd.start_downloading(url, name)
if "n" == input("Do you want to continue [Y/n] ? ").lower(): if "n" == input("Do you want to continue [Y/n] ? ").lower():
wpd.stop_downloading()
break break
print("Good bye !") print("Good bye !")