EndMove/WebPicDownloader
EndMove
/
WebPicDownloader
Archived
2
1
Fork 1
Code Issues Pull Requests Projects Releases 1 Activity

The thread system not being sufficient remodulation of it

This commit is contained in:
Jérémi N ‘EndMove’ 2022-09-05 16:14:05 +02:00
parent 63e67772a3
commit 539b75cb09
Signed by: EndMove
GPG Key ID: 65C4A02E1F5371A4
2 changed files with 124 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
controller/HomeController.py
View File

@ -1,4 +1,6 @@
import time
from controller.MainController import MainController
from model.WebPicDownloader import WebPicDownloader
from util.AsyncTask import AsyncTask
@ -16,7 +18,7 @@ class HomeController:
# Variables
__main_controller = None
__view = None
__webpic = None
__webpic: WebPicDownloader = None
__download_task = None
# Constructor
@ -43,7 +45,9 @@ class HomeController:
* :view: -> The view that this controller manage.
"""
self.__view = view
self.__webpic.set_messenger_callback(view.add_log)
self.__webpic.set_messenger_callback(self.on_webpic_messenger)
self.__webpic.set_success_callback(self.on_webpic_success)
self.__webpic.set_failure_callback(self.on_webpic_failure)
# END View method
# START View events
@ -65,29 +69,41 @@ class HomeController:
* :name: -> The name of the folder in which put pictures.
"""
if url.strip() and name.strip():
self.__download_task = AsyncTask(
rcallback=self.__async_task_start,
rargs=(url, name),
qcallback=self.__async_task_stop
)
self.__download_task.start()
self.__webpic.start_downloading(url, name)
else:
self.__view.show_error_message("Opss, the url or folder name are not valid!")
# END View events
# START Webpic events
def on_webpic_messenger(self, message: str) -> None:
"""
"""
self.__view.add_log(message)
def on_webpic_success(self) -> None:
"""
"""
self.__view.show_success_message("The download has been successfully completed.")
def on_webpic_failure(self) -> None:
"""
"""
self.__view.show_error_message("A critical error preventing the download occurred, check the logs.")
# END Webpic events
# START Controller methods
def on_quit(self) -> bool:
"""
[event function for controller]
=> Call this event when a request to exit is thrown.
"""
if self.__download_task and self.__download_task.is_alive():
if self.__webpic.is_alive():
if self.__main_controller.show_question_dialog(
"Are you sure?",
"Do you really want to quit while the download is running?\nThis will stop the download."
):
self.__download_task.stop()
self.__download_task.join()
self.__webpic.stop_downloading()
time.sleep(4)
return False
else:
return True

201
model/WebPicDownloader.py
View File

@ -1,11 +1,11 @@
import os
import threading
from threading import Semaphore, Thread
from urllib import request
from urllib.error import HTTPError, URLError
from bs4 import BeautifulSoup, Tag, ResultSet
class WebPicDownloader(threading.Thread):
class WebPicDownloader(Thread):
"""
WebPicDownloader
@ -21,18 +21,16 @@ class WebPicDownloader(threading.Thread):
@version 1.2.0
"""
# Variables
path: str = None # Main working folder directory
headers: dict = None # Header parameters
dl_setting: dict = None # Parameter dictionary
callbacks: dict = None # Callback dictionary
use_thread: bool = None # Indicates if the script must launch the download in a separate thread
async_run: bool = None # Idicate if the task can still run (for use in a thread)
__callbacks: dict = None # Callback dictionary
__settings: dict = None #
__dl_infos: dict = None #
__sem: Semaphore = None #
_exit: bool = None # When set to True quit the thread
# Constructor
def __init__(self, path: str = None, headers: dict = None, asynchrone: bool = False,
messenger = None, success = None, failure = None) -> None:
def __init__(self, path: str = None, headers: dict = None, messenger = None,
success = None, failure = None) -> None:
"""
Constructor
=> It is important to initialize the WebPicDownloader object properly. The callback
@ -46,24 +44,30 @@ class WebPicDownloader(threading.Thread):
* :success: -> Callback function success (see setter).
* :failure: -> Callback function failure (see setter).
"""
super().__init__()
self.path = path if path else os.getcwd()
self.headers = headers if headers else {
'User-Agent': "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
}
self.dl_setting = {
'url': None,
'name': None
}
self.callbacks = {
super().__init__(daemon=True)
self.__callbacks = {
'messenger': messenger if messenger else lambda msg: print(msg),
'success': success if success else lambda: print("Success!"),
'failure': failure if failure else lambda: print("failure!")
}
self.use_thread = asynchrone
self.async_run = True
self.__settings = {
'root_path': path if path else os.getcwd(),
'headers': headers if headers else {
'User-Agent': "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
}
}
self.__dl_infos = {
'website_url': 'url',
'download_name': 'name',
'download_path': 'full_path',
'tot_image_count': 0,
'dl_image_count': 0,
'running': False
}
self.__sem = Semaphore(0)
self.__exit = False
self.start() # start deamon
# Internal functions
def __get_html(self, url: str) -> str:
@ -74,7 +78,7 @@ class WebPicDownloader(threading.Thread):
* :url: -> The url of the site for which we want to get the content of the HTML page.
* RETURN -> Web page content.
"""
req = request.Request(url, headers=self.headers)
req = request.Request(url, headers=self.__settings.get('headers'))
response = request.urlopen(req)
return response.read().decode('utf-8')
@ -132,7 +136,7 @@ class WebPicDownloader(threading.Thread):
* :url: -> Image url on the web.
* :filename: -> Full path with name of the future image.
"""
req = request.Request(url, headers=self.headers)
req = request.Request(url, headers=self.__settings.get('headers'))
raw_img = request.urlopen(req).read()
with open(filename, 'wb') as img:
img.write(raw_img)
@ -155,88 +159,16 @@ class WebPicDownloader(threading.Thread):
Internal Function #do-not-use#
=> Use the messenger callback to send a message.
"""
self.callbacks.get('messenger')(message)
self.__callbacks.get('messenger')(message)
# Public functions
def start_downloading(self, url: str, folder_name: str) -> None:
"""
Start downloading all pictures of a website.
Depending on the configuration the download will either be started in the
main process or in a separate thread.
* :url: -> The url of the website to annalyse.
* :folder_name: -> The name of the folder in which to upload the photos.
"""
# Updating settings
self.dl_setting['url'] = url
self.dl_setting['name'] = folder_name
# Start processing
if self.use_thread:
self.async_run = True
self.start()
else:
self.run()
def stop_downloading(self) -> None:
"""
Stops the download after the current item is processed and exit the downloading thread.
<!> this function is available only if asynchronous option is set to True in constructor. <!>
"""
if self.use_thread:
self.async_run = False
else:
raise Exception("Can not stop a no multi thread execution")
def run(self) -> None:
"""
<!>
This function must not be used! It is an internal function called automatically
when the download task starts.
<!>
"""
try:
# init variables
download_count = 0 # count to 0
download_folder = f"{self.path}/{self.dl_setting.get('name')}/" # format path
website_html = self.__get_html(self.dl_setting.get('url')) # website html
images = self.__find_all_img(website_html) # find all img balises ing html
self.__initialize_folder(download_folder) # initialize download folder
self.__msg(f"WebPicDownloader found {len(images)} images on the website.")
# process pictures
for i, img in enumerate(images):
if self.use_thread and not self.async_run:
return
try:
self.__msg(f"Start downloading image {i}.")
img_link = self.__find_img_link(img)
self.__download_img(img_link, f"{download_folder}image-{i}.{self.__find_image_type(img_link)}")
self.__msg(f"Download of image {i}, done!")
download_count += 1
except Exception as err:
self.__msg(f"ERROR: Unable to process image {i} -> err[{err}].")
self.__msg(f"WebPicDownloader has processed {download_count} images out of {len(images)}.")
self.callbacks.get('success')() # success, launch callback and return
return
except HTTPError as err:
self.__msg(f"ERROR: An http error occured -> err[{err}].")
except (ValueError, URLError) as err:
self.__msg(f"ERROT: An error occured with the url -> err[{err}].")
except Exception as err:
self.__msg(f"ERROR: An unknown error occured -> err[{err}]")
self.callbacks.get('failure')() # failure, launch callback and exit
def set_success_callback(self, callback) -> None:
"""
Setter to define the callback function when the download succeeded.
* :callback: -> the callback function to call when the download is a success.
"""
self.callbacks['success'] = callback
self.__callbacks['success'] = callback
def set_failure_callback(self, callback) -> None:
"""
@ -244,7 +176,7 @@ class WebPicDownloader(threading.Thread):
* :callback: -> the callback function to call when the download is a failure.
"""
self.callbacks['failure'] = callback
self.__callbacks['failure'] = callback
def set_messenger_callback(self, callback) -> None:
"""
@ -252,15 +184,76 @@ class WebPicDownloader(threading.Thread):
* :callback: -> the callback function to call when a message event is emited.
"""
self.callbacks['messenger'] = callback
self.__callbacks['messenger'] = callback
def start_downloading(self, url: str, name: str) -> None:
"""
TODO desc
"""
if self.__dl_infos.get('running'):
print("bussy")
else:
self.__dl_infos['website_url'] = url
self.__dl_infos['download_name'] = name
self.__sem.release()
def stop_downloading(self, block=False) -> None:
"""
TODO DESC
"""
self.__exit = True
self.__sem.release()
if block:
self.join()
# Thread corp function
def run(self) -> None:
while True:
self.__sem.acquire()
if self.__exit:
return
self.__dl_infos['running'] = True # reserv run
try:
html = self.__get_html(self.__dl_infos.get('website_url')) # website html
images = self.__find_all_img(html) # find all img balises ing html
self.__dl_infos['tot_image_count'] = len(images) # count total image
self.__dl_infos['dl_image_count'] = 0 # set download count to 0
self.__dl_infos['download_path'] = f"{self.__settings.get('root_path')}/{self.__dl_infos.get('download_name')}/" # format path
self.__initialize_folder(self.__dl_infos.get('download_path')) # Init download folder
self.__msg(f"WebPicDownloader found {self.__dl_infos.get('tot_image_count')} images on the website.")
# process pictures
for i, img in enumerate(images):
try:
self.__msg(f"Start downloading image {i}.")
img_link = self.__find_img_link(img) # find image link
self.__download_img(img_link, f"{self.__dl_infos.get('download_path')}image-{i}.{self.__find_image_type(img_link)}") # download the image
self.__msg(f"Download of image {i}, done!")
self.__dl_infos['dl_image_count'] += 1 # increment download counter
except Exception as err:
self.__msg(f"ERROR: Unable to process image {i} -> err[{err}].")
self.__msg(f"WebPicDownloader has processed {self.__dl_infos.get('dl_image_count')} images out of {self.__dl_infos.get('tot_image_count')}.")
self.__callbacks.get('success')() # success, launch callback
except Exception as err:
self.__msg(f"ERROR: An error occured -> err[{err}]")
self.__callbacks.get('failure')() # error, launch callback
self.__dl_infos['running'] = False # free run
if __name__ == "__main__":
# Internal entry point for testing and consol use.
wpd = WebPicDownloader()
def lol(msg):
pass
wpd.set_messenger_callback(lol)
while True:
url = input("Website URL ? ")
name = input("Folder name ? ")
wpd.start_downloading(url, name)
if "n" == input("Do you want to continue [Y/n] ? ").lower():
wpd.stop_downloading()
break
print("Good bye !")