The thread system not being sufficient remodulation of it
This commit is contained in:
parent
63e67772a3
commit
539b75cb09
@ -1,4 +1,6 @@
|
|||||||
|
import time
|
||||||
from controller.MainController import MainController
|
from controller.MainController import MainController
|
||||||
|
from model.WebPicDownloader import WebPicDownloader
|
||||||
from util.AsyncTask import AsyncTask
|
from util.AsyncTask import AsyncTask
|
||||||
|
|
||||||
|
|
||||||
@ -16,7 +18,7 @@ class HomeController:
|
|||||||
# Variables
|
# Variables
|
||||||
__main_controller = None
|
__main_controller = None
|
||||||
__view = None
|
__view = None
|
||||||
__webpic = None
|
__webpic: WebPicDownloader = None
|
||||||
__download_task = None
|
__download_task = None
|
||||||
|
|
||||||
# Constructor
|
# Constructor
|
||||||
@ -43,7 +45,9 @@ class HomeController:
|
|||||||
* :view: -> The view that this controller manage.
|
* :view: -> The view that this controller manage.
|
||||||
"""
|
"""
|
||||||
self.__view = view
|
self.__view = view
|
||||||
self.__webpic.set_messenger_callback(view.add_log)
|
self.__webpic.set_messenger_callback(self.on_webpic_messenger)
|
||||||
|
self.__webpic.set_success_callback(self.on_webpic_success)
|
||||||
|
self.__webpic.set_failure_callback(self.on_webpic_failure)
|
||||||
# END View method
|
# END View method
|
||||||
|
|
||||||
# START View events
|
# START View events
|
||||||
@ -65,29 +69,41 @@ class HomeController:
|
|||||||
* :name: -> The name of the folder in which put pictures.
|
* :name: -> The name of the folder in which put pictures.
|
||||||
"""
|
"""
|
||||||
if url.strip() and name.strip():
|
if url.strip() and name.strip():
|
||||||
self.__download_task = AsyncTask(
|
self.__webpic.start_downloading(url, name)
|
||||||
rcallback=self.__async_task_start,
|
|
||||||
rargs=(url, name),
|
|
||||||
qcallback=self.__async_task_stop
|
|
||||||
)
|
|
||||||
self.__download_task.start()
|
|
||||||
else:
|
else:
|
||||||
self.__view.show_error_message("Opss, the url or folder name are not valid!")
|
self.__view.show_error_message("Opss, the url or folder name are not valid!")
|
||||||
# END View events
|
# END View events
|
||||||
|
|
||||||
|
# START Webpic events
|
||||||
|
def on_webpic_messenger(self, message: str) -> None:
|
||||||
|
"""
|
||||||
|
"""
|
||||||
|
self.__view.add_log(message)
|
||||||
|
|
||||||
|
def on_webpic_success(self) -> None:
|
||||||
|
"""
|
||||||
|
"""
|
||||||
|
self.__view.show_success_message("The download has been successfully completed.")
|
||||||
|
|
||||||
|
def on_webpic_failure(self) -> None:
|
||||||
|
"""
|
||||||
|
"""
|
||||||
|
self.__view.show_error_message("A critical error preventing the download occurred, check the logs.")
|
||||||
|
# END Webpic events
|
||||||
|
|
||||||
# START Controller methods
|
# START Controller methods
|
||||||
def on_quit(self) -> bool:
|
def on_quit(self) -> bool:
|
||||||
"""
|
"""
|
||||||
[event function for controller]
|
[event function for controller]
|
||||||
=> Call this event when a request to exit is thrown.
|
=> Call this event when a request to exit is thrown.
|
||||||
"""
|
"""
|
||||||
if self.__download_task and self.__download_task.is_alive():
|
if self.__webpic.is_alive():
|
||||||
if self.__main_controller.show_question_dialog(
|
if self.__main_controller.show_question_dialog(
|
||||||
"Are you sure?",
|
"Are you sure?",
|
||||||
"Do you really want to quit while the download is running?\nThis will stop the download."
|
"Do you really want to quit while the download is running?\nThis will stop the download."
|
||||||
):
|
):
|
||||||
self.__download_task.stop()
|
self.__webpic.stop_downloading()
|
||||||
self.__download_task.join()
|
time.sleep(4)
|
||||||
return False
|
return False
|
||||||
else:
|
else:
|
||||||
return True
|
return True
|
||||||
|
@ -1,11 +1,11 @@
|
|||||||
import os
|
import os
|
||||||
import threading
|
from threading import Semaphore, Thread
|
||||||
from urllib import request
|
from urllib import request
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from bs4 import BeautifulSoup, Tag, ResultSet
|
from bs4 import BeautifulSoup, Tag, ResultSet
|
||||||
|
|
||||||
|
|
||||||
class WebPicDownloader(threading.Thread):
|
class WebPicDownloader(Thread):
|
||||||
"""
|
"""
|
||||||
WebPicDownloader
|
WebPicDownloader
|
||||||
|
|
||||||
@ -21,18 +21,16 @@ class WebPicDownloader(threading.Thread):
|
|||||||
@version 1.2.0
|
@version 1.2.0
|
||||||
"""
|
"""
|
||||||
# Variables
|
# Variables
|
||||||
path: str = None # Main working folder directory
|
__callbacks: dict = None # Callback dictionary
|
||||||
headers: dict = None # Header parameters
|
__settings: dict = None #
|
||||||
dl_setting: dict = None # Parameter dictionary
|
__dl_infos: dict = None #
|
||||||
|
__sem: Semaphore = None #
|
||||||
callbacks: dict = None # Callback dictionary
|
|
||||||
|
_exit: bool = None # When set to True quit the thread
|
||||||
use_thread: bool = None # Indicates if the script must launch the download in a separate thread
|
|
||||||
async_run: bool = None # Idicate if the task can still run (for use in a thread)
|
|
||||||
|
|
||||||
# Constructor
|
# Constructor
|
||||||
def __init__(self, path: str = None, headers: dict = None, asynchrone: bool = False,
|
def __init__(self, path: str = None, headers: dict = None, messenger = None,
|
||||||
messenger = None, success = None, failure = None) -> None:
|
success = None, failure = None) -> None:
|
||||||
"""
|
"""
|
||||||
Constructor
|
Constructor
|
||||||
=> It is important to initialize the WebPicDownloader object properly. The callback
|
=> It is important to initialize the WebPicDownloader object properly. The callback
|
||||||
@ -46,24 +44,30 @@ class WebPicDownloader(threading.Thread):
|
|||||||
* :success: -> Callback function success (see setter).
|
* :success: -> Callback function success (see setter).
|
||||||
* :failure: -> Callback function failure (see setter).
|
* :failure: -> Callback function failure (see setter).
|
||||||
"""
|
"""
|
||||||
super().__init__()
|
super().__init__(daemon=True)
|
||||||
self.path = path if path else os.getcwd()
|
self.__callbacks = {
|
||||||
self.headers = headers if headers else {
|
|
||||||
'User-Agent': "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
|
|
||||||
}
|
|
||||||
self.dl_setting = {
|
|
||||||
'url': None,
|
|
||||||
'name': None
|
|
||||||
}
|
|
||||||
|
|
||||||
self.callbacks = {
|
|
||||||
'messenger': messenger if messenger else lambda msg: print(msg),
|
'messenger': messenger if messenger else lambda msg: print(msg),
|
||||||
'success': success if success else lambda: print("Success!"),
|
'success': success if success else lambda: print("Success!"),
|
||||||
'failure': failure if failure else lambda: print("failure!")
|
'failure': failure if failure else lambda: print("failure!")
|
||||||
}
|
}
|
||||||
|
self.__settings = {
|
||||||
self.use_thread = asynchrone
|
'root_path': path if path else os.getcwd(),
|
||||||
self.async_run = True
|
'headers': headers if headers else {
|
||||||
|
'User-Agent': "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.__dl_infos = {
|
||||||
|
'website_url': 'url',
|
||||||
|
'download_name': 'name',
|
||||||
|
'download_path': 'full_path',
|
||||||
|
'tot_image_count': 0,
|
||||||
|
'dl_image_count': 0,
|
||||||
|
'running': False
|
||||||
|
}
|
||||||
|
self.__sem = Semaphore(0)
|
||||||
|
self.__exit = False
|
||||||
|
|
||||||
|
self.start() # start deamon
|
||||||
|
|
||||||
# Internal functions
|
# Internal functions
|
||||||
def __get_html(self, url: str) -> str:
|
def __get_html(self, url: str) -> str:
|
||||||
@ -74,7 +78,7 @@ class WebPicDownloader(threading.Thread):
|
|||||||
* :url: -> The url of the site for which we want to get the content of the HTML page.
|
* :url: -> The url of the site for which we want to get the content of the HTML page.
|
||||||
* RETURN -> Web page content.
|
* RETURN -> Web page content.
|
||||||
"""
|
"""
|
||||||
req = request.Request(url, headers=self.headers)
|
req = request.Request(url, headers=self.__settings.get('headers'))
|
||||||
response = request.urlopen(req)
|
response = request.urlopen(req)
|
||||||
return response.read().decode('utf-8')
|
return response.read().decode('utf-8')
|
||||||
|
|
||||||
@ -132,7 +136,7 @@ class WebPicDownloader(threading.Thread):
|
|||||||
* :url: -> Image url on the web.
|
* :url: -> Image url on the web.
|
||||||
* :filename: -> Full path with name of the future image.
|
* :filename: -> Full path with name of the future image.
|
||||||
"""
|
"""
|
||||||
req = request.Request(url, headers=self.headers)
|
req = request.Request(url, headers=self.__settings.get('headers'))
|
||||||
raw_img = request.urlopen(req).read()
|
raw_img = request.urlopen(req).read()
|
||||||
with open(filename, 'wb') as img:
|
with open(filename, 'wb') as img:
|
||||||
img.write(raw_img)
|
img.write(raw_img)
|
||||||
@ -155,88 +159,16 @@ class WebPicDownloader(threading.Thread):
|
|||||||
Internal Function #do-not-use#
|
Internal Function #do-not-use#
|
||||||
=> Use the messenger callback to send a message.
|
=> Use the messenger callback to send a message.
|
||||||
"""
|
"""
|
||||||
self.callbacks.get('messenger')(message)
|
self.__callbacks.get('messenger')(message)
|
||||||
|
|
||||||
# Public functions
|
# Public functions
|
||||||
def start_downloading(self, url: str, folder_name: str) -> None:
|
|
||||||
"""
|
|
||||||
Start downloading all pictures of a website.
|
|
||||||
Depending on the configuration the download will either be started in the
|
|
||||||
main process or in a separate thread.
|
|
||||||
|
|
||||||
* :url: -> The url of the website to annalyse.
|
|
||||||
* :folder_name: -> The name of the folder in which to upload the photos.
|
|
||||||
"""
|
|
||||||
# Updating settings
|
|
||||||
self.dl_setting['url'] = url
|
|
||||||
self.dl_setting['name'] = folder_name
|
|
||||||
|
|
||||||
# Start processing
|
|
||||||
if self.use_thread:
|
|
||||||
self.async_run = True
|
|
||||||
self.start()
|
|
||||||
else:
|
|
||||||
self.run()
|
|
||||||
|
|
||||||
def stop_downloading(self) -> None:
|
|
||||||
"""
|
|
||||||
Stops the download after the current item is processed and exit the downloading thread.
|
|
||||||
|
|
||||||
<!> this function is available only if asynchronous option is set to True in constructor. <!>
|
|
||||||
"""
|
|
||||||
if self.use_thread:
|
|
||||||
self.async_run = False
|
|
||||||
else:
|
|
||||||
raise Exception("Can not stop a no multi thread execution")
|
|
||||||
|
|
||||||
def run(self) -> None:
|
|
||||||
"""
|
|
||||||
<!>
|
|
||||||
This function must not be used! It is an internal function called automatically
|
|
||||||
when the download task starts.
|
|
||||||
<!>
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
# init variables
|
|
||||||
download_count = 0 # count to 0
|
|
||||||
download_folder = f"{self.path}/{self.dl_setting.get('name')}/" # format path
|
|
||||||
website_html = self.__get_html(self.dl_setting.get('url')) # website html
|
|
||||||
images = self.__find_all_img(website_html) # find all img balises ing html
|
|
||||||
|
|
||||||
self.__initialize_folder(download_folder) # initialize download folder
|
|
||||||
self.__msg(f"WebPicDownloader found {len(images)} images on the website.")
|
|
||||||
|
|
||||||
# process pictures
|
|
||||||
for i, img in enumerate(images):
|
|
||||||
if self.use_thread and not self.async_run:
|
|
||||||
return
|
|
||||||
try:
|
|
||||||
self.__msg(f"Start downloading image {i}.")
|
|
||||||
img_link = self.__find_img_link(img)
|
|
||||||
self.__download_img(img_link, f"{download_folder}image-{i}.{self.__find_image_type(img_link)}")
|
|
||||||
self.__msg(f"Download of image {i}, done!")
|
|
||||||
download_count += 1
|
|
||||||
except Exception as err:
|
|
||||||
self.__msg(f"ERROR: Unable to process image {i} -> err[{err}].")
|
|
||||||
self.__msg(f"WebPicDownloader has processed {download_count} images out of {len(images)}.")
|
|
||||||
self.callbacks.get('success')() # success, launch callback and return
|
|
||||||
return
|
|
||||||
|
|
||||||
except HTTPError as err:
|
|
||||||
self.__msg(f"ERROR: An http error occured -> err[{err}].")
|
|
||||||
except (ValueError, URLError) as err:
|
|
||||||
self.__msg(f"ERROT: An error occured with the url -> err[{err}].")
|
|
||||||
except Exception as err:
|
|
||||||
self.__msg(f"ERROR: An unknown error occured -> err[{err}]")
|
|
||||||
self.callbacks.get('failure')() # failure, launch callback and exit
|
|
||||||
|
|
||||||
def set_success_callback(self, callback) -> None:
|
def set_success_callback(self, callback) -> None:
|
||||||
"""
|
"""
|
||||||
Setter to define the callback function when the download succeeded.
|
Setter to define the callback function when the download succeeded.
|
||||||
|
|
||||||
* :callback: -> the callback function to call when the download is a success.
|
* :callback: -> the callback function to call when the download is a success.
|
||||||
"""
|
"""
|
||||||
self.callbacks['success'] = callback
|
self.__callbacks['success'] = callback
|
||||||
|
|
||||||
def set_failure_callback(self, callback) -> None:
|
def set_failure_callback(self, callback) -> None:
|
||||||
"""
|
"""
|
||||||
@ -244,7 +176,7 @@ class WebPicDownloader(threading.Thread):
|
|||||||
|
|
||||||
* :callback: -> the callback function to call when the download is a failure.
|
* :callback: -> the callback function to call when the download is a failure.
|
||||||
"""
|
"""
|
||||||
self.callbacks['failure'] = callback
|
self.__callbacks['failure'] = callback
|
||||||
|
|
||||||
def set_messenger_callback(self, callback) -> None:
|
def set_messenger_callback(self, callback) -> None:
|
||||||
"""
|
"""
|
||||||
@ -252,15 +184,76 @@ class WebPicDownloader(threading.Thread):
|
|||||||
|
|
||||||
* :callback: -> the callback function to call when a message event is emited.
|
* :callback: -> the callback function to call when a message event is emited.
|
||||||
"""
|
"""
|
||||||
self.callbacks['messenger'] = callback
|
self.__callbacks['messenger'] = callback
|
||||||
|
|
||||||
|
def start_downloading(self, url: str, name: str) -> None:
|
||||||
|
"""
|
||||||
|
TODO desc
|
||||||
|
"""
|
||||||
|
if self.__dl_infos.get('running'):
|
||||||
|
print("bussy")
|
||||||
|
else:
|
||||||
|
self.__dl_infos['website_url'] = url
|
||||||
|
self.__dl_infos['download_name'] = name
|
||||||
|
self.__sem.release()
|
||||||
|
|
||||||
|
def stop_downloading(self, block=False) -> None:
|
||||||
|
"""
|
||||||
|
TODO DESC
|
||||||
|
"""
|
||||||
|
self.__exit = True
|
||||||
|
self.__sem.release()
|
||||||
|
if block:
|
||||||
|
self.join()
|
||||||
|
|
||||||
|
# Thread corp function
|
||||||
|
def run(self) -> None:
|
||||||
|
while True:
|
||||||
|
self.__sem.acquire()
|
||||||
|
|
||||||
|
if self.__exit:
|
||||||
|
return
|
||||||
|
|
||||||
|
self.__dl_infos['running'] = True # reserv run
|
||||||
|
try:
|
||||||
|
html = self.__get_html(self.__dl_infos.get('website_url')) # website html
|
||||||
|
images = self.__find_all_img(html) # find all img balises ing html
|
||||||
|
|
||||||
|
self.__dl_infos['tot_image_count'] = len(images) # count total image
|
||||||
|
self.__dl_infos['dl_image_count'] = 0 # set download count to 0
|
||||||
|
self.__dl_infos['download_path'] = f"{self.__settings.get('root_path')}/{self.__dl_infos.get('download_name')}/" # format path
|
||||||
|
|
||||||
|
self.__initialize_folder(self.__dl_infos.get('download_path')) # Init download folder
|
||||||
|
self.__msg(f"WebPicDownloader found {self.__dl_infos.get('tot_image_count')} images on the website.")
|
||||||
|
|
||||||
|
# process pictures
|
||||||
|
for i, img in enumerate(images):
|
||||||
|
try:
|
||||||
|
self.__msg(f"Start downloading image {i}.")
|
||||||
|
img_link = self.__find_img_link(img) # find image link
|
||||||
|
self.__download_img(img_link, f"{self.__dl_infos.get('download_path')}image-{i}.{self.__find_image_type(img_link)}") # download the image
|
||||||
|
self.__msg(f"Download of image {i}, done!")
|
||||||
|
self.__dl_infos['dl_image_count'] += 1 # increment download counter
|
||||||
|
except Exception as err:
|
||||||
|
self.__msg(f"ERROR: Unable to process image {i} -> err[{err}].")
|
||||||
|
self.__msg(f"WebPicDownloader has processed {self.__dl_infos.get('dl_image_count')} images out of {self.__dl_infos.get('tot_image_count')}.")
|
||||||
|
self.__callbacks.get('success')() # success, launch callback
|
||||||
|
except Exception as err:
|
||||||
|
self.__msg(f"ERROR: An error occured -> err[{err}]")
|
||||||
|
self.__callbacks.get('failure')() # error, launch callback
|
||||||
|
self.__dl_infos['running'] = False # free run
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
# Internal entry point for testing and consol use.
|
# Internal entry point for testing and consol use.
|
||||||
wpd = WebPicDownloader()
|
wpd = WebPicDownloader()
|
||||||
|
def lol(msg):
|
||||||
|
pass
|
||||||
|
wpd.set_messenger_callback(lol)
|
||||||
while True:
|
while True:
|
||||||
url = input("Website URL ? ")
|
url = input("Website URL ? ")
|
||||||
name = input("Folder name ? ")
|
name = input("Folder name ? ")
|
||||||
wpd.start_downloading(url, name)
|
wpd.start_downloading(url, name)
|
||||||
if "n" == input("Do you want to continue [Y/n] ? ").lower():
|
if "n" == input("Do you want to continue [Y/n] ? ").lower():
|
||||||
|
wpd.stop_downloading()
|
||||||
break
|
break
|
||||||
print("Good bye !")
|
print("Good bye !")
|
Reference in New Issue
Block a user