diff --git a/README.md b/README.md index 2f2914b..8f2bbf2 100644 --- a/README.md +++ b/README.md @@ -3,17 +3,17 @@ pyloader - A simple python downloader [![Build Status](https://travis-ci.org/linuxwhatelse/pyloader.svg?branch=master)](https://travis-ci.org/linuxwhatelse/pyloader) [![pypi](https://img.shields.io/pypi/v/lwe-pyloader.svg)](https://pypi.python.org/pypi/lwe-pyloader) -**pyloader** is a simple, easy to use, multi-threaded downloader with queuing support. +**pyloader** is a simple, easy to use, multi-threaded downloader with queuing support. -It is **NOT** a command-line utility but instead something you can (if you want) implement -in one (or more, I don't care :)) of your applications. +It is **NOT** a command-line utility but instead something you can (if you want) implement +in one (or more, I don't care :)) of your applications. -I was in need for such a thing and that's why I wrote it myself after only finding command-line utilities. -(I haven't spent a lot of time searching though) +I wrote project-specific downloader a few times now and finally decided to create a proper module for it as +I couldn't find an existing one (Haven't spent that much time searching though). ## Important notice As of right now, this is **Beta**, so treat it as such ;) -I added some unittests but there're still many more to go +I added some unittests but there're still more to go! ## ToDo Things to implement: @@ -26,106 +26,22 @@ What you need: * The great python [requests](https://github.com/kennethreitz/requests) module ## Installation -Just run: +### From pypi (recommanded) ```bash pip install lwe-pyloader ``` - +### From source +```bash +git clone https://github.com/linuxwhatelse/pyloader +cd pyloader +python setup.py install +``` ## Usage -The source has been commented quite well so at any point in time you might just: +The source has been commented quite well so at any point you might just: ```python import pyloader help(pyloader) ``` -But to get you started, here are some examples :) -```python -import pyloader - -def progress_callback(progress): - # !!!IMPORTANT NOTE!!! - # The callback will NOT be called within a separate thread - # (to ensure consistancy) and WILL block the download - # for as long as your callback runs! - # Think twice about what you do in here. - # Usually you just want to persist/visualize data. - - # Use ``help(pyloader.Progress)`` to know what's available - print(progress.percent) - - # To cancel the download - return True - # If you don't want to cancel, - return False - # or return nothing at all - -def url_resolver(item): - # Here you would resolve the url to something - # that can actually be downloaded. - # Useful in case resource-urls would expire after - # a set amount of time. - return item.url - -if __name__ == '__main__': - # Create a loader instance - dl = pyloader.Loader.get_loader() - dl.configure( - max_concurrent=3, - update_interval=3, - progress_cb=progress_callback, - url_resolve_cb=url_resolver, - daemon=False - ) - - # Start the loader - # Make sure you know how the `daemon` flag - # affects the liftime of your program - dl.start() - - # Create a downloadable item - # Make sure to read the docstrings via - # help(pyloader.DLable) - # for available arguments - item = pyloader.DLable( - url = 'http://download.thinkbroadband.com/5MB.zip', - target_dir = '~/Downloads/', - ) - - # Queue an item or... - dl.queue(item) - # ...alternatively you can force a download, - # ignoring ``max_concurrent`` - dl.download(item) - - # If you don't use a callback (to, if necessary, cancel a download), - # you can stop one via: - dl.stop(item) - # or via its uid - dl.stop(item.uid) - - # You can also clear the queue by - dl.clear_queue() - # and active items as well - dl.clear_active() - # though you'll never need the second one as there - # are little to none cases where items are actually - # stuck within the active queue - - # To check the downloaders state you can: - print(dl.is_alive) # True if both necessary main threads are still alive and kicking - print(dl.is_active) # True if items are queued and/or downloading - - print(dl.queued) # Returns the amount of queued items - print(dl.active) # Returns the amount of active/downloading items - - print(dl.max_concurrent) # The amount of maximum concurrent allowed downloads - dl.max_concurrent = 5 # Set the amount up to 5 and starts new downloads (if any) - - # To stop all downloads and end the downloader just do: - dl.exit() - - # Important to note here, queued/active items will **NOT** be persisted upon exit (or any other point in time) - # It's up to you to keep track of the items ;) -``` -Well, that should be enough to get you going (I hope) +To get you started though, check the included [examples](examples). Happy coding! :) diff --git a/examples/simple.py b/examples/simple.py new file mode 100644 index 0000000..34b654c --- /dev/null +++ b/examples/simple.py @@ -0,0 +1,104 @@ +import time +import pyloader + + +def progress_callback(progress): + """!!! IMPORTANT !!! + + This callback will NOT be called from a subsequent thread but will use + the downloads thread instead. + This means the download blocks for as long as the callback is executed. + + Usually you only want to persist/visualize the progress. + """ + # Use `help(pyloader.Progress)` to see all the goodies + print(progress.dlable.file_name, '{0:.2f}%'.format(progress.percent)) + + # `return True` if the download should be canceled + return False + + +def url_resolver(item): + """At times you may have urls that would expire while the item is queued. + + This callback will be called right before the download starts and allowes + you to alter the `DLable` instance (and the url that goes along with it). + """ + # item.url = 'http://new.url' + return item + + +if __name__ == '__main__': + # Create a loader instance + loader = pyloader.Loader.get_loader() + loader.configure( + max_concurrent=3, + update_interval=1, + progress_cb=progress_callback, + url_resolve_cb=url_resolver, + daemon=False + ) + + # Start the loader + # Make sure you know how the `daemon` flag + # affects the liftime of your program + loader.start() + + # Create downloadable items + item1 = pyloader.DLable( + url='http://download.thinkbroadband.com/5MB.zip', + target_dir='./', + file_name='item1.zip' + ) + + item2 = pyloader.DLable( + url='http://download.thinkbroadband.com/5MB.zip', + target_dir='./', + file_name='item2.zip' + ) + # Queue an item or... + loader.queue(item1) + + # ...alternatively you can force a download, + # ignoring `max_concurrent` + loader.download(item2) + + # If you don't use a callback (which would allow you to cancel a + # download if necessary), you can `stop` one too + loader.stop(item1) + # or via its uid + loader.stop(item1.uid) + + # You can also clear all queued items + loader.clear_queued() + + # True if both necessary main threads are still alive and kicking + print('Is alive:', loader.is_alive()) + + # True if items are queued and/or downloading + print('Is active:', loader.is_active()) + + # Amount of queued items + print('Queued items:', loader.queued) + + # Amount of active/downloading items + print('Active items:', loader.active) + + # The amount of maximum concurrent allowed downloads + print('Max concurrent:', loader.max_concurrent) + # Change the amount of max concurrent downloads + # Will also trigger new downloads if necessary + loader.max_concurrent = 5 + + # How often the progress callback will be called in seconds + print('Update interval:', loader.update_interval) + # Change the interval + loader.update_interval = 0.5 + + # Wait for all downloads to finish + while loader.is_active(): + time.sleep(0.25) + + # Exit downloader and stop all downloads + # `pyloader` does NOT persist anything. This is up to you! + loader.exit()