gistfile1.py

import logging
import os
import requests

logging.basicConfig(level=logging.DEBUG)

class DownloadSession(requests.Session):
    buffer_size = 1024 * 1024
    def __init__(self, user, password):
        super(DownloadSession, self).__init__()
        self.auth = (user, password)
        self.verify = False
        self.stream = True

    def _stream_helper(self, response):
        socket_fp = response.raw._fp.fp.raw._sock
        socket_fp.setblocking(1)
        socket = socket_fp.makefile('rb')
        while True:
            data = socket.read(self.buffer_size)
            if not data:
                break
            yield data

    def download_to(self, remote, local):
        if os.path.isfile(local):
            logging.warn('%s already exists, skipping...' % local)
            return
        with open(local, 'wb') as f:
            resp = self.get(remote)
            resp.raise_for_status()
            length = int(resp.headers.get('content-length', 0))
            dl_count = 0
            ffd = f.fileno()
            logging.info('Downloading %s...' % local)
            for buf in self._stream_helper(resp):
                f.write(buf)
                dl_count += len(buf)
                logging.info('Status: %d%%' % round(dl_count / length * 100, 1))
                f.flush()
                os.fsync(ffd)
            logging.info('Download complete: %s' % local)

    def download_all(self, remotes, local_folder):
        for remote in remotes:
            filename = remote.split('/')[-1]
            local = os.path.join(local_folder, filename)
            self.download_to(remote, local)

    def get_download_list(self, location):
        resp = self.get(location, stream=False)
        resp.raise_for_status()
        return [line for line in resp.text.split('\n')]


class BuzzDownloader(DownloadSession):
    base_url = 'https://buzz.3on.fr/'
    basket_url = base_url + 'basket/list'
    
    def get_download_list(self, location=None):
        if location is None:
            location = self.basket_url
        return super(BuzzDownloader, self).get_download_list(location)

    def download_basket(self, local_folder='.'):
        self.download_all(self.get_download_list(), local_folder)

if __name__ == '__main__':
    bd = BuzzDownloader('shin', 'XXXXXXXXXXXXXXXXX')
    bd.download_basket('.')