#~ Script to DL all sinfest webcomics
#~ one can edit the START and END parameters to DL a subset
STARTDAY = 17  # 17
STARTMONTH = 01  # 01
STARTYEAR = 2011  # 2000
ENDDAY = 31
ENDMONTH = 01
ENDYEAR = 2012
ROOTDIR = 'Sinfest'
ROOTURL = "http://www.sinfest.net/comikaze/comics/"
IMGEXT = ".gif"
#~ Code start - no edition from here unless you know what you're doing
#~ --------------------
from distutils.file_util import copy_file
from urllib import urlopen
import os.path
import os
#~ ROOTDIR=os.getcwd()+ROOTDIR
#~ Check if date is out of range
 
 
def checkdate(d, m, y, ed, em, ey, sd, sm, sy):  # end date is ed, em, ey
    if y > ey or (y == ey and m > em) or (y == ey and m == em and d > ed):
        return 0  # date over date range
    elif y < sy or (y == sy and m < sm) or (y == sy and m == sm and d < sd):
        return 0
    else:
        return 1  # date below end date and above start date
#~ make the file name teh sae format than sinfest server
 
 
def getfilename(y, m, d, IMGEXT):
    M =`m`
    D =`d`
    Y =`y`
    if len(M) < 2:
        M = '0' + M
    if len(D) < 2:
        D = '0' + D
    filename = Y + '-' + M + '-' + D + IMGEXT
    return filename
#~ run the loop
if not os.path.isdir(ROOTDIR):
    os.mkdir(ROOTDIR)
for y in range(STARTYEAR, ENDYEAR + 1):
    for m in range(1, 13):
        for d in range(1, 32):
            filename = getfilename(y, m, d, IMGEXT)
            if checkdate(d, m, y, ENDDAY, ENDMONTH, ENDYEAR,
                         STARTDAY, STARTMONTH, STARTYEAR):
                if os.path.isfile(ROOTDIR + '/' + filename):
                    print '[' + filename + '] already exists'
                else:
                    src = urlopen(ROOTURL + filename)
                    if src.info().gettype() == 'image/gif':
                        dst = open(ROOTDIR + '/' + filename, 'wb')
                        dst.write(src.read())
                        dst.close()
                        print '[' + filename + ']' + ' copied'
                    else:
                        print "MIMETYPE ERROR (probably ERROR 404) ignored"
                    src.close()

# -*- coding: utf-8 -*-
#! /usr/bin/python
# Script to DL all sinfest webcomics
# one can edit the START and END parameters to DL a subset
#-------------------------------------------------------------------------------
# Name:        getSinfest
# Purpose:
#
# Author:      Atrament
#
# Created:     12/04/2013
# Copyright:   (c) Atrament 2013
# Licence:     
#-------------------------------------------------------------------------------
 
import datetime
from urllib.request import urlopen
import os.path
import os
from urllib.error import HTTPError
 
 
def main():
    debut=datetime.date(2000,1,17)
    curGif=debut
    if not os.path.isdir("Sinfest"):
        os.mkdir("Sinfest")
    while curGif<=datetime.date.today():
        if not os.path.isfile("Sinfest/"+curGif.isoformat()+".gif"):
            try:
                src=urlopen("http://www.sinfest.net/comikaze/comics/"+curGif.isoformat()+".gif")
                dst=open("Sinfest/"+curGif.isoformat()+".gif",'wb')
                dst.write(src.read())
                dst.close()
                print("    "+curGif.isoformat()+".gif : fetched.")
                src.close()
            except HTTPError as e:
                print("on "+curGif.isoformat()+" error happened. So is life.")
        else:
            print(curGif.isoformat()+" : ok")
        curGif+=datetime.timedelta(days=1)
 
if __name__ == '__main__':
    main()

# -*- coding: UTF-8 -*-
# ---------------------------------------------------------------------
# Author:      Atrament
# Licence:     CC-BY-SA https://creativecommons.org/licenses/by-sa/4.0/
# ---------------------------------------------------------------------
# All libs are part of standard distribution for python 3.4
import imghdr
import os
import queue
from threading import Thread
import datetime
from urllib.error import HTTPError, URLError
from urllib.request import urlopen
import zipfile
import sys
 
 
# Useful functions
def make_cbz(directory):
    for year in range(2000, datetime.date.today().year + 1):
        with zipfile.ZipFile("Sinfest-{}.cbz".format(year), "w") as archive:
            for gif_file in [x for x in os.listdir(directory) if x.split("-")[0] == str(year)]:
                archive.write(directory + '/' + gif_file, arcname=gif_file)
        print("Sinfest-{}.cbz has been generated".format(year))
 
 
def confirm(prompt):
    if input(prompt + " (y/n)") in "yY":
        return True
    else:
        return False
 
 
def file_needs_download(filename):
    """Checks whether a file exists, is corrupt, so has to be downloaded again
    also cleans garbage if detected"""
    if not os.path.isfile(filename):
        # many comics are *supposed* to be missing,
        # no need to output for these (uncomment for debug)
        # print("IS NOT FILE :", filename)
        return True
    elif os.path.getsize(filename) == 0:
        print("WRONG SIZE for", filename)
        return True
    elif filename.split(".")[-1] != "gif":
        print(filename, "IS NOT GIF")
        return True
    elif filename.split(".")[-1] in {"jpg", "gif", "png"} and imghdr.what(filename) is None:
        # Encoding error...
        print("WRONG FILE STRUCTURE for", filename)
        return True
    else:
        return False
 
 
def conditional_download(filename, base_url):
    if file_needs_download(filename):
        try:
            src = urlopen(base_url + filename)
            dst = open(filename, 'wb')
            dst.write(src.read())
            # gracefully close theses accesses.
            dst.close()
            src.close()
            print("\t" + filename + " : fetched.")
        except HTTPError:
            # many days do not have a comic published.
            # no need to flood the console for this.
            pass
        except URLError:
            pass
            print("network error on " + filename)
        finally:
            # clean garbage on disk, useful if failure occurred.
            file_needs_download(filename)
 
 
class ThreadedWorker():
    def __init__(self, function=None, number_of_threads=8):
        self.queue = queue.Queue()
 
        def func():
            while True:
                item = self.queue.get()
                if function:
                    function(item)
                else:
                    print(item, "is being processed.")
                self.queue.task_done()
 
        self.function = func
 
        for i in range(number_of_threads):
            t = Thread(target=self.function, name="Thread-{:03}".format(i))
            t.daemon = True
            t.start()
 
    def put(self, object_to_queue):
        self.queue.put(object_to_queue)
 
    def join(self):
        self.queue.join()
 
    def feed(self, iterator):
        for task in iterator:
            self.queue.put(task)
 
 
def download_sinfest(target_folder):
    """
    Creates a directory and fetches Sinfest comics to populate it in full.
    """
    if not os.path.isdir(target_folder):
        os.makedirs(target_folder)
    os.chdir(target_folder)
 
    f = lambda filename: conditional_download(filename, "http://www.sinfest.net/btphp/comics/")
    # Make a worker with this function and run it
    t = ThreadedWorker(function=f, number_of_threads=20)
    # structure of comprehended list is a bit complex to generate all file names
    files = [(datetime.date(2000, 1, 17) + datetime.timedelta(days=x)).isoformat() + ".gif"
             for x in range((datetime.date.today() - datetime.date(2000, 1, 17)).days + 1)]
    t.feed(files)
    t.join()
 
 
if __name__ == "__main__":
    if any(("y", "Y", "-y", "-Y" in sys.argv)):
        folder = os.path.expanduser("~/Pictures/Sinfest/").replace("\\", "/")
        print("\nproceeding to download...")
        download_sinfest(folder)
        print("\ngenerating comic book files (.cbz)...")
        os.chdir(folder)
        os.chdir("..")
        make_cbz(folder)
    else:
        folder = os.path.expanduser("~/Pictures/Sinfest/").replace("\\", "/")
        while not confirm("Target to downloads is {} ?".format(folder)):
            folder = input("Please enter new folder (N to abort) :")
            if folder in "nN":
                exit(0)
        if confirm("Proceed to download ?"):
            download_sinfest(folder)
        if confirm("Do you want to generate cbz (comic books) files ?"):
            os.chdir(folder)
            os.chdir("..")
            make_cbz(folder)
        input("Finished. Please press Enter")

#! /usr/bin/env python3.5
import imghdr
import os
import datetime
import zipfile
from concurrent.futures import ThreadPoolExecutor
 
import requests
import begin
 
 
# Useful functions
def make_cbz(dst_directory, src_directory):
    for year in range(2000, datetime.date.today().year + 1):
        with zipfile.ZipFile("Sinfest-{}.cbz".format(year), "w") as archive:
            for filename in os.listdir(src_directory):
                if filename.startswith(str(year)):
                    archive.write(src_directory + "/" + filename, arcname=filename)
        print("Sinfest-{}.cbz has been generated".format(year))
 
 
def file_needs_download(filename):
    """Checks whether a file exists, is corrupt, so has to be downloaded again
    also cleans garbage if detected"""
    if not os.path.isfile(filename):
        # many comics are *supposed* to be missing,
        # no need to output for these (uncomment for debug)
        # print("IS NOT FILE :", filename)
        return True
    elif os.path.getsize(filename) == 0:
        print("WRONG SIZE for", filename)
        return True
    elif filename.split(".")[-1] != "gif":
        print(filename, "IS NOT GIF")
        return True
    elif filename.split(".")[-1] in {"jpg", "gif", "png"} and imghdr.what(filename) is None:
        # Encoding error...
        print("WRONG FILE STRUCTURE for", filename)
        return True
    else:
        return False
 
 
def conditional_download(filename, base_url, caller=None):
    if file_needs_download(filename):
        src = requests.get(base_url + filename)
        # manage failure to download
        if src.status_code == 404:
            src.close()
            return  # ignore it, that file is simply missing.
        if src.status_code != 200:  # an error other than 404 occurred
            # print("Error {} on {}".format(src.status_code, filename))
            src.close()
            if caller:  # retry that file later
                caller.submit(conditional_download, filename, base_url, caller)
        # actually copy that file
        dst = open(filename, 'wb')
        dst.write(src.content)
        # gracefully close theses accesses.
        dst.close()
        src.close()
        print("\t{} : fetched.".format(filename))
 
 
def download_sinfest(target_folder):
    """
    Source function for the process
    Creates a directory and fetches Sinfest comics to populate it in full.
    """
    if not os.path.isdir(target_folder):
        os.makedirs(target_folder)
    os.chdir(target_folder)
 
    with ThreadPoolExecutor(max_workers=64) as executor:
        for file in ("".join([(datetime.date(2000, 1, 17) + datetime.timedelta(days=x)).isoformat(), ".gif"])
                     for x in range((datetime.date.today() - datetime.date(2000, 1, 17)).days + 1)):
            executor.submit(conditional_download, file, "http://www.sinfest.net/btphp/comics/", executor)
 
 
@begin.start
def run(path: "folder in which the comics must be downloaded" = os.path.expanduser("~/Sinfest/"),
        makecbz: "Compile CBZ comic book archives" = False):
    """Download the Sinfest WebComics"""
    download_sinfest(path)
    if makecbz:
        dst = path[:-1] if path.endswith('/') else path
        dst = "/".join(dst.split("/")[:-1])
        make_cbz(dst, path)
    print("Finished. Goodbye.")
    exit()