import unicodedata
import requests
import logging
+import multiprocessing
+import enum
from shutil import copyfile
from bs4 import BeautifulSoup
PER_PAGE_REGEX = re.compile(r'"per_page":(\d*),')
NO_WHITESPACE_REGEX = re.compile(r'[-\s]+')
-VERSION = "0.5.1"
+DOWNLOADER_COUNT = 1
+RETRY_COUNT = 3
+
+VERSION = "0.7.0"
+
+class State(enum.Enum):
+ OK = enum.auto()
+ FAILED = enum.auto()
+ ALREADY_DOWNLOADED = enum.auto()
+
def strip_ws(value):
""" Remove whitespace from a string """
'ascii', 'ignore').decode()
value = str(re.sub(r'[^\w\s-]', '', value).strip())
value = str(NO_WHITESPACE_REGEX.sub('-', value))
- #value = str(re.sub(r'[-\s]+', '-', value))
return value
+class Downloader(multiprocessing.Process):
+ """
+ Class to handle downloading the things we have found to get.
+ """
+
+ def __init__(self, thing_queue, download_directory):
+ multiprocessing.Process.__init__(self)
+ # TODO: add parameters
+ self.thing_queue = thing_queue
+ self.download_directory = download_directory
+
+ def run(self):
+ """ actual download loop.
+ """
+ while True:
+ thing_id = self.thing_queue.get()
+ if thing_id is None:
+ logging.info("Shutting download queue")
+ self.thing_queue.task_done()
+ break
+ logging.info("Handling id {}".format(thing_id))
+ Thing(thing_id).download(self.download_directory)
+ self.thing_queue.task_done()
+ return
+
+
+
+
class Grouping:
""" Holds details of a group of things for download
- use Collection or Designs instead.
"""
- def __init__(self):
+ def __init__(self, quick):
self.things = []
self.total = 0
self.req_id = None
self.last_page = 0
self.per_page = None
+ # Should we stop downloading when we hit a known datestamp?
+ self.quick = quick
# These should be set by child classes.
self.url = None
self.download_dir = None
logging.info("Downloading {} thing(s).".format(self.total))
for idx, thing in enumerate(self.things):
logging.info("Downloading thing {}".format(idx))
- Thing(thing).download(self.download_dir)
+ RC = Thing(thing).download(self.download_dir)
+ if self.quick and RC==State.ALREADY_DOWNLOADED:
+ logging.info("Caught up, stopping.")
+ return
class Collection(Grouping):
""" Holds details of a collection. """
- def __init__(self, user, name, directory):
- Grouping.__init__(self)
+ def __init__(self, user, name, directory, quick):
+ Grouping.__init__(self, quick)
self.user = user
self.name = name
self.url = "{}/{}/collections/{}".format(
class Designs(Grouping):
""" Holds details of all of a users' designs. """
- def __init__(self, user, directory):
- Grouping.__init__(self)
+ def __init__(self, user, directory, quick):
+ Grouping.__init__(self, quick)
self.user = user
self.url = "{}/{}/designs".format(URL_BASE, self.user)
self.download_dir = os.path.join(
"bad status code {} for thing {} - try again later?".format(req.status_code, self.thing_id))
return
- self.download_dir = os.path.join(base_dir, self.title)
+ self.old_download_dir = os.path.join(base_dir, self.title)
+ self.download_dir = os.path.join(base_dir, "{} - {}".format(self.thing_id, self.title))
logging.debug("Parsing {} ({})".format(self.thing_id, self.title))
if not os.path.exists(self.download_dir):
- # Not yet downloaded
- self._parsed = True
- return
+ if os.path.exists(self.old_download_dir):
+ logging.info("Found previous style download directory. Moving it")
+ copyfile(self.old_download_dir, self.download_dir)
+ else:
+ # Not yet downloaded
+ self._parsed = True
+ return
timestamp_file = os.path.join(self.download_dir, 'timestamp.txt')
if not os.path.exists(timestamp_file):
self._parsed = True
def download(self, base_dir):
- """ Download all files for a given thing. """
+ """ Download all files for a given thing.
+ Returns True iff the thing is now downloaded (not iff it downloads the thing!)
+ """
if not self._parsed:
self._parse(base_dir)
if not self._parsed:
logging.error(
"Unable to parse {} - aborting download".format(self.thing_id))
- return
+ return State.FAILED
if not self._needs_download:
- print("{} already downloaded - skipping.".format(self.title))
- return
+ print("{} - {} already downloaded - skipping.".format(self.thing_id, self.title))
+ return State.ALREADY_DOWNLOADED
# Have we already downloaded some things?
timestamp_file = os.path.join(self.download_dir, 'timestamp.txt')
target_dir = "{}_old_{}".format(self.download_dir, prev_count)
os.rename(self.download_dir, target_dir)
else:
- prev_dir = "{}_{}".format(self.download_dir, self.last_time)
+ prev_dir = "{}_{}".format(self.download_dir, slugify(self.last_time))
os.rename(self.download_dir, prev_dir)
# Get the list of files to download
except Exception as exception:
logging.error("Failed to download {} - {}".format(name, exception))
os.rename(self.download_dir, "{}_failed".format(self.download_dir))
- return
+ return State.FAILED
# People like images
image_dir = os.path.join(self.download_dir, 'images')
except Exception as exception:
print("Failed to download {} - {}".format(filename, exception))
os.rename(self.download_dir, "{}_failed".format(self.download_dir))
- return
+ return State.FAILED
# instructions are good too.
logging.info("Downloading readme")
except Exception as exception:
print("Failed to write timestamp file - {}".format(exception))
os.rename(self.download_dir, "{}_failed".format(self.download_dir))
- return
+ return State.FAILED
self._needs_download = False
logging.debug("Download of {} finished".format(self.title))
+ return State.OK
-def do_batch(batch_file, download_dir):
+def do_batch(batch_file, download_dir, quick):
""" Read a file in line by line, parsing each as a set of calls to this script."""
with open(batch_file) as handle:
for line in handle:
line = line.strip()
+ if not line:
+ # Skip empty lines
+ continue
logging.info("Handling instruction {}".format(line))
command_arr = line.split()
if command_arr[0] == "thing":
logging.debug(
"Handling batch collection instruction: {}".format(line))
Collection(command_arr[1], command_arr[2],
- download_dir).download()
+ download_dir, quick).download()
continue
if command_arr[0] == "user":
logging.debug(
"Handling batch collection instruction: {}".format(line))
- Designs(command_arr[1], download_dir).download()
+ Designs(command_arr[1], download_dir, quick).download()
continue
logging.warning("Unable to parse current instruction. Skipping.")
help="Target directory to download into")
parser.add_argument("-f", "--log-file",
help="Place to log debug information to")
+ parser.add_argument("-q", "--quick", action="store_true",
+ help="Assume date ordering on posts")
+
subparsers = parser.add_subparsers(
help="Type of thing to download", dest="subcommand")
collection_parser = subparsers.add_parser(
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
+
+ # Start downloader
+ thing_queue = multiprocessing.JoinableQueue()
+ logging.debug("starting {} downloader(s)".format(DOWNLOADER_COUNT))
+ downloaders = [Downloader(thing_queue, args.directory) for _ in range(DOWNLOADER_COUNT)]
+ for downloader in downloaders:
+ downloader.start()
+
+
if args.subcommand.startswith("collection"):
for collection in args.collections:
- Collection(args.owner, collection, args.directory).download()
+ Collection(args.owner, collection, args.directory, args.quick).download()
if args.subcommand == "thing":
for thing in args.things:
- Thing(thing).download(args.directory)
+ thing_queue.put(thing)
if args.subcommand == "user":
for user in args.users:
- Designs(user, args.directory).download()
+ Designs(user, args.directory, args.quick).download()
if args.subcommand == "version":
print("thingy_grabber.py version {}".format(VERSION))
if args.subcommand == "batch":
- do_batch(args.batch_file, args.directory)
+ do_batch(args.batch_file, args.directory, args.quick)
+ # Stop the downloader processes
+ for downloader in downloaders:
+ thing_queue.put(None)
if __name__ == "__main__":
main()
HCoop / clinton / thingy_grabber.git / blobdiff