thingy_grabber.py

   1 #!/usr/bin/env python3
   2 """
   3 Thingiverse bulk downloader
   4 """
   5
   6 import re
   7 import sys
   8 import os
   9 import argparse
  10 import unicodedata
  11 import requests
  12 import logging
  13 from shutil import copyfile
  14 from bs4 import BeautifulSoup
  15
  16 URL_BASE = "https://www.thingiverse.com"
  17 URL_COLLECTION = URL_BASE + "/ajax/thingcollection/list_collected_things"
  18 USER_COLLECTION = URL_BASE + "/ajax/user/designs"
  19
  20 ID_REGEX = re.compile(r'"id":(\d*),')
  21 TOTAL_REGEX = re.compile(r'"total":(\d*),')
  22 LAST_PAGE_REGEX = re.compile(r'"last_page":(\d*),')
  23 # This appears to be fixed at 12, but if it changes would screw the rest up.
  24 PER_PAGE_REGEX = re.compile(r'"per_page":(\d*),')
  25 NO_WHITESPACE_REGEX = re.compile(r'[-\s]+')
  26
  27 VERSION = "0.5.1"
  28
  29 def strip_ws(value):
  30     """ Remove whitespace from a string """
  31     return str(NO_WHITESPACE_REGEX.sub('-', value))
  32
  33
  34 def slugify(value):
  35     """
  36     Normalizes string, converts to lowercase, removes non-alpha characters,
  37     and converts spaces to hyphens.
  38     """
  39     value = unicodedata.normalize('NFKD', value).encode(
  40         'ascii', 'ignore').decode()
  41     value = str(re.sub(r'[^\w\s-]', '', value).strip())
  42     value = str(NO_WHITESPACE_REGEX.sub('-', value))
  43     #value = str(re.sub(r'[-\s]+', '-', value))
  44     return value
  45
  46
  47 class Grouping:
  48     """ Holds details of a group of things for download
  49         This is effectively (although not actually) an abstract class
  50         - use Collection or Designs instead.
  51     """
  52
  53     def __init__(self):
  54         self.things = []
  55         self.total = 0
  56         self.req_id = None
  57         self.last_page = 0
  58         self.per_page = None
  59         # These should be set by child classes.
  60         self.url = None
  61         self.download_dir = None
  62         self.collection_url = None
  63
  64     def _get_small_grouping(self, req):
  65         """ Handle small groupings """
  66         soup = BeautifulSoup(req.text, features='lxml')
  67         links = soup.find_all('a', {'class': 'card-img-holder'})
  68         self.things = [x['href'].split(':')[1] for x in links]
  69         self.total = len(self.things)
  70
  71         return self.things
  72
  73     def get(self):
  74         """ retrieve the things of the grouping. """
  75         if self.things:
  76             # We've already done it.
  77             return self.things
  78
  79         # Check for initialisation:
  80         if not self.url:
  81             logging.error("No URL set - object not initialised properly?")
  82             raise ValueError("No URL set - object not initialised properly?")
  83
  84         # Get the internal details of the grouping.
  85         logging.debug("Querying {}".format(self.url))
  86         c_req = requests.get(self.url)
  87         total = TOTAL_REGEX.search(c_req.text)
  88         if total is None:
  89             # This is a small (<13) items grouping. Pull the list from this req.
  90             return self._get_small_grouping(c_req)
  91         self.total = total.groups()[0]
  92         self.req_id = ID_REGEX.search(c_req.text).groups()[0]
  93         self.last_page = int(LAST_PAGE_REGEX.search(c_req.text).groups()[0])
  94         self.per_page = PER_PAGE_REGEX.search(c_req.text).groups()[0]
  95         parameters = {
  96             'base_url': self.url,
  97             'page': '1',
  98             'per_page': '12',
  99             'id': self.req_id
 100         }
 101         for current_page in range(1, self.last_page + 1):
 102             parameters['page'] = current_page
 103             req = requests.post(self.collection_url, parameters)
 104             soup = BeautifulSoup(req.text, features='lxml')
 105             links = soup.find_all('a', {'class': 'card-img-holder'})
 106             self.things += [x['href'].split(':')[1] for x in links]
 107
 108         return self.things
 109
 110     def download(self):
 111         """ Downloads all the files in a collection """
 112         if not self.things:
 113             self.get()
 114
 115         if not self.download_dir:
 116             raise ValueError(
 117                 "No download_dir set - invalidly initialised object?")
 118
 119         base_dir = os.getcwd()
 120         try:
 121             os.mkdir(self.download_dir)
 122         except FileExistsError:
 123             logging.info("Target directory {} already exists. Assuming a resume."
 124                          .format(self.download_dir))
 125         logging.info("Downloading {} thing(s).".format(self.total))
 126         for idx, thing in enumerate(self.things):
 127             logging.info("Downloading thing {}".format(idx))
 128             Thing(thing).download(self.download_dir)
 129
 130
 131 class Collection(Grouping):
 132     """ Holds details of a collection. """
 133
 134     def __init__(self, user, name, directory):
 135         Grouping.__init__(self)
 136         self.user = user
 137         self.name = name
 138         self.url = "{}/{}/collections/{}".format(
 139             URL_BASE, self.user, strip_ws(self.name))
 140         self.download_dir = os.path.join(directory,
 141                                          "{}-{}".format(slugify(self.user), slugify(self.name)))
 142         self.collection_url = URL_COLLECTION
 143
 144
 145 class Designs(Grouping):
 146     """ Holds details of all of a users' designs. """
 147
 148     def __init__(self, user, directory):
 149         Grouping.__init__(self)
 150         self.user = user
 151         self.url = "{}/{}/designs".format(URL_BASE, self.user)
 152         self.download_dir = os.path.join(
 153             directory, "{} designs".format(slugify(self.user)))
 154         self.collection_url = USER_COLLECTION
 155
 156
 157 class Thing:
 158     """ An individual design on thingiverse. """
 159
 160     def __init__(self, thing_id):
 161         self.thing_id = thing_id
 162         self.last_time = None
 163         self._parsed = False
 164         self._needs_download = True
 165         self.text = None
 166         self.title = None
 167         self.download_dir = None
 168
 169     def _parse(self, base_dir):
 170         """ Work out what, if anything needs to be done. """
 171         if self._parsed:
 172             return
 173
 174         url = "{}/thing:{}/files".format(URL_BASE, self.thing_id)
 175         try:
 176           req = requests.get(url)
 177         except requests.exceptions.ConnectionError as error:
 178           logging.error("Unable to connect for thing {}: {}".format(self.thing_id, error))
 179           return
 180
 181         self.text = req.text
 182         soup = BeautifulSoup(self.text, features='lxml')
 183         #import code
 184         #code.interact(local=dict(globals(), **locals()))
 185         try:
 186           self.title = slugify(soup.find_all('h1')[0].text.strip())
 187         except IndexError:
 188           logging.warning("No title found for thing {}".format(self.thing_id))
 189           self.title = self.thing_id
 190
 191         if req.status_code == 404:
 192           logging.warning("404 for thing {} - DMCA or invalid number?".format(self.thing_id))
 193           return
 194
 195         if req.status_code > 299:
 196           logging.warning("bad status code {}  for thing {} - try again later?".format(req.status_code, self.thing_id))
 197           return
 198
 199         self.download_dir = os.path.join(base_dir, self.title)
 200
 201         logging.debug("Parsing {} ({})".format(self.thing_id, self.title))
 202
 203         if not os.path.exists(self.download_dir):
 204             # Not yet downloaded
 205             self._parsed = True
 206             return
 207
 208         timestamp_file = os.path.join(self.download_dir, 'timestamp.txt')
 209         if not os.path.exists(timestamp_file):
 210             # Old download from before
 211             logging.warning(
 212                 "Old-style download directory found. Assuming update required.")
 213             self._parsed = True
 214             return
 215
 216         try:
 217             with open(timestamp_file, 'r') as timestamp_handle:
 218                 self.last_time = timestamp_handle.readlines()[0]
 219             logging.info("last downloaded version: {}".format(self.last_time))
 220         except FileNotFoundError:
 221             # Not run on this thing before.
 222             logging.info(
 223                 "Old-style download directory found. Assuming update required.")
 224             self.last_time = None
 225             self._parsed = True
 226             return
 227
 228         # OK, so we have a timestamp, lets see if there is anything new to get
 229         file_links = soup.find_all('a', {'class': 'file-download'})
 230         for file_link in file_links:
 231             timestamp = file_link.find_all('time')[0]['datetime']
 232             logging.debug("Checking {} (updated {})".format(
 233                 file_link["title"], timestamp))
 234             if timestamp > self.last_time:
 235                 logging.info(
 236                     "Found new/updated file {}".format(file_link["title"]))
 237                 self._needs_download = True
 238                 self._parsed = True
 239                 return
 240         # Got here, so nope, no new files.
 241         self._needs_download = False
 242         self._parsed = True
 243
 244     def download(self, base_dir):
 245         """ Download all files for a given thing. """
 246         if not self._parsed:
 247             self._parse(base_dir)
 248
 249         if not self._parsed:
 250           logging.error("Unable to parse {} - aborting download".format(self.thing_id))
 251           return
 252
 253         if not self._needs_download:
 254             print("{} already downloaded - skipping.".format(self.title))
 255             return
 256
 257         # Have we already downloaded some things?
 258         timestamp_file = os.path.join(self.download_dir, 'timestamp.txt')
 259         prev_dir = None
 260         if os.path.exists(self.download_dir):
 261             if not os.path.exists(timestamp_file):
 262                 # edge case: old style dir w/out timestamp.
 263                 logging.warning(
 264                     "Old style download dir found for {}".format(self.title))
 265                 prev_count = 0
 266                 target_dir = "{}_old".format(self.download_dir)
 267                 while os.path.exists(target_dir):
 268                     prev_count = prev_count + 1
 269                     target_dir = "{}_old_{}".format(self.download_dir, prev_count)
 270                 os.rename(self.download_dir, target_dir)
 271             else:
 272                 prev_dir = "{}_{}".format(self.download_dir, self.last_time)
 273                 os.rename(self.download_dir, prev_dir)
 274
 275         # Get the list of files to download
 276         soup = BeautifulSoup(self.text, features='lxml')
 277         file_links = soup.find_all('a', {'class': 'file-download'})
 278
 279         new_file_links = []
 280         old_file_links = []
 281         new_last_time = None
 282
 283         if not self.last_time:
 284             # If we don't have anything to copy from, then it is all new.
 285             new_file_links = file_links
 286             try:
 287               new_last_time = file_links[0].find_all('time')[0]['datetime']
 288             except:
 289               import code
 290               code.interact(local=dict(globals(), **locals()))
 291
 292             for file_link in file_links:
 293                 timestamp = file_link.find_all('time')[0]['datetime']
 294                 logging.debug("Found file {} from {}".format(
 295                     file_link["title"], timestamp))
 296                 if timestamp > new_last_time:
 297                     new_last_time = timestamp
 298         else:
 299             for file_link in file_links:
 300                 timestamp = file_link.find_all('time')[0]['datetime']
 301                 logging.debug("Checking {} (updated {})".format(
 302                     file_link["title"], timestamp))
 303                 if timestamp > self.last_time:
 304                     new_file_links.append(file_link)
 305                 else:
 306                     old_file_links.append(file_link)
 307                 if not new_last_time or timestamp > new_last_time:
 308                     new_last_time = timestamp
 309
 310         logging.debug("new timestamp {}".format(new_last_time))
 311
 312         # OK. Time to get to work.
 313         logging.debug("Generating download_dir")
 314         os.mkdir(self.download_dir)
 315         # First grab the cached files (if any)
 316         logging.info("Copying {} unchanged files.".format(len(old_file_links)))
 317         for file_link in old_file_links:
 318             old_file = os.path.join(prev_dir, file_link["title"])
 319             new_file = os.path.join(self.download_dir, file_link["title"])
 320             try:
 321                 logging.debug("Copying {} to {}".format(old_file, new_file))
 322                 copyfile(old_file, new_file)
 323             except FileNotFoundError:
 324                 logging.warning(
 325                     "Unable to find {} in old archive, redownloading".format(file_link["title"]))
 326                 new_file_links.append(file_link)
 327
 328         # Now download the new ones
 329         files = [("{}{}".format(URL_BASE, x['href']), x["title"])
 330                  for x in new_file_links]
 331         logging.info("Downloading {} new files of {}".format(
 332             len(new_file_links), len(file_links)))
 333         try:
 334             for url, name in files:
 335                 file_name = os.path.join(self.download_dir, name)
 336                 logging.debug("Downloading {} from {} to {}".format(
 337                     name, url, file_name))
 338                 data_req = requests.get(url)
 339                 with open(file_name, 'wb') as handle:
 340                     handle.write(data_req.content)
 341         except Exception as exception:
 342             logging.error("Failed to download {} - {}".format(name, exception))
 343             os.rename(self.download_dir, "{}_failed".format(self.download_dir))
 344             return
 345
 346         # People like images
 347         image_dir = os.path.join(self.download_dir, 'images')
 348         imagelinks = soup.find_all('span', {'class': 'gallery-slider'})[0] \
 349                          .find_all('div', {'class': 'gallery-photo'})
 350         logging.info("Downloading {} images.".format(len(imagelinks)))
 351         try:
 352             os.mkdir(image_dir)
 353             for imagelink in imagelinks:
 354                 url = next(filter(None,[imagelink[x] for x in ['data-full',
 355                                                                'data-large',
 356                                                                'data-medium',
 357                                                                'data-thumb']]), None)
 358                 if not url:
 359                     logging.warning("Unable to find any urls for {}".format(imagelink))
 360                     continue
 361
 362                 filename = os.path.basename(url)
 363                 if filename.endswith('stl'):
 364                     filename = "{}.png".format(filename)
 365                 image_req = requests.get(url)
 366                 with open(os.path.join(image_dir, filename), 'wb') as handle:
 367                     handle.write(image_req.content)
 368         except Exception as exception:
 369             print("Failed to download {} - {}".format(filename, exception))
 370             os.rename(self.download_dir, "{}_failed".format(self.download_dir))
 371             return
 372
 373         # instructions are good too.
 374         logging.info("Downloading readme")
 375         try:
 376             readme_txt = soup.find('meta', property='og:description')['content']
 377             with open(os.path.join(self.download_dir,'readme.txt'), 'w') as readme_handle:
 378                 readme_handle.write("{}\n".format(readme_txt))
 379         except (TypeError, KeyError) as exception:
 380             logging.warning("No readme? {}".format(exception))
 381         except IOError as exception:
 382             logging.warning("Failed to write readme! {}".format(exception))
 383
 384         # Best get some licenses
 385         logging.info("Downloading license")
 386         try:
 387             license_txt = soup.find('div',{'class':'license-text'}).text
 388             if license_txt:
 389                 with open(os.path.join(self.download_dir,'license.txt'), 'w') as license_handle:
 390                     license_handle.write("{}\n".format(license_txt))
 391         except AttributeError as exception:
 392             logging.warning("No license? {}".format(exception))
 393         except IOError as exception:
 394             logging.warning("Failed to write license! {}".format(exception))
 395
 396
 397         try:
 398             # Now write the timestamp
 399             with open(timestamp_file, 'w') as timestamp_handle:
 400                 timestamp_handle.write(new_last_time)
 401         except Exception as exception:
 402             print("Failed to write timestamp file - {}".format(exception))
 403             os.rename(self.download_dir, "{}_failed".format(self.download_dir))
 404             return
 405         self._needs_download = False
 406         logging.debug("Download of {} finished".format(self.title))
 407
 408
 409 def do_batch(batch_file, download_dir):
 410     """ Read a file in line by line, parsing each as a set of calls to this script."""
 411     with open(batch_file) as handle:
 412         for line in handle:
 413             line = line.strip()
 414             logging.info("Handling instruction {}".format(line))
 415             command_arr = line.split()
 416             if command_arr[0] == "thing":
 417                 logging.debug(
 418                     "Handling batch thing instruction: {}".format(line))
 419                 Thing(command_arr[1]).download(download_dir)
 420                 continue
 421             if command_arr[0] == "collection":
 422                 logging.debug(
 423                     "Handling batch collection instruction: {}".format(line))
 424                 Collection(command_arr[1], command_arr[2],
 425                            download_dir).download()
 426                 continue
 427             if command_arr[0] == "user":
 428                 logging.debug(
 429                     "Handling batch collection instruction: {}".format(line))
 430                 Designs(command_arr[1], download_dir).download()
 431                 continue
 432             logging.warning("Unable to parse current instruction. Skipping.")
 433
 434
 435 def main():
 436     """ Entry point for script being run as a command. """
 437     parser = argparse.ArgumentParser()
 438     parser.add_argument("-l", "--log-level", choices=[
 439                         'debug', 'info', 'warning'], default='info', help="level of logging desired")
 440     parser.add_argument("-d", "--directory",
 441                         help="Target directory to download into")
 442     parser.add_argument("-f", "--log-file",
 443                         help="Place to log debug information to")
 444     subparsers = parser.add_subparsers(
 445         help="Type of thing to download", dest="subcommand")
 446     collection_parser = subparsers.add_parser(
 447         'collection', help="Download one or more entire collection(s)")
 448     collection_parser.add_argument(
 449         "owner", help="The owner of the collection(s) to get")
 450     collection_parser.add_argument(
 451         "collections", nargs="+",  help="Space seperated list of the name(s) of collection to get")
 452     thing_parser = subparsers.add_parser(
 453         'thing', help="Download a single thing.")
 454     thing_parser.add_argument("things", nargs="*", help="Space seperated list of thing ID(s) to download")
 455     user_parser = subparsers.add_parser(
 456         "user",  help="Download all things by one or more users")
 457     user_parser.add_argument("users", nargs="+", help="A space seperated list of the user(s) to get the designs of")
 458     batch_parser = subparsers.add_parser(
 459         "batch", help="Perform multiple actions written in a text file")
 460     batch_parser.add_argument(
 461         "batch_file", help="The name of the file to read.")
 462     subparsers.add_parser("version", help="Show the current version")
 463
 464     args = parser.parse_args()
 465     if not args.subcommand:
 466         parser.print_help()
 467         sys.exit(1)
 468     if not args.directory:
 469         args.directory = os.getcwd()
 470
 471     logger = logging.getLogger()
 472     formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 473     logger.setLevel(logging.DEBUG)
 474     console_handler = logging.StreamHandler()
 475     console_handler.setLevel(args.log_level.upper())
 476
 477     logger.addHandler(console_handler)
 478     if args.log_file:
 479         file_handler = logging.FileHandler(args.log_file)
 480         file_handler.setLevel(logging.DEBUG)
 481         file_handler.setFormatter(formatter)
 482         logger.addHandler(file_handler)
 483
 484     if args.subcommand.startswith("collection"):
 485         for collection in args.collections:
 486             Collection(args.owner, collection, args.directory).download()
 487     if args.subcommand == "thing":
 488         for thing in args.things:
 489             Thing(thing).download(args.directory)
 490     if args.subcommand == "user":
 491         for user in args.users:
 492             Designs(user, args.directory).download()
 493     if args.subcommand == "version":
 494         print("thingy_grabber.py version {}".format(VERSION))
 495     if args.subcommand == "batch":
 496         do_batch(args.batch_file, args.directory)
 497
 498
 499 if __name__ == "__main__":
 500     main()