From: Oliver Matthews <oliver@codersoffortune.net>
Date: Fri, 1 Nov 2019 18:00:14 +0000 (+0000)
Subject: move failed downloads sideways
X-Git-Tag: v0.2~4
X-Git-Url: http://git.hcoop.net/clinton/thingy_grabber.git/commitdiff_plain/a7152c35bfb21e27530cc5d491d26e7a0147ceeb

move failed downloads sideways
---

diff --git a/README.md b/README.md
index 11496a5..5687736 100644
--- a/README.md
+++ b/README.md
@@ -3,8 +3,14 @@ Script for archiving thingiverse things. Due to this being a glorified webscrape
 
 ## Usage:
 `thingy_grabber.py user_name collection_name`
+
 Where `user_name` is the name of the creator of the collection (not nes. your name!) and `collection_name` is the name of the collection you want.
 
+This will create a series of directorys `user-collection/thing-name` for each thing in the collection.
+If a thing's directory already exists, it will be skipped.
+
+If for some reason a download fails, it will get moved sideways to `thing-name-failed` - this way if you rerun it, it will only reattmpt any failed things.
+
 ## Requirements
 python3, beautifulsoup4, requests, lxml
 
@@ -15,4 +21,5 @@ python3, beautifulsoup4, requests, lxml
 - download a single thing
 - download things by designer
 - less perfunctory error checking / handling
-- resume failed things
+- attempt to use -failed dirs for resuming
+- detect updated models and redownload them
diff --git a/thingy_grabber.py b/thingy_grabber.py
index f3f9941..f2b57d8 100755
--- a/thingy_grabber.py
+++ b/thingy_grabber.py
@@ -125,10 +125,17 @@ def download_thing(thing):
     file_links = file_soup.find_all('a', {'class':'file-download'})
     files = [("{}{}".format(URL_BASE, x['href']), x["title"]) for x in file_links]
 
-    for url, name in files:
-        data_req = requests.get(url)
-        with open(name, 'wb') as handle:
-            handle.write(data_req.content)
+    try:
+        for url, name in files:
+            data_req = requests.get(url)
+            with open(name, 'wb') as handle:
+                handle.write(data_req.content)
+    except Exception as exception:
+        print("Failed to download {} - {}".format(name, exception))
+        os.chdir(base_dir)
+        os.rename(title, "{}_failed".format(title))
+        return
+
     os.chdir(base_dir)
 
 def main():