get readme & license details

diff --git a/README.md b/README.md
index 96bd041..2d9a103 100644 (file)
--- a/README.md
+++ b/README.md
@@ -75,6 +75,8 @@ python3, beautifulsoup4, requests, lxml
 - If there is an updated file, the old directory will be moved to `name_timestamp` where `timestamp` is the last upload time of the old files. The code will then copy unchanged files across and download any new ones.
 
 ## Changelog
+* v0.6.1
+  - now downloads readme.txt and licence details
 * v0.6.0
   - added support for downloading multiple things/design sets/collections from the command line
 * v0.5.0

diff --git a/thingy_grabber.py b/thingy_grabber.py
index 6566187..63c929a 100755 (executable)
--- a/thingy_grabber.py
+++ b/thingy_grabber.py
@@ -340,6 +340,30 @@ class Thing:
             os.rename(self.download_dir, "{}_failed".format(self.download_dir))
             return
 
+        # instructions are good too.
+        logging.info("Downloading readme")
+        try:
+            readme_txt = soup.find('meta', property='og:description')['content']
+            with open(os.path.join(self.download_dir,'readme.txt'), 'w') as readme_handle:
+                readme_handle.write("{}\n".format(readme_txt))
+        except (TypeError, KeyError) as exception:
+            logging.warning("No readme? {}".format(exception))
+        except IOError as exception:
+            logging.warning("Failed to write readme! {}".format(exception))
+
+        # Best get some licenses
+        logging.info("Downloading license")
+        try:
+            license_txt = soup.find('div',{'class':'license-text'}).text
+            if license_txt:
+                with open(os.path.join(self.download_dir,'license.txt'), 'w') as license_handle:
+                    license_handle.write("{}\n".format(license_txt))
+        except AttributeError as exception:
+            logging.warning("No license? {}".format(exception))
+        except IOError as exception:
+            logging.warning("Failed to write license! {}".format(exception))
+
+
         try:
             # Now write the timestamp
             with open(timestamp_file, 'w') as timestamp_handle: