super().__init__()
self.reset()
self.strict = False
- self.convert_charrefs= True
+ self.convert_charrefs = True
self.text = StringIO()
def handle_data(self, d):
s.feed(html)
return s.get_data()
+
@dataclass
class ThingLink:
thing_id: str
self.url = None
self.download_dir = None
-
@property
def get(self):
""" retrieve the things of the grouping. """
raise ValueError(
"No download_dir set - invalidly initialised object?")
- base_dir = os.getcwd()
try:
os.mkdir(self.download_dir)
except FileExistsError:
"""
return Thing(ThingLink(thing_id, "", ""))
-
def _parse(self, base_dir, api_key):
""" Work out what, if anything needs to be done. """
if self._parsed:
except KeyError:
logging.warning("No description found for thing {}?".format(self.thing_id))
-
if details:
try:
self._details = MLStripper.strip_tags(details)
logging.warning("Unable to strip HTML from readme: {}".format(e))
self._details = details
-
if not self.name:
# Probably generated with factory method.
try:
return State.OK
-def do_batch(batch_file, download_dir, quick, compress):
+def do_batch(batch_file, download_dir, quick, compress, api_key):
""" Read a file in line by line, parsing each as a set of calls to this script."""
with open(batch_file) as handle:
for line in handle:
if command_arr[0] == "thing":
logging.debug(
"Handling batch thing instruction: {}".format(line))
- Thing.from_thing_id(command_arr[1]).download(download_dir, compress)
+ Thing.from_thing_id(command_arr[1]).download(download_dir, compress, api_key)
continue
if command_arr[0] == "collection":
logging.debug(
"Handling batch collection instruction: {}".format(line))
Collection(command_arr[1], command_arr[2],
- download_dir, quick, compress).download()
+ download_dir, quick, compress, api_key).download()
continue
if command_arr[0] == "user":
logging.debug(
"Handling batch collection instruction: {}".format(line))
- Designs(command_arr[1], download_dir, quick, compress).download()
+ Designs(command_arr[1], download_dir, quick, compress, api_key).download()
continue
logging.warning("Unable to parse current instruction. Skipping.")
console_handler = logging.StreamHandler()
console_handler.setLevel(args.log_level.upper())
-
if args.api_key:
api_key = args.api_key
else:
if args.subcommand == "version":
print("thingy_grabber.py version {}".format(VERSION))
if args.subcommand == "batch":
- do_batch(args.batch_file, args.directory, args.quick, args.compress)
+ do_batch(args.batch_file, args.directory, args.quick, args.compress, api_key)
# Stop the downloader processes
for _ in downloaders: