+# Currently useless as it gives the same info as the matching element in API_USER_COLLECTIONS
+API_COLLECTION = API_BASE + "/collections/{}/?" + ACCESS_QP
+API_COLLECTION_THINGS = API_BASE + "/collections/{}/things/?" + ACCESS_QP
+
+API_THING_DETAILS = API_BASE + "/things/{}/?" + ACCESS_QP
+API_THING_FILES = API_BASE + "/things/{}/files/?" + ACCESS_QP
+API_THING_IMAGES = API_BASE + "/things/{}/images/?" + ACCESS_QP
+API_THING_DOWNLOAD = "/download/?" + ACCESS_QP
+
+DOWNLOADER_COUNT = 1
+RETRY_COUNT = 3
+
+MAX_PATH_LENGTH = 250
+
+VERSION = "0.10.5"
+
+TIMESTAMP_FILE = "timestamp.txt"
+
+SESSION = requests.Session()
+
+
+class MLStripper(HTMLParser):
+ """ Turns HTML markup into plain text
+ """
+
+ def error(self, message):
+ raise ValueError(message)
+
+ def __init__(self):
+ super().__init__()
+ self.reset()
+ self.strict = False
+ self.convert_charrefs = True
+ self.text = StringIO()
+
+ def handle_data(self, d):
+ self.text.write(d)
+
+ def get_data(self):
+ return self.text.getvalue()
+
+ @staticmethod
+ def strip_tags(html):
+ s = MLStripper()
+ s.feed(html)
+ return s.get_data()
+
+
+@dataclass
+class ThingLink:
+ thing_id: str
+ name: str
+ api_link: str
+
+
+@dataclass
+class FileLink:
+ name: str
+ last_update: datetime.datetime
+ link: str
+
+
+@dataclass
+class ImageLink:
+ name: str
+ link: str
+
+
+class FileLinks:
+ def __init__(self, initial_links=None):
+ if initial_links is None:
+ initial_links = []
+ self.links = []
+ self.last_update = None
+ for link in initial_links:
+ self.append(link)
+
+ def __iter__(self):
+ return iter(self.links)
+
+ def __getitem__(self, item):
+ return self.links[item]
+
+ def __len__(self):
+ return len(self.links)
+
+ def append(self, link):
+ try:
+ self.last_update = max(self.last_update, link.last_update)
+ except TypeError:
+ self.last_update = link.last_update
+ self.links.append(link)
+
+
+class State(enum.Enum):
+ OK = enum.auto()
+ FAILED = enum.auto()
+ ALREADY_DOWNLOADED = enum.auto()
+
+
+def sanitise_url(url):
+ """ remove api keys from an url
+ """
+ return re.sub(r'access_token=\w*',
+ 'access_token=***',
+ url)
+
+
+def strip_time(date_obj):
+ """ Takes a datetime object and returns another with the time set to 00:00
+ """
+ return datetime.datetime.combine(date_obj.date(), datetime.time())
+
+
+def rename_unique(dir_name, target_dir_name):
+ """ Move a directory sideways to a new name, ensuring it is unique.
+ """
+ target_dir = target_dir_name
+ inc = 0
+ while os.path.exists(target_dir):
+ target_dir = "{}_{}".format(target_dir_name, inc)
+ inc += 1
+ os.rename(dir_name, target_dir)
+ return target_dir
+
+
+def fail_dir(dir_name):
+ """ When a download has failed, move it sideways.
+ """
+ return rename_unique(dir_name, "{}_failed".format(dir_name))
+
+
+def truncate_name(file_name):
+ """ Ensure the filename is not too long for, well windows basically.
+ """
+ path = os.path.abspath(file_name)
+ if len(path) <= MAX_PATH_LENGTH:
+ return path
+ base, extension = os.path.splitext(path)
+ inc = 0
+ new_path = "{}_{}{}".format(base, inc, extension)
+ while os.path.exists(new_path):
+ new_path = "{}_{}{}".format(base, inc, extension)
+ inc += 1
+ return new_path