Merge branch 'master' into feature/tandfonline

2 files changed, 27 insertions, 0 deletions

diff --git a/paper2remarkable/utils.py b/paper2remarkable/utils.py
index 5b7ba2c..09082a5 100644
--- a/paper2remarkable/utils.py
+++ b/paper2remarkable/utils.py
@@ -112,6 +112,28 @@ def get_content_type_with_retry(url, tries=5, cookiejar=None):
             continue
         return res.headers.get("Content-Type", None)
 
+    # In rare cases, a HEAD request fails but a GET request does work. So here
+    # we try to get the content type from a GET request.
+    count = 0
+    jar = {} if cookiejar is None else cookiejar
+    while count < tries:
+        count += 1
+        error = False
+        try:
+            res = requests.get(
+                url, headers=HEADERS, cookies=jar, allow_redirects=True
+            )
+        except requests.exceptions.ConnectionError:
+            error = True
+        if error or not res.ok:
+            logger.warning(
+                "(%i/%i) Error getting headers for %s. Retrying in 5 seconds."
+                % (count, tries, url)
+            )
+            time.sleep(5)
+            continue
+        return res.headers.get("Content-Type", None)
+
 
 def follow_redirects(url):
     """Follow redirects from the URL (at most 100)"""
diff --git a/tests/test_ui.py b/tests/test_ui.py
index 835f594..1cca0cd 100644
--- a/tests/test_ui.py
+++ b/tests/test_ui.py
@@ -101,6 +101,11 @@ class TestUI(unittest.TestCase):
                 "https://confcats_isif.s3.amazonaws.com/web-files/journals/entries/Nonlinear%20Kalman%20Filters.pdf",
             ),
             (
+                PdfUrl,
+                "https://publications.aston.ac.uk/id/eprint/38334/1/5th_Artificial_Neural_Networks.pdf",
+                "https://publications.aston.ac.uk/id/eprint/38334/1/5th_Artificial_Neural_Networks.pdf",
+            ),
+            (
                 JMLR,
                 "https://www.jmlr.org/papers/volume17/14-526/14-526.pdf",
                 "https://www.jmlr.org/papers/volume17/14-526/14-526.pdf",