From 2b4fc42e99f08052fe73f92845f4b155d05fee25 Mon Sep 17 00:00:00 2001 From: Gertjan van den Burg Date: Tue, 7 Jan 2020 14:26:31 +0000 Subject: Carry cookies over when redirecting This was needed for ACM, and is likely beneficial for other sites as well. --- paper2remarkable/utils.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/paper2remarkable/utils.py b/paper2remarkable/utils.py index cc8a417..de27973 100644 --- a/paper2remarkable/utils.py +++ b/paper2remarkable/utils.py @@ -96,15 +96,17 @@ def get_page_with_retry(url, tries=5): def follow_redirects(url): - """Follow redirects from the URL (at most 10)""" + """Follow redirects from the URL (at most 100)""" it = 0 - while it < 10: - req = requests.head(url, allow_redirects=False) + jar = {} + while it < 100: + req = requests.head(url, allow_redirects=False, cookies=jar) if req.status_code == 200: break if not "Location" in req.headers: break url = req.headers["Location"] + jar = req.cookies it += 1 return url -- cgit v1.2.3