From ef8697e7e9bf942d90ad842b5a817a6db118245b Mon Sep 17 00:00:00 2001 From: stevepodell Date: Sun, 27 Sep 2026 11:28:10 -0700 Subject: [PATCH 1/2] Probable fix for PDF scraping in WeVoteServer Also temporairly enabled a bunch of logging to diagnose potential problems. --- apis_v1/views/views_extension.py | 26 +++++++++++++------------- candidate/controllers.py | 3 ++- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/apis_v1/views/views_extension.py b/apis_v1/views/views_extension.py index 5b339bdbb..842cbf132 100644 --- a/apis_v1/views/views_extension.py +++ b/apis_v1/views/views_extension.py @@ -80,7 +80,7 @@ def process_pdf_to_html(pdf_url, return_version): output_from_subprocess = 'exception occurred before output was captured' status = '' success = False - # logger.error('entry to process_pdf_to_html:' + pdf_url + ' ' + str(return_version)) + logger.error('entry to process_pdf_to_html:' + pdf_url + ' ' + str(return_version)) # Version report, only used to debug connectivity to the Tika server if return_version: @@ -101,7 +101,7 @@ def process_pdf_to_html(pdf_url, return_version): } return json_data - # logger.error('immediately after return_version: ' + str(return_version)) + logger.error('immediately after return_version: ' + str(return_version)) pdf_file_name = os.path.basename(pdf_url) absolute_html_file = build_absolute_path_for_tempfile(pdf_file_name).replace('.pdf', '.html') try: @@ -119,26 +119,26 @@ def process_pdf_to_html(pdf_url, return_version): try: raw = scraper.get(pdf_url) pdf_text_text = raw.content # in bytes, not using str(raw.content) - # logger.error('cloudscraper attempt with base PDF url : ' + pdf_url + - # ' returned bytes: ' + str(len(pdf_text_text))) + logger.error('cloudscraper attempt with base PDF url : ' + pdf_url + + ' returned bytes: ' + str(len(pdf_text_text))) success = True # Probably got a http 403 forbidden, due to cloudscraper unsuccessfully handling a Cloudflare challenge # Now try to use Google's (hopefully) cached version of the page except Exception as scraper_or_tempfile_error: status = "First pass with base url failed with a " + str(scraper_or_tempfile_error) - # logger.error('cloudscraper with base PDF url or tempfile write exception: ' + - # str(scraper_or_tempfile_error)) + logger.error('cloudscraper with base PDF url or tempfile write exception: ' + + str(scraper_or_tempfile_error)) if not success: logger.error('first pass === not success') is_pdf = False try: - # logger.error('first pass === not success, pdf_url: ' + pdf_url) + logger.error('first pass === not success, pdf_url: ' + pdf_url) encoded = quote(pdf_url, safe='') - # logger.error('encoded success: ' + encoded) + logger.error('encoded success: ' + encoded) google_cached_pdf_url = 'https://webcache.googleusercontent.com/search?q=cache:' + encoded - # logger.error('cloudscraper attempt with google cached PDF url: ' + google_cached_pdf_url) + logger.error('cloudscraper attempt with google cached PDF url: ' + google_cached_pdf_url) headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) ' @@ -149,14 +149,14 @@ def process_pdf_to_html(pdf_url, return_version): 'Accept-Language': 'en-US,en;q=0.8', 'Connection': 'keep-alive'} r = requests.get(google_cached_pdf_url, headers) - # logger.error('after requests.get: ' + google_cached_pdf_url) + logger.error('after requests.get: ' + google_cached_pdf_url) # skip saving the pdf file (since we don't have one), and write the final html file to the temp dir html_text_text = r.text out_file = open(absolute_html_file, 'w') out_file.write(html_text_text) - # logger.error('requests was successful with google cached PDF url : ' + google_cached_pdf_url + - # ' returned bytes: ' + str(len(pdf_text_text))) + logger.error('requests was successful with google cached PDF url : ' + google_cached_pdf_url + + ' returned bytes: ' + str(len(pdf_text_text))) success = True except Exception as scraper_or_tempfile_error2: # Out of luck status += ", Second pass with google cached PDF url failed with a: " + str(scraper_or_tempfile_error2) @@ -175,7 +175,7 @@ def process_pdf_to_html(pdf_url, return_version): output_from_subprocess = 'Tika status: ' + str(tika_response.status_code) with open(absolute_html_file, 'w') as out_file: out_file.write(tika_response.text) - # logger.error('Tika PUT output: ' + output_from_subprocess) + logger.error('Tika PUT output: ' + output_from_subprocess) except Exception as tika_error: status += ', ' + str(tika_error) logger.error('Tika PUT request exception: ' + str(tika_error)) diff --git a/candidate/controllers.py b/candidate/controllers.py index 241e6bbb4..4dd456862 100644 --- a/candidate/controllers.py +++ b/candidate/controllers.py @@ -3384,7 +3384,8 @@ def find_organization_endorsements_of_candidates_on_one_web_page(site_url, endor if site_url.lower().endswith(".pdf"): print("PDF Detected ", site_url) - response = process_pdf_to_html(site_url) + return_version = False + response = process_pdf_to_html(site_url, return_version) if positive_value_exists(response['s3_url_for_html']): # Overwrite the site_url parameter, with a url to an html representation of the PDF file site_url = response['s3_url_for_html'] From b5b1a4851e1ec35e361e3755381ac317b72f2245 Mon Sep 17 00:00:00 2001 From: stevepodell Date: Sun, 27 Sep 2026 11:31:41 -0700 Subject: [PATCH 2/2] Probable fix for PDF scraping in WeVoteServer Also temporairly enabled a bunch of logging to diagnose potential problems. --- apis_v1/views/views_extension.py | 2 +- candidate/controllers.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apis_v1/views/views_extension.py b/apis_v1/views/views_extension.py index 842cbf132..96b234766 100644 --- a/apis_v1/views/views_extension.py +++ b/apis_v1/views/views_extension.py @@ -80,7 +80,7 @@ def process_pdf_to_html(pdf_url, return_version): output_from_subprocess = 'exception occurred before output was captured' status = '' success = False - logger.error('entry to process_pdf_to_html:' + pdf_url + ' ' + str(return_version)) + logger.error('entry to process_pdf_to_html: ' + pdf_url + ' ' + str(return_version)) # Version report, only used to debug connectivity to the Tika server if return_version: diff --git a/candidate/controllers.py b/candidate/controllers.py index 4dd456862..e12e58175 100644 --- a/candidate/controllers.py +++ b/candidate/controllers.py @@ -3384,6 +3384,7 @@ def find_organization_endorsements_of_candidates_on_one_web_page(site_url, endor if site_url.lower().endswith(".pdf"): print("PDF Detected ", site_url) + return_version = False response = process_pdf_to_html(site_url, return_version) if positive_value_exists(response['s3_url_for_html']):