diff --git a/dirhunt/crawler_url.py b/dirhunt/crawler_url.py index 246cce9..642606e 100644 --- a/dirhunt/crawler_url.py +++ b/dirhunt/crawler_url.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -import cgi +import email.message import socket from bs4 import BeautifulSoup @@ -71,7 +71,9 @@ def start(self): self.crawler.results.put(Error(self, e)) self.close() return self - content_type = cgi.parse_header(resp.headers.get('Content-Type', ''))[0] + ct = email.message.Message() + ct['Content-Type'] = resp.headers.get('Content-Type', '') + content_type = ct.get_content_type() soup = BeautifulSoup(text, 'html.parser') if content_type == 'text/html' else None except RequestException as e: self.crawler.current_processed_count += 1 @@ -112,7 +114,9 @@ def maybe_rewrite(self): def must_be_downloaded(self, response): """The file must be downloaded to obtain information. """ - return self.maybe_directory() or (cgi.parse_header(response.headers.get('Content-Type', ''))[0] in [ + ct = email.message.Message() + ct['Content-Type'] = response.headers.get('Content-Type', '') + return self.maybe_directory() or (ct.get_content_type() in [ 'text/css', 'application/javascript' ]) diff --git a/dirhunt/processors.py b/dirhunt/processors.py index b8c18ea..445f468 100644 --- a/dirhunt/processors.py +++ b/dirhunt/processors.py @@ -99,7 +99,7 @@ def search_index_files(self): def search_keywords(self, text): if sys.version_info > (3,) and isinstance(text, bytes): - text = text.decode('utf-8') + text = text.decode('utf-8', errors='replace') for keyword in self.crawler_url.crawler.interesting_keywords: if keyword in text: self.keywords_found.add(keyword) @@ -252,9 +252,9 @@ class ProcessCssStyleSheet(ProcessBase): def process(self, text, soup=None): if sys.version_info > (3,) and isinstance(text, bytes): - text = text.decode('utf-8') + text = text.decode('utf-8', errors='replace') self.search_keywords(text) - urls = [full_url_address(url, self.crawler_url.url) for url in re.findall(': *url\(["\']?(.+?)["\']?\)', text)] + urls = [full_url_address(url, self.crawler_url.url) for url in re.findall(r'url\([\'"]?(.+?)[\'"]?\)', text)] for url in urls: self.add_url(url, depth=0, type='asset') return urls @@ -270,7 +270,7 @@ class ProcessJavaScript(ProcessBase): def process(self, text, soup=None): if sys.version_info > (3,) and isinstance(text, bytes): - text = text.decode('utf-8') + text = text.decode('utf-8', errors='replace') self.search_keywords(text) urls = [full_url_address(url[0], self.crawler_url.url) for url in re.findall(TEXT_PLAIN_PATH_STRING_REGEX, text, re.VERBOSE)] @@ -426,7 +426,7 @@ def tag_visible(element): if isinstance(element, Comment): return False return True - texts = soup.findAll(text=True) + texts = soup.find_all(string=True) visible_texts = filter(tag_visible, texts) for text in visible_texts: if text.strip():