|
|
- from urllib.parse import urlparse
- from readabilipy import simple_json_from_html_string
- import requests
-
- def clean_articles(rows):
- #article_id, url, title, byline
- out = []
-
- for row in rows:
- parsed_uri = urlparse(row[1])
- result = '{uri.netloc}'.format(uri=parsed_uri)
- out.append([row[0], row[1], row[2], row[3], result])
-
- return out
-
-
- def get_article(url):
- headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
- response = requests.get(url, headers=headers)
-
- article = simple_json_from_html_string(response.text, use_readability=True)
- parsed_uri = urlparse(url)
- result = '{uri.netloc}'.format(uri=parsed_uri)
-
- return article, result
|
