From aff396e8c0adde4e2df6eee1fce8775898771036 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 21 Jun 2017 16:22:54 +1000 Subject: [PATCH 1/5] UNSC Resolution scraper with Beautiful Soup / CSS selectors --- code/unsc-with-bs4-csssel.py | 91 ++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 code/unsc-with-bs4-csssel.py diff --git a/code/unsc-with-bs4-csssel.py b/code/unsc-with-bs4-csssel.py new file mode 100644 index 0000000..3b8d3f9 --- /dev/null +++ b/code/unsc-with-bs4-csssel.py @@ -0,0 +1,91 @@ +"""Using Beautiful Soup with CSS Selectors to collect UNSC Resolutions +""" + +import time +import requests +from requests.compat import urljoin +import bs4 + + +def get_year_urls(): + """Return a list of (year_url, year) pairs + """ + # WARNING: the final / is very important when doing urljoin below + base_url = 'http://www.un.org/en/sc/documents/resolutions/' + response = requests.get(base_url) + soup = bs4.BeautifulSoup(response.content, 'html.parser') + tables = soup.select('#content > table') + # It's a good idea to check you captured something + # and not more than you expected + assert len(tables) == 1 + + table = tables[0] + links = table.select('a') + + out = [] + for link in links: + year_url = urljoin(base_url, link.attrs['href']) + year = link.text + # As well as converting to a number, this validates the year text is + # what we expect + year = int(year) + out.append((year_url, year)) + + # Check we got something + assert len(out) + return out + + +def get_resolutions_for_year(year_url, year): + """Return a list of dicts, each detailing 1 UNSC resolution from given year + """ + response = requests.get(year_url) + soup = bs4.BeautifulSoup(response.content, 'html.parser') + tables = soup.select('#content > table') + if year != 1960 and year != 1964: + # 1960 and 1964 have the entire page repeated twice! + # Let's just use the first copy... + assert len(tables) == 1 + + symbol_cells = tables[0].select('td:nth-of-type(1)') + + out = [] + for symbol_cell in symbol_cells: + links = symbol_cell.select('a') + if not links: + # http://www.un.org/en/sc/documents/resolutions/2013.shtml + # has a row which breaks our scraper. Skip it. + print('Found a cell that does not have a view link: ', + symbol_cell.text) + continue + url = links[0].attrs['href'] + title_cell = symbol_cell.find_next_sibling() + out.append({'year': year, + 'title': title_cell.text, + 'symbol': symbol_cell.text, + 'url': urljoin(year_url, url)}) + return out + + +def scrape_unsc_resolutions(): + # Note that for some projects you might be scraping lots of data + # over a long time, and so might not want to store all the data in + # memory at the same time like this code does. + out = [] + for year_url, year in get_year_urls(): + time.sleep(0.01) + print('Processing:', year_url) + out += get_resolutions_for_year(year_url, year) + return out + + +if __name__ == '__main__': + import csv + + resolutions = scrape_unsc_resolutions() + + with open('unsc-resolutions.csv') as out_file: + writer = csv.DictWriter(out_file, ['year', 'symbol', 'title', 'url']) + writer.writeheader() + for resolution in resolutions: + writer.writerow(resolution) From e852baf25e17e4660a7ce5962e4c01da60233672 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 21 Jun 2017 17:27:42 +1000 Subject: [PATCH 2/5] Add missing write mode --- code/unsc-with-bs4-csssel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/code/unsc-with-bs4-csssel.py b/code/unsc-with-bs4-csssel.py index 3b8d3f9..fa4f384 100644 --- a/code/unsc-with-bs4-csssel.py +++ b/code/unsc-with-bs4-csssel.py @@ -84,7 +84,7 @@ def scrape_unsc_resolutions(): resolutions = scrape_unsc_resolutions() - with open('unsc-resolutions.csv') as out_file: + with open('unsc-resolutions.csv', 'w') as out_file: writer = csv.DictWriter(out_file, ['year', 'symbol', 'title', 'url']) writer.writeheader() for resolution in resolutions: From c576641b74c1090c5a979b320001567c6c03f8ac Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Wed, 21 Jun 2017 17:35:01 +1000 Subject: [PATCH 3/5] UNSC scraper with webdriver and CSS Selectors --- code/unsc-with-bs4-csssel.py | 1 + code/unsc-with-webdriver-csssel.py | 99 ++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 code/unsc-with-webdriver-csssel.py diff --git a/code/unsc-with-bs4-csssel.py b/code/unsc-with-bs4-csssel.py index fa4f384..8c7a75a 100644 --- a/code/unsc-with-bs4-csssel.py +++ b/code/unsc-with-bs4-csssel.py @@ -59,6 +59,7 @@ def get_resolutions_for_year(year_url, year): symbol_cell.text) continue url = links[0].attrs['href'] + # TODO: Handle the 2014 quirk!! title_cell = symbol_cell.find_next_sibling() out.append({'year': year, 'title': title_cell.text, diff --git a/code/unsc-with-webdriver-csssel.py b/code/unsc-with-webdriver-csssel.py new file mode 100644 index 0000000..c51844c --- /dev/null +++ b/code/unsc-with-webdriver-csssel.py @@ -0,0 +1,99 @@ +"""Using Beautiful Soup with CSS Selectors to collect UNSC Resolutions +""" + +import time +from selenium import webdriver + + +def get_year_urls(driver): + """Return a list of (year_url, year) pairs + """ + # WARNING: the final / is very important when doing urljoin below + base_url = 'http://www.un.org/en/sc/documents/resolutions/' + driver.get(base_url) + tables = driver.find_elements_by_css_selector('#content > table') + # It's a good idea to check you captured something + # and not more than you expected + assert len(tables) == 1 + + table = tables[0] + links = table.find_elements_by_css_selector('a') + + out = [] + for link in links: + # Note that the webdriver returns a resolved URL: not 1977.shtml + # as in the HTML, rather http://www.un.org/en/sc/documents/resolutions/1977.shtml + year_url = link.get_attribute('href') + year = link.text + # As well as converting to a number, this validates the year text is + # what we expect + year = int(year) + out.append((year_url, year)) + + # Check we got something + assert len(out) + return out + + +def get_resolutions_for_year(driver, year_url, year): + """Return a list of dicts, each detailing 1 UNSC resolution from given year + """ + driver.get(year_url) + tables = driver.find_elements_by_css_selector('#content > table') + if year != 1960 and year != 1964: + # 1960 and 1964 have the entire page repeated twice! + # Let's just use the first copy... + assert len(tables) == 1 + + symbol_cells = tables[0].find_elements_by_css_selector('td:nth-of-type(1)') + + out = [] + for symbol_cell in symbol_cells: + links = symbol_cell.find_elements_by_css_selector('a') + if not links: + # http://www.un.org/en/sc/documents/resolutions/2013.shtml + # has a row which breaks our scraper. Skip it. + print('Found a cell that does not have a view link: ', + symbol_cell.text) + continue + url = links[0].get_attribute('href') + # TODO: Handle the 2014 quirky date column!! + title_cell = symbol_cell.get_property('nextElementSibling') + out.append({'year': year, + 'title': title_cell.text, + 'symbol': symbol_cell.text, + 'url': url}) + return out + + +def scrape_unsc_resolutions(): + # Note that for some projects you might be scraping lots of data + # over a long time, and so might not want to store all the data in + # memory at the same time like this code does. + # driver = selenium.webdriver.PhantomJS(service_args=['--load-images=no']) + + options = webdriver.ChromeOptions() + # Don't show web browser visually + options.add_argument('headless') + # Don't download images + options.add_experimental_option("prefs", + {"profile.managed_default_content_settings.images": 2}) + driver = webdriver.Chrome(chrome_options=options) + + out = [] + for year_url, year in get_year_urls(driver): + print('Processing:', year_url) + out += get_resolutions_for_year(driver, year_url, year) + return out + + +if __name__ == '__main__': + import csv + + resolutions = scrape_unsc_resolutions() + + with open('unsc-resolutions.csv', 'w') as out_file: + writer = csv.DictWriter(out_file, ['year', 'symbol', 'title', 'url']) + writer.writeheader() + for resolution in resolutions: + writer.writerow(resolution) From cb76f8c04cde85cb7302d9793ce8067eb0e85146 Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Thu, 22 Jun 2017 01:02:03 +1000 Subject: [PATCH 4/5] Fix another two quirks --- code/unsc-with-bs4-csssel.py | 21 ++++++++++++++------- code/unsc-with-webdriver-csssel.py | 15 ++++++++++----- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/code/unsc-with-bs4-csssel.py b/code/unsc-with-bs4-csssel.py index 8c7a75a..b713767 100644 --- a/code/unsc-with-bs4-csssel.py +++ b/code/unsc-with-bs4-csssel.py @@ -13,7 +13,7 @@ def get_year_urls(): # WARNING: the final / is very important when doing urljoin below base_url = 'http://www.un.org/en/sc/documents/resolutions/' response = requests.get(base_url) - soup = bs4.BeautifulSoup(response.content, 'html.parser') + soup = bs4.BeautifulSoup(response.content, 'lxml') tables = soup.select('#content > table') # It's a good idea to check you captured something # and not more than you expected @@ -40,17 +40,24 @@ def get_resolutions_for_year(year_url, year): """Return a list of dicts, each detailing 1 UNSC resolution from given year """ response = requests.get(year_url) - soup = bs4.BeautifulSoup(response.content, 'html.parser') + # NOTE: a loose breaks parsing with html.parser engine in 2017.shtml + soup = bs4.BeautifulSoup(response.content, 'lxml') tables = soup.select('#content > table') if year != 1960 and year != 1964: # 1960 and 1964 have the entire page repeated twice! - # Let's just use the first copy... + # Let's just use the first copy in all cases... assert len(tables) == 1 - symbol_cells = tables[0].select('td:nth-of-type(1)') + rows = tables[0].select('tr') out = [] - for symbol_cell in symbol_cells: + for row in rows: + cells = row.select('td') + if len(cells) < 2: + # ignore the header + continue + symbol_cell = cells[0] + title_cell = cells[-1] links = symbol_cell.select('a') if not links: # http://www.un.org/en/sc/documents/resolutions/2013.shtml @@ -59,12 +66,12 @@ def get_resolutions_for_year(year_url, year): symbol_cell.text) continue url = links[0].attrs['href'] - # TODO: Handle the 2014 quirk!! - title_cell = symbol_cell.find_next_sibling() out.append({'year': year, 'title': title_cell.text, + # TODO: whitespace needs cleaning in these! 'symbol': symbol_cell.text, 'url': urljoin(year_url, url)}) + assert len(out) > 1 or year == 1959 return out diff --git a/code/unsc-with-webdriver-csssel.py b/code/unsc-with-webdriver-csssel.py index c51844c..22f3beb 100644 --- a/code/unsc-with-webdriver-csssel.py +++ b/code/unsc-with-webdriver-csssel.py @@ -42,13 +42,19 @@ def get_resolutions_for_year(driver, year_url, year): tables = driver.find_elements_by_css_selector('#content > table') if year != 1960 and year != 1964: # 1960 and 1964 have the entire page repeated twice! - # Let's just use the first copy... + # Let's just use the first copy in all cases... assert len(tables) == 1 - symbol_cells = tables[0].find_elements_by_css_selector('td:nth-of-type(1)') + rows = tables[0].find_elements_by_css_selector('tr') out = [] - for symbol_cell in symbol_cells: + for row in rows: + cells = row.find_elements_by_css_selector('td') + if len(cells) < 2: + # ignore the header + continue + symbol_cell = cells[0] + title_cell = cells[-1] links = symbol_cell.find_elements_by_css_selector('a') if not links: # http://www.un.org/en/sc/documents/resolutions/2013.shtml @@ -57,12 +63,11 @@ def get_resolutions_for_year(driver, year_url, year): symbol_cell.text) continue url = links[0].get_attribute('href') - # TODO: Handle the 2014 quirky date column!! - title_cell = symbol_cell.get_property('nextElementSibling') out.append({'year': year, 'title': title_cell.text, 'symbol': symbol_cell.text, 'url': url}) + assert len(out) > 1 or year == 1959 return out From 6e3ee1eebf39a8ac85856f65865f1d8f27b1a6fa Mon Sep 17 00:00:00 2001 From: Joel Nothman Date: Thu, 22 Jun 2017 01:05:02 +1000 Subject: [PATCH 5/5] Scraping UNSC Resolutions with lxml --- code/unsc-with-lxml-csssel.py | 102 ++++++++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 code/unsc-with-lxml-csssel.py diff --git a/code/unsc-with-lxml-csssel.py b/code/unsc-with-lxml-csssel.py new file mode 100644 index 0000000..b7d2033 --- /dev/null +++ b/code/unsc-with-lxml-csssel.py @@ -0,0 +1,102 @@ +"""Using Beautiful tree with CSS Selectors to collect UNSC Resolutions +""" + +import time +import requests +from requests.compat import urljoin +from lxml import etree + + +def inner_text(element): + return ''.join(element.itertext()) + + +def get_year_urls(): + """Return a list of (year_url, year) pairs + """ + # WARNING: the final / is very important when doing urljoin below + base_url = 'http://www.un.org/en/sc/documents/resolutions/' + response = requests.get(base_url) + tree = etree.HTML(response.content) + tables = tree.cssselect('#content > table') + # It's a good idea to check you captured something + # and not more than you expected + assert len(tables) == 1 + + table = tables[0] + links = table.cssselect('a') + + out = [] + for link in links: + year_url = urljoin(base_url, link.attrib['href']) + year = link.text + # As well as converting to a number, this validates the year text is + # what we expect + year = int(year) + out.append((year_url, year)) + + # Check we got something + assert len(out) + return out + + +def get_resolutions_for_year(year_url, year): + """Return a list of dicts, each detailing 1 UNSC resolution from given year + """ + response = requests.get(year_url) + tree = etree.HTML(response.content) + tables = tree.cssselect('#content > table') + if year != 1960 and year != 1964: + # 1960 and 1964 have the entire page repeated twice! + # Let's just use the first copy in all cases... + assert len(tables) == 1 + + rows = tables[0].cssselect('tr') + + out = [] + for row in rows: + cells = row.cssselect('td') + if len(cells) < 2: + # ignore the header + continue + symbol_cell = cells[0] + title_cell = cells[-1] + links = symbol_cell.cssselect('a') + if not links: + # http://www.un.org/en/sc/documents/resolutions/2013.shtml + # has a row which breaks our scraper. Skip it. + print('Found a cell that does not have a view link: ', + inner_text(symbol_cell)) + continue + url = links[0].attrib['href'] + out.append({'year': year, + 'title': inner_text(title_cell), + # TODO: whitespace needs cleaning in these! + 'symbol': inner_text(symbol_cell), + 'url': urljoin(year_url, url)}) + assert len(out) > 1 or year == 1959 + return out + + +def scrape_unsc_resolutions(): + # Note that for some projects you might be scraping lots of data + # over a long time, and so might not want to store all the data in + # memory at the same time like this code does. + out = [] + for year_url, year in get_year_urls(): + time.sleep(0.01) + print('Processing:', year_url) + out += get_resolutions_for_year(year_url, year) + return out + + +if __name__ == '__main__': + import csv + + resolutions = scrape_unsc_resolutions() + + with open('unsc-resolutions.csv', 'w') as out_file: + writer = csv.DictWriter(out_file, ['year', 'symbol', 'title', 'url']) + writer.writeheader() + for resolution in resolutions: + writer.writerow(resolution)