[WIP] [#604] Normalize organisations names

processors/base/helpers/__init__.py

@@ -13,20 +13,19 @@
 import datetime
 import urlparse
 import csv
+import unidecode
 import fuzzywuzzy.fuzz
 import fuzzywuzzy.process
 import iso3166
+import dedupe
 
 from . import pybossa_tasks_updater
 
 logger = logging.getLogger(__name__)
 PyBossaTasksUpdater = pybossa_tasks_updater.PyBossaTasksUpdater
 
-
 # Module API
 
-EDIT_DISTANCE_THRESHOLD = 75
-
 
 def get_variables(object, filter=None):
     """Exract variables from object to dict using name filter.
@@ -52,7 +51,6 @@ def slugify_string(string):
 class JSONEncoder(json.JSONEncoder):
     """JSON encoder with datetime, date set support.
     """
-
     def default(self, obj):
         if isinstance(obj, datetime.datetime):
             return obj.strftime('%Y-%m-%dT%H:%M:%SZ')
@@ -301,6 +299,7 @@ def get_canonical_location_name(location):
     # Extracted from: https://github.com/datasets/country-codes/blob/master/data/country-codes.csv
     CSV_PATH = os.path.join(os.path.dirname(__file__), 'data/countries.csv')
     DISTANCE_SCORER = fuzzywuzzy.fuzz.token_sort_ratio
+    EDIT_DISTANCE_THRESHOLD = 75
 
     cleaned_location = remove_string_punctuation(location)
     current_match = location
@@ -315,8 +314,10 @@ def get_canonical_location_name(location):
                 relevant_info = [unicode(country[field], encoding='utf-8')
                                  for field in reader.fieldnames[0:5]]
 
-                location_info = [remove_string_punctuation(location_info) for location_info in relevant_info]
-                _, score = fuzzywuzzy.process.extractOne(cleaned_location, location_info, scorer=DISTANCE_SCORER)
+                location_info = [remove_string_punctuation(location_info)
+                                 for location_info in relevant_info]
+                _, score = fuzzywuzzy.process.extractOne(
+                    cleaned_location, location_info, scorer=DISTANCE_SCORER)
 
                 if score > current_score:
                     current_match = iso3166.countries.get(country['ISO3166-1-Alpha-3']).name
@@ -329,3 +330,88 @@ def get_canonical_location_name(location):
     logger.debug('Location "%s" normalized as "%s"', cleaned_location, current_match)
 
     return current_match
+
+
+def get_canonical_organisation_name(conn, organisation):
+    """Find a canonical organisation name according to
+    a pre-built set of clusters
+
+    Args:
+        conn (dict): connection dict
+        organisation (str): the organisation to be normalized
+    """
+    CLUSTER_QUERY = 'SELECT canonical FROM organisation_clusters ' + \
+                    'WHERE :organisation=ANY(variations)'
+
+    normalized_form = organisation
+
+    # Try to find the organisation in some cluster
+    try:
+        normalized_form = conn['warehouse'].query(CLUSTER_QUERY,
+                                                  organisation=organisation).next()['canonical']
+        logger.debug('Organisation "%s" normalized as "%s"', organisation, normalized_form)
+    except StopIteration:
+        logger.debug('Organisation "%s" not normalized', organisation)
+
+    return normalized_form
+
+
+def _dedup_cluster(cluster_entries):
+    """Sample, train and build organisation clusters.
+    Labeled training example used for clustering purposes
+    is stored in data/organisation_training_data.json and
+    can be regenerated using Dedupe consoleLabel tool
+    See: http://dedupe.readthedocs.io/en/latest/API-documentation.html#consoleLabel
+
+    Args:
+        cluster_entries (list): a list of candidates to
+        be grouped in equivalent sets
+    """
+    SAMPLE_SIZE = 10000
+    MATCH_THRESHOLD = 0.75
+    TRAINING_FILE = os.path.join(os.path.dirname(__file__),
+                                 'data/organisation_training_data.json')
+
+    fields = [{'field': 'name', 'type': 'String'}]
+    deduper = dedupe.Dedupe(fields)
+    deduper.sample(cluster_entries, SAMPLE_SIZE)
+
+    with open(TRAINING_FILE) as training_file:
+        deduper.readTraining(training_file)
+
+    deduper.train()
+
+    return deduper.match(cluster_entries, MATCH_THRESHOLD)
+
+
+def update_organisation_clusters(conn):
+    """Build a readable list of organisation clusters to be used
+    when normalizing new entries
+
+    Args:
+        conn (dict): connection dict
+    """
+    logger.debug('Recomputing organisations cluster data for further access')
+
+    conn['warehouse'].begin()
+    conn['warehouse']['organisation_clusters'].delete()
+
+    stored_organisations = list(conn['database']['organisations'].all())
+
+    cluster_members = {entry['id']: {'name': unidecode.unidecode(entry['name'])}
+                       for entry in stored_organisations}
+
+    clustered_dupes = _dedup_cluster(cluster_members)
+
+    for (_, cluster) in enumerate(clustered_dupes):
+        id_set = cluster[0]
+        cluster_membership = [str(cluster_members[c]['name']) for c in id_set]
+
+        cluster = {
+            'canonical': max(cluster_membership, key=len).replace("'", "''"),
+            'variations': cluster_membership
+        }
+
+        conn['warehouse']['organisation_clusters'].insert(cluster)
+
+    conn['warehouse'].commit()

processors/base/writers/organisation.py

@@ -33,7 +33,7 @@ def write_organisation(conn, organisation, source_id, trial_id=None):
     timestamp = datetime.datetime.utcnow()
 
     # Get name
-    name = helpers.clean_string(organisation['name'])
+    name = helpers.get_canonical_organisation_name(organisation['name'])
     if len(name) <= 1:
         return None
 

processors/normalization_clusters_updater/__init__.py

@@ -0,0 +1,7 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from .processor import process

processors/normalization_clusters_updater/processor.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import processors.base.helpers
+
+
+# Module API
+
+def process(conf, conn):
+    """Update clusters used when normalizing trials entities
+    """
+    processors.base.helpers.update_organisation_clusters(conn)

requirements.in

@@ -13,3 +13,5 @@ fuzzywuzzy
 python-Levenshtein
 iso3166
 raven
+unidecode
+dedupe

requirements.txt

@@ -4,58 +4,79 @@
 #
 #    pip-compile --output-file requirements.txt requirements.in
 #
+affinegap==1.10           # via canonicalize, dedupe
 alembic==0.8.10           # via dataset
 appdirs==1.4.0            # via setuptools
 boto3==1.4.4
 botocore==1.5.7           # via boto3, s3transfer
+btrees==4.4.1             # via dedupe, zope.index
+canonicalize==1.3         # via dedupe
+categorical-distance==1.9  # via dedupe
 cchardet==1.1.2           # via tabulator
 chardet==2.3.0            # via normality
 click==6.7                # via jsontableschema, python-dotenv, tabulator
 contextlib2==0.5.4        # via raven
 datapackage==0.6.1
 dataset==0.7.1
+dedupe-hcluster==0.3.2    # via dedupe
+dedupe==1.6.1
 docutils==0.13.1          # via botocore
+doublemetaphone==0.1      # via dedupe
 et-xmlfile==1.0.1         # via openpyxl
 ezodf==0.3.2              # via tabulator
+fastcluster==1.1.22       # via dedupe
 functools32==3.2.3.post2  # via jsonschema
-future==0.16.0            # via jsontableschema
+future==0.16.0            # via dedupe, dedupe-hcluster, jsontableschema, rlr
 futures==3.0.5            # via s3transfer
 fuzzywuzzy==0.14.0
+haversine==0.4.5          # via dedupe
+highered==0.2.1           # via dedupe
 ijson==2.3                # via tabulator
-iso3166==0.7
+iso3166==0.8
 jdcal==1.3                # via openpyxl
 jmespath==0.9.1           # via boto3, botocore
 jsonlines==1.1.0          # via tabulator
-jsonschema==2.5.1         # via datapackage, jsontableschema
+jsonschema==2.6.0         # via datapackage, jsontableschema
 jsontableschema-sql==0.5.0
 jsontableschema==0.9.0    # via datapackage, jsontableschema-sql
+levenshtein-search==1.4.2  # via dedupe
 linear-tsv==1.0.0         # via tabulator
 lxml==3.7.2               # via tabulator
 mako==1.0.6               # via alembic
 markupsafe==0.23          # via mako
 normality==0.3.9          # via dataset
+numpy==1.12.0             # via canonicalize, categorical-distance, dedupe-hcluster, highered, pyhacrf-datamade, pylbfgs, rlr, simplecosine
 openpyxl==2.4.2           # via tabulator
 packaging==16.8           # via setuptools
+persistent==4.2.2         # via btrees, zope.index
 psycopg2==2.6.2
 pybossa-client==1.1.1
+pyhacrf-datamade==0.2.0   # via highered
+pylbfgs==0.2.0.3          # via pyhacrf-datamade, rlr
 pyparsing==2.1.10         # via packaging
 pypdf2==1.26.0
 python-dateutil==2.6.0    # via botocore, jsontableschema, python-documentcloud
 python-documentcloud==1.0.4
-python-dotenv==0.6.2
+python-dotenv==0.6.3
 python-editor==1.0.3      # via alembic
 python-levenshtein==0.12.0
 pyyaml==3.12              # via dataset
 raven==5.32.0
 requests==2.13.0
 rfc3986==0.4.1            # via jsontableschema
 rfc3987==1.3.7            # via python-documentcloud
+rlr==2.4                  # via dedupe
 s3transfer==0.1.10        # via boto3
-six==1.10.0               # via datapackage, dataset, jsonlines, jsontableschema-sql, linear-tsv, normality, packaging, python-dateutil, python-documentcloud, setuptools, tabulator
+simplecosine==1.1         # via dedupe
+simplejson==3.10.0        # via dedupe
+six==1.10.0               # via datapackage, dataset, jsonlines, jsontableschema-sql, linear-tsv, normality, packaging, python-dateutil, python-documentcloud, setuptools, tabulator, zope.index
 sqlalchemy==1.1.5
 tabulator==0.14.0         # via datapackage, jsontableschema, jsontableschema-sql
 unicodecsv==0.14.1        # via datapackage, jsontableschema, tabulator
+unidecode==0.4.20
 xlrd==1.0.0               # via tabulator
+zope.index==4.2.0         # via dedupe
+zope.interface==4.3.3     # via btrees, persistent, zope.index
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools                # via python-levenshtein
+# setuptools                # via python-levenshtein, zope.index, zope.interface

tests/conftest.py

@@ -45,6 +45,7 @@ def conn(request):
     api_session = APISession()
     WarehouseSession = sessionmaker(bind=conn['warehouse'].engine)
     warehouse_session = WarehouseSession()
+
     def teardown():
         truncate_database(conn['database'].engine)
         truncate_database(conn['warehouse'].engine)