diff --git a/processors/base/helpers/__init__.py b/processors/base/helpers/__init__.py index 94d9a69..7a02d98 100644 --- a/processors/base/helpers/__init__.py +++ b/processors/base/helpers/__init__.py @@ -13,20 +13,19 @@ import datetime import urlparse import csv +import unidecode import fuzzywuzzy.fuzz import fuzzywuzzy.process import iso3166 +import dedupe from . import pybossa_tasks_updater logger = logging.getLogger(__name__) PyBossaTasksUpdater = pybossa_tasks_updater.PyBossaTasksUpdater - # Module API -EDIT_DISTANCE_THRESHOLD = 75 - def get_variables(object, filter=None): """Exract variables from object to dict using name filter. @@ -52,7 +51,6 @@ def slugify_string(string): class JSONEncoder(json.JSONEncoder): """JSON encoder with datetime, date set support. """ - def default(self, obj): if isinstance(obj, datetime.datetime): return obj.strftime('%Y-%m-%dT%H:%M:%SZ') @@ -301,6 +299,7 @@ def get_canonical_location_name(location): # Extracted from: https://github.com/datasets/country-codes/blob/master/data/country-codes.csv CSV_PATH = os.path.join(os.path.dirname(__file__), 'data/countries.csv') DISTANCE_SCORER = fuzzywuzzy.fuzz.token_sort_ratio + EDIT_DISTANCE_THRESHOLD = 75 cleaned_location = remove_string_punctuation(location) current_match = location @@ -315,8 +314,10 @@ def get_canonical_location_name(location): relevant_info = [unicode(country[field], encoding='utf-8') for field in reader.fieldnames[0:5]] - location_info = [remove_string_punctuation(location_info) for location_info in relevant_info] - _, score = fuzzywuzzy.process.extractOne(cleaned_location, location_info, scorer=DISTANCE_SCORER) + location_info = [remove_string_punctuation(location_info) + for location_info in relevant_info] + _, score = fuzzywuzzy.process.extractOne( + cleaned_location, location_info, scorer=DISTANCE_SCORER) if score > current_score: current_match = iso3166.countries.get(country['ISO3166-1-Alpha-3']).name @@ -329,3 +330,88 @@ def get_canonical_location_name(location): logger.debug('Location "%s" normalized as "%s"', cleaned_location, current_match) return current_match + + +def get_canonical_organisation_name(conn, organisation): + """Find a canonical organisation name according to + a pre-built set of clusters + + Args: + conn (dict): connection dict + organisation (str): the organisation to be normalized + """ + CLUSTER_QUERY = 'SELECT canonical FROM organisation_clusters ' + \ + 'WHERE :organisation=ANY(variations)' + + normalized_form = organisation + + # Try to find the organisation in some cluster + try: + normalized_form = conn['warehouse'].query(CLUSTER_QUERY, + organisation=organisation).next()['canonical'] + logger.debug('Organisation "%s" normalized as "%s"', organisation, normalized_form) + except StopIteration: + logger.debug('Organisation "%s" not normalized', organisation) + + return normalized_form + + +def _dedup_cluster(cluster_entries): + """Sample, train and build organisation clusters. + Labeled training example used for clustering purposes + is stored in data/organisation_training_data.json and + can be regenerated using Dedupe consoleLabel tool + See: http://dedupe.readthedocs.io/en/latest/API-documentation.html#consoleLabel + + Args: + cluster_entries (list): a list of candidates to + be grouped in equivalent sets + """ + SAMPLE_SIZE = 10000 + MATCH_THRESHOLD = 0.75 + TRAINING_FILE = os.path.join(os.path.dirname(__file__), + 'data/organisation_training_data.json') + + fields = [{'field': 'name', 'type': 'String'}] + deduper = dedupe.Dedupe(fields) + deduper.sample(cluster_entries, SAMPLE_SIZE) + + with open(TRAINING_FILE) as training_file: + deduper.readTraining(training_file) + + deduper.train() + + return deduper.match(cluster_entries, MATCH_THRESHOLD) + + +def update_organisation_clusters(conn): + """Build a readable list of organisation clusters to be used + when normalizing new entries + + Args: + conn (dict): connection dict + """ + logger.debug('Recomputing organisations cluster data for further access') + + conn['warehouse'].begin() + conn['warehouse']['organisation_clusters'].delete() + + stored_organisations = list(conn['database']['organisations'].all()) + + cluster_members = {entry['id']: {'name': unidecode.unidecode(entry['name'])} + for entry in stored_organisations} + + clustered_dupes = _dedup_cluster(cluster_members) + + for (_, cluster) in enumerate(clustered_dupes): + id_set = cluster[0] + cluster_membership = [str(cluster_members[c]['name']) for c in id_set] + + cluster = { + 'canonical': max(cluster_membership, key=len).replace("'", "''"), + 'variations': cluster_membership + } + + conn['warehouse']['organisation_clusters'].insert(cluster) + + conn['warehouse'].commit() diff --git a/processors/base/helpers/data/organisation_training_data.json b/processors/base/helpers/data/organisation_training_data.json new file mode 100644 index 0000000..bc1bcda --- /dev/null +++ b/processors/base/helpers/data/organisation_training_data.json @@ -0,0 +1 @@ +{"distinct": [{"__class__": "tuple", "__value__": [{"": null, "slug": "vejle_hospital", "name": "vejle hospital", "id": "acbce76f-4566-4480-b1d9-e6b09b7c9c66"}, {"": null, "slug": "abteilung_f_r_an_sthesie_intensiv_und_schmerzmedizin_wilhelminenspital_der_stadt_wien", "name": "abteilung fur anasthesie, intensiv- und schmerzmedizin, wilhelminenspital der stadt wien", "id": "603c1f5a-38fa-4ebd-a4a3-34994a98c4bb"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "rgshospitalet", "name": "rgshospitalet", "id": "bbc263d9-ca8f-4fa8-9eee-c55046b5a00e"}, {"": null, "slug": "rikshospitalet", "name": "rikshospitalet", "id": "95284310-8707-4697-ae32-b0baf209234a"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "cog", "name": "cog", "id": "fb22fcd9-6e10-46ca-8c5b-e91fa89c9ccc"}, {"": null, "slug": "the_clinical_research_centre_rcsi_education_and_research_centre", "name": "the clinical research centre, rcsi education and research centre", "id": "ed85ec10-7b68-4063-93ee-367e4172a8fe"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "cantacuzino_national_institute_of_research_and_development_for_microbiology_and_immunology", "name": "cantacuzino national institute of research and development for microbiology and immunology", "id": "a4fd4f24-7255-4175-b9ce-916dba0f9161"}, {"": null, "slug": "artbc", "name": "artbc", "id": "e33662f6-111c-484e-90c7-dd4c29665375"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "takeda_belgium", "name": "takeda belgium", "id": "1f95d3ff-53af-4aec-b4a9-44685de7e6c6"}, {"": null, "slug": "karolinska_institutet_department_of_laboratory_medicine_clinical_physiology_c1_82_karolinska_univ_hosp_huddinge", "name": "karolinska institutet, department of laboratory medicine, clinical physiology c1:82,karolinska univ. hosp., huddinge", "id": "a6ac3ddb-4262-4bc0-b724-41727d6e68a1"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "department_of_medicine_i_university_hospital_st_josef_hospital_ruhr_university_bochum", "name": "department of medicine i, university hospital st. josef-hospital, ruhr university bochum", "id": "32f14029-d2e7-4b3b-8e79-6951ae0e26ce"}, {"": null, "slug": "finox_ag", "name": "finox ag", "id": "3f4da7a9-e745-4bc1-9384-dee2274a2ab8"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "university_of_turku", "name": "university of turku", "id": "90b1b931-4335-4914-a8b7-154e9a2ec266"}, {"": null, "slug": "university_of_hertfordshire", "name": "university of hertfordshire", "id": "69fd2257-0edd-41e9-87ce-db4a5c3e5144"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "servier_research_and_development_ictr_for_united_kingdom_only", "name": "servier research and development ictr (for united kingdom only)", "id": "510e1e01-3c6b-4beb-a646-bda21b3c8013"}, {"": null, "slug": "service_desk", "name": "service desk", "id": "4bc8802c-5b07-4c6d-a4db-f2edd0f68d85"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "nhs_greater_glasgow_and_clyde", "name": "nhs greater glasgow and clyde", "id": "046d6658-d5b1-4caa-8e4b-a52c60c09ffd"}, {"": null, "slug": "greater_glasgow_and_clyde", "name": "greater glasgow and clyde", "id": "613c1a9b-76b3-42ee-8979-2c8d31737f57"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "molteni", "name": "molteni", "id": "382a6dc5-0d89-4add-a86a-3f320c9289fd"}, {"": null, "slug": "medizinische_universit_t_wien_universit_tsklinik_f_r_innere_medizin_3_abteilung_f_r_gastroenterologie_und_hepatologie", "name": "medizinische universitat wien, universitatsklinik fur innere medizin 3, abteilung fur gastroenterologie und hepatologie", "id": "22dcd280-772d-451a-a261-cbef7a27c26c"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "university_of_essen", "name": "university of essen", "id": "b134ac14-8681-4bbc-a87d-4c4a479265ce"}, {"": null, "slug": "university_of_cincinnati_academic_medical_center_department_of_neurology", "name": "university of cincinnati - academic medical center -department of neurology", "id": "9559be98-7795-4a89-99c5-91e471ca09c8"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "servicio_de_cirug_a_general_y_digestiva_hospital_universitario_de_bellvitge", "name": "servicio de cirugia general y digestiva hospital universitario de bellvitge", "id": "262e70eb-f592-4b5f-8e06-12a5152ab7c9"}, {"": null, "slug": "gercor", "name": "gercor", "id": "f1d7cb83-6a05-46a3-b6bc-98e6330b2e8c"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "universit_tsklinikum_aachen", "name": "universitatsklinikum aachen", "id": "499f6fb9-3fb0-465d-9455-f34cddb4fc9b"}, {"": null, "slug": "universit_tsklinikum_freiburg_vertreten_durch_den_leitenden_rztlichen_direktor", "name": "universitatsklinikum freiburg, vertreten durch den leitenden arztlichen direktor", "id": "b92d234b-0791-4116-8b8e-f8276b862029"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "odense_university_hospital_department_m", "name": "odense university hospital, department m", "id": "d0aeb96f-5b11-4b42-816d-69232a6b5d89"}, {"": null, "slug": "aarhus_university_hospital_department_m", "name": "aarhus university hospital, department m", "id": "8256073c-6fbe-11e6-abe8-0242ac12000b"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "cambridge_university_hospitals_nhs_foundation_trust_addenbrooke_s_hospital", "name": "cambridge university hospitals nhs foundation trust, addenbrooke's hospital", "id": "e9951d7b-ee02-4804-aca2-6e2b99549974"}, {"": null, "slug": "camall", "name": "camall", "id": "9c050f54-7160-11e6-a0cd-0242ac12000b"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "amneal_pharms_co", "name": "amneal pharms co", "id": "2ebc30fa-7145-11e6-a0cd-0242ac12000b"}, {"": null, "slug": "amneal_pharms_ny", "name": "amneal pharms ny", "id": "eee857cc-7141-11e6-a0cd-0242ac12000b"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "institution_of_paediatric_anaesthesia_and_intensiv_care_astrid_lindgren_childrens_hospital_karolin", "name": "institution of paediatric anaesthesia and intensiv care, astrid lindgren childrens hospital, karolin", "id": "48024dc3-3391-43e7-9f84-ffd02e53a977"}, {"": null, "slug": "scs", "name": "scs", "id": "e1b1c73c-7146-11e6-a0cd-0242ac12000b"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "hospital_universitari_de_bellvitge_idibell", "name": "hospital universitari de bellvitge - idibell", "id": "302c2638-8bd6-11e6-a449-0242ac12000b"}, {"": null, "slug": "hospira", "name": "hospira", "id": "2c899cf4-2f23-4023-b9ab-f8d5356588d9"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "pharmamar_sau", "name": "pharmamar sau", "id": "a81bbb65-9b77-48ec-9afc-97b008ea886e"}, {"": null, "slug": "pharma_mar_sa_unipersonal", "name": "pharma mar sa unipersonal", "id": "90979692-2a94-4a4b-9c9c-f6c047edea3e"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "sanofi_aventis_ag", "name": "sanofi-aventis ag", "id": "21db6c72-5da2-4c64-b673-432321c1b314"}, {"": null, "slug": "de`veloppement", "name": "sanofi-aventis recherche &", "id": "c783b132-b299-4872-bd1f-6e8e14938814"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "anrs", "name": "anrs", "id": "e944e7a9-f50b-4f25-bcf5-7daefb39030b"}, {"": null, "slug": "klinikum_der_universit_t_m_nchen_kinderklinik_und_poliklinik_im_dr_von_haunersches_kinderspital", "name": "klinikum der universitat munchen, kinderklinik und poliklinik im dr. von haunersches kinderspital", "id": "f6959636-eb34-484b-8514-f49f9e548af3"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "university_hospital_ulm", "name": "university hospital ulm", "id": "1cf1b0d8-2fa0-4d08-a1a9-ef176b6a1973"}, {"": null, "slug": "university_hospital_erlangen", "name": "university hospital erlangen", "id": "58384182-25d9-414e-8817-b3450bdd9f86"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "imperial_college_london_joint_research_compliance_office", "name": "imperial college london, joint research compliance office", "id": "9a928883-62f4-40e8-bf1e-c36ca297350b"}, {"": null, "slug": "imperial_ciollege", "name": "imperial ciollege", "id": "8433852c-4927-4c6a-a156-dce63bfd0815"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "akh_wien_klinik_f_r_innere_medizin_i_onkologie", "name": "akh wien, klinik fur innere medizin i, onkologie", "id": "1eec3f23-406c-471b-948a-24dcf52e62a1"}, {"": null, "slug": "akh_linz", "name": "akh linz", "id": "ce11fc95-3d22-4c28-94dc-35be2d9f6091"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "universit_tsklinik_f_r_zahn_mund_und_kieferheilkunde_med_uni_graz_department_f_r_zahn_rztliche_chirurgie", "name": "universitatsklinik fur zahn-, mund- und kieferheilkunde, med uni graz, department fur zahnarztliche chirurgie", "id": "b152d898-42dc-4921-9c80-8b64e81118a9"}, {"": null, "slug": "cerepp", "name": "cerepp", "id": "bcb33d21-b979-4fbd-8657-abd096c96ec6"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "merck_serono_sa_geneva_an_affiliate_of_merck_kgaa_darmstadt_germany", "name": "merck serono sa - geneva, an affiliate of merck kgaa, darmstadt, germany", "id": "7e7f14c2-9ede-49fa-aa4c-1c3644b22daf"}, {"": null, "slug": "taro", "name": "taro", "id": "019c34b0-7129-11e6-a89e-0242ac12000b"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "icin", "name": "icin", "id": "9bec9948-4662-436f-9fe4-9c3aef6c4ffc"}, {"": null, "slug": "university_of_oslo_faculty_of_medicine_institute_of_health_and_society", "name": "university of oslo, faculty of medicine, institute of health and society", "id": "5b3734a2-82e2-4c16-9c4d-66d561ed0d51"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "solvay", "name": "solvay", "id": "e6e76398-715e-11e6-a0cd-0242ac12000b"}, {"": null, "slug": "universit_tsklinik_f_r_psychiatrie_und_psychotherapie_klin_abteilung_f_biologische_psychiatrie", "name": "universitatsklinik fur psychiatrie und psychotherapie, klin. abteilung f. biologische psychiatrie", "id": "3ab32bf7-811c-42fa-8337-eb49e15069f8"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "medizinische_universit_t_wien_abteilung_f_r_allgemeine_an_sthesie_und_intensivmedizin", "name": "medizinische universitat wien, abteilung fur allgemeine anasthesie und intensivmedizin", "id": "dbb52f33-a8fc-4aa5-b128-948fe7582742"}, {"": null, "slug": "alza", "name": "alza", "id": "a81351d0-71a0-11e6-a0cd-0242ac12000b"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "azienda_ospedaliera_senese", "name": "azienda ospedaliera senese", "id": "42587571-741a-4e64-b485-4deee66c414c"}, {"": null, "slug": "azienda_ospedaliera_san_paolo", "name": "azienda ospedaliera san paolo", "id": "63bac87d-d6a3-4be4-9eef-4efa52f0c0a5"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "jazz", "name": "jazz", "id": "85d2f332-7335-11e6-a0cd-0242ac12000b"}, {"": null, "slug": "kompetenznetz_vorhofflimmern_e_v_atrial_fibrillation_competence_network_afnet_e_v", "name": "kompetenznetz vorhofflimmern e.v. [atrial fibrillation competence network (afnet) e.v.]", "id": "fc21ea5c-d162-46f5-bd49-bedac7f5851d"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "gcllsg_dept_of_internal_medicine_i_university_hospital_cologne", "name": "gcllsg - dept. of internal medicine i, university hospital cologne", "id": "23e32f33-64c2-48d7-98ad-bda4e558f1cc"}, {"": null, "slug": "oys", "name": "oys", "id": "c8194b0b-d077-4ec6-bae6-5e27ea8ba8c4"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "nopho", "name": "nopho", "id": "57164cff-6f3b-448f-a2a2-cb44b3def10b"}, {"": null, "slug": "pr_giampiero_campanelli_universit_di_milano_dipartimento_di_scienze_chirurgiche_ospedale_maggior", "name": "pr giampiero campanelli universita di milano, dipartimento di scienze chirurgiche, ospedale maggior", "id": "566f2525-88ca-4ad9-a985-788e49f4240e"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "trigen", "name": "trigen", "id": "b12f961f-65f0-45b2-bb50-b17bc54604b1"}, {"": null, "slug": "trius_therapeutics_a_wholly_owned_subsidiary_of_cubist", "name": "trius therapeutics, a wholly owned subsidiary of cubist", "id": "2313c489-596d-4d80-8c94-aa42d3a2f015"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "centre_hospitalier_intercommunal_de_toulon_la_seyne_sur_mer", "name": "centre hospitalier intercommunal de toulon - la seyne sur mer", "id": "7eb33d6a-b44c-48d4-8dcd-f18317ecdcc3"}, {"": null, "slug": "wendt", "name": "wendt", "id": "0186bc10-7161-11e6-a0cd-0242ac12000b"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "universit_tsklinikum_heidelberg", "name": "universitatsklinikum heidelberg", "id": "a5592bcf-b2a9-4eef-87b4-32f428e1cd67"}, {"": null, "slug": "klinik und poliklinik fur psychiatrie und psychotherapie", "name": "universitatsklinikum bonn", "id": "db0cf76f-03db-4f45-95e7-853b95e8ea9b"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "eortc_european_organisation_for_the_research_and_treatment_of_cancer", "name": "eortc-european organisation for the research and treatment of cancer", "id": "e41b5c84-e4f0-430d-a8dd-ca88bfb595c4"}, {"": null, "slug": "solti", "name": "solti", "id": "cd03b2c6-847a-491f-8428-3060750c8336"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "centre_hospitalier_universitaire_d_angers", "name": "centre hospitalier universitaire d'angers", "id": "025af88b-9529-4444-90b5-86481ce363f0"}, {"": null, "slug": "centre_hospitalier_universitaire_de_nancy", "name": "centre hospitalier universitaire de nancy", "id": "e37ff462-7351-402c-828d-d17f23d6fa72"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "medizinische_universit_t_innsbruck_universit_tsklinik_f_r_innere_medizin_i", "name": "medizinische universitat innsbruck, universitatsklinik fur innere medizin i", "id": "12e2a808-1753-47e6-920b-91e744004e00"}, {"": null, "slug": "medizinische_universit_t_wien_universit_tsklinik_f_r_innere_medizin_3_abteilung_f_r_gastroenterologie_und_hepatologie", "name": "medizinische universitat wien, universitatsklinik fur innere medizin 3, abteilung fur gastroenterologie und hepatologie", "id": "22dcd280-772d-451a-a261-cbef7a27c26c"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "reade", "name": "reade", "id": "8288b4e1-f4a9-4fed-a996-813ec40471aa"}, {"": null, "slug": "merck_sharp_dohme_corp_a_subsidiary_of_merck_co_inc_referred_to_as_merck", "name": "merck sharp & dohme corp., a subsidiary of merck & co., inc. (referred to as merck)", "id": "98ff4a8d-9011-429f-9c2f-4eef1084cbc6"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "laboratoires_merck_sharp_dohme_chibret", "name": "laboratoires merck sharp & dohme - chibret", "id": "e12cc357-91a1-4ab2-b87c-8a5b71225095"}, {"": null, "slug": "laboratoires_iprad", "name": "laboratoires iprad", "id": "040a1a3b-306a-4a08-be78-b178e1d033da"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "university_of_p_cs", "name": "university of pecs", "id": "af3ad0cf-6349-4c9b-9761-a57815b4f7c5"}, {"": null, "slug": "university_of_plymouth", "name": "university of plymouth", "id": "468a4598-0f63-4c9e-a9fe-9e53d5416109"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "amgen_s_a", "name": "amgen s.a", "id": "2fc05d7c-54ee-45c1-a844-bc160cb92e68"}, {"": null, "slug": "amgen_inc_one_amgen_center_drive_thousand_oaks_california_91320", "name": "amgen, inc. one amgen center drive thousand oaks, california 91320", "id": "75d02ed4-527a-4c3a-83a0-e15e97fc7866"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "free_state_of_bavaria_represented_by_the_university_of_regensburg_represented_by_prof_edward_k_geissler", "name": "free state of bavaria, represented by the university of regensburg, represented by prof. edward k. geissler", "id": "3d6ade49-c992-4715-9594-000bd02e5eb1"}, {"": null, "slug": "futurum", "name": "futurum", "id": "399faf0c-52c9-4ee0-9025-873e502aab24"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "fundaci_n_para_la_investigaci_n_biom_dica_hospital_gregorio_mara_n", "name": "fundacion para la investigacion biomedica hospital gregorio maranon", "id": "fafbf14a-7edf-4f4d-a87f-703785475b5c"}, {"": null, "slug": "fundacion_pethema", "name": "fundacion pethema", "id": "d7afcd5b-0be6-4039-a08b-52e26e0f5e64"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "cnic", "name": "cnic", "id": "13d37a6e-a22f-454d-8c96-519556903e54"}, {"": null, "slug": "professor_hans_erik_johnsen_md_dmsc_department_of_haematology_aalborg_hospital", "name": "professor hans erik johnsen, md, dmsc, department of haematology, aalborg hospital", "id": "7e791d01-4f46-4ceb-abb6-a5863262e615"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "university_of_oslo_institute_for_basic_medical_sciences_department_of_nutrition", "name": "university of oslo, institute for basic medical sciences, department of nutrition", "id": "d400ed48-6133-4340-94ec-e93df8540486"}, {"": null, "slug": "university_of_surrey", "name": "university of surrey", "id": "f1549a9a-52e0-4629-86bc-c853a0a91aa7"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "universit_tsklinik_f_r_innere_medizin_iii_klinische_abteilung_f_r_rheumatologie_medizinische_universit_t_wien", "name": "universitatsklinik fur innere medizin iii, klinische abteilung fur rheumatologie, medizinische universitat wien", "id": "daf847ac-432d-4bf6-a247-4298f0fa9e13"}, {"": null, "slug": "universit_tsklinkum_erlangen", "name": "universitatsklinkum erlangen", "id": "ffcd68a4-22bd-48b7-b48b-e5bbb42c4c31"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "aesrx", "name": "aesrx", "id": "7ff80805-62e2-4470-b0b0-87440be5e547"}, {"": null, "slug": "gel_tamo_cooperative_spanish_group_of_lymphoma_autologous_bone_marrow_transplant", "name": "gel/tamo (cooperative spanish group of lymphoma/autologous bone marrow transplant)", "id": "74823312-7a82-4064-89ef-0e812f9d60d3"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "vu_university_medical_centre", "name": "vu university medical centre", "id": "d24a8fcd-b259-45c2-8a36-5604474c370e"}, {"": null, "slug": "university_medical_centre_st_radboud_department_of_dermatology", "name": "university medical centre st. radboud, department of dermatology", "id": "3bbe38b9-b3f4-4d58-811e-8dba91c20f92"}]}], "match": [{"__class__": "tuple", "__value__": [{"": null, "slug": "novartis_farmaceutica", "name": "novartis farmaceutica", "id": "07d428a0-cf4e-4f85-9961-fc028370360e"}, {"": null, "slug": "novartis_farmac_utica", "name": "novartis farmaceutica", "id": "bcb0e637-6442-4916-88f9-8cd0d5df3534"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "merck_serono_international_a_branch_of_laboratories_serono_s_a", "name": "merck serono international, a branch of laboratories serono s.a", "id": "1a03f05f-1e23-469f-adc4-6156f3bc81aa"}, {"": null, "slug": "merck_serono_international_a_branch_of_laboratoires_serono_s_a", "name": "merck serono international, a branch of laboratoires serono s.a", "id": "9516dca0-e67e-443e-ae03-feeb289be70c"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "glaxosmithkline_a_e_b_e", "name": "glaxosmithkline a.e.b.e", "id": "11145a7d-a298-4cf6-ae4b-46d2af4892d4"}, {"": null, "slug": "glaxosmithkline_ab", "name": "glaxosmithkline ab", "id": "2df0fbc6-3b70-46ef-bfe4-035678c0f668"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "fidia_farmaceutici_spa", "name": "fidia farmaceutici spa", "id": "6e29d46e-fef6-4308-b00f-34243d2810ee"}, {"": null, "slug": "fidia_farmaceutici_s_p_a", "name": "fidia farmaceutici s.p.a", "id": "75ca420a-d39f-494a-9c9a-098d855b7ff7"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "pfizer_inc_east_42nd_street_new_york_ny10017_usa", "name": "pfizer inc, east 42nd street, new york, ny10017, usa", "id": "1efa9e61-6090-40bb-8daa-1ea7310830f4"}, {"": null, "slug": "pfizer_inc_235_east_42nd_street_new_york_ny10017", "name": "pfizer inc 235 east 42nd street, new york, ny10017", "id": "dbdc03f8-f997-4242-91be-566d7f5870a5"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "fundaci_n_para_la_investigaci_n_biomedica_del_hospital_12_de_octubre", "name": "fundacion para la investigacion biomedica del hospital 12 de octubre", "id": "cf84194a-b23a-4b1f-87a1-24b1fdec7e48"}, {"": null, "slug": "fundacion_para_la_investigacion_biomedica_hospital_universitario_12_de_octubre", "name": "fundacion para la investigacion biomedica hospital universitario 12 de octubre", "id": "c86fa310-e3d1-4bf7-be4a-c9fed815a879"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "medizinische_universit_t_wien_abteilung_f_r_allgemeine_an_sthesie_und_intensivmedizin", "name": "medizinische universitat wien, abteilung fur allgemeine anasthesie und intensivmedizin", "id": "dbb52f33-a8fc-4aa5-b128-948fe7582742"}, {"": null, "slug": "medizinische_universit_t_wien_abteilung_f_r_interne_medizin_i", "name": "medizinische universitat wien, abteilung fur interne medizin i", "id": "1de8b2fe-fdec-4fc9-be80-679c9ac33ce2"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "johnson_johnson_pharmaceutical_research_and_development_l_l_c", "name": "johnson & johnson pharmaceutical research and development, l.l.c", "id": "5e39e1cf-f761-4acc-bd2c-247647d86e60"}, {"": null, "slug": "johnson_and_johnson_pharmaceutical_research_and_development_una_divisi_n_de_janssen_cilag_s_a", "name": "johnson and johnson pharmaceutical research and development, una division de janssen-cilag, s.a", "id": "e6d2bf79-88b5-4e40-aa2b-a36c8a77b81d"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "fondazione_michelangelo_onlus_avanzamento_dello_studio_e_cura_dei_tumori", "name": "fondazione michelangelo onlus - avanzamento dello studio e cura dei tumori", "id": "dc1c972c-880f-11e6-a998-0242ac12000b"}, {"": null, "slug": "fondazione_michelangelo_avanzamento_dello_studio_e_della_cura_dei_tumori", "name": "fondazione michelangelo - avanzamento dello studio e della cura dei tumori", "id": "886dbcfd-131f-4dde-b49f-376738e72b32"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "radboud_university_medical_center", "name": "radboud university medical center", "id": "e0a9ebe2-090f-4317-bf41-30ea0e2e6a42"}, {"": null, "slug": "radboud_umc", "name": "radboud umc", "id": "97f181b3-9ad4-48ad-a41d-0342ef33b8c0"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "institut_de_recherches_internationales_servier", "name": "institut de recherches internationales servier", "id": "659a4aa2-7f2f-4ed8-918a-4c0718e095e4"}, {"": null, "slug": "i_r_i_s_institut_de_recherches_internationales_servier", "name": "i.r.i.s. institut de recherches internationales servier", "id": "7d798544-5e49-4b24-899f-fbc24edba20c"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "fachbereich_medizin_der_johann_wolfgang_goethe_universit_t_frankfurt_m", "name": "fachbereich medizin der johann wolfgang goethe-universitat frankfurt/m", "id": "505ca89c-289a-4ce1-a562-196a014465a2"}, {"": null, "slug": "dekan_des_fachbereichs_medizin_der_johann_wolfgang_goethe_universit_t_frankfurt_m", "name": "dekan des fachbereichs medizin der johann wolfgang goethe-universitat frankfurt/m", "id": "75bd6ca6-1cd0-449d-b2ee-2c5c83df2d75"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "pfizer_inc_235_east_42nd_streeet_new_york_ny_10017", "name": "pfizer inc., 235 east 42nd streeet, new york, ny 10017", "id": "0068e377-8873-41b9-baad-e3768b77d0b1"}, {"": null, "slug": "pfizer_inc_235_east_42nd_street_new_york_city_ny_10017", "name": "pfizer inc, 235 east 42nd street, new york city, ny 10017", "id": "dae5bd5c-6fa0-11e6-9663-0242ac12000b"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "sanofi_aventis_recherche_development", "name": "sanofi-aventis recherche & development", "id": "a935e55e-be0e-40d6-a84b-6d0e74a97880"}, {"": null, "slug": "sanofi_aventis_recherche_d2veloppement", "name": "sanofi-aventis recherche & d2veloppement", "id": "9c45d693-515d-4149-92e4-d7272d06a7df"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "wyeth_research_division_of_wyeth_pharmaceuticals_inc_clinical_research_development", "name": "wyeth research division of wyeth pharmaceuticals, inc. clinical research & development", "id": "1d53dcae-9b86-416e-9d92-a62412245dcc"}, {"": null, "slug": "wyeth_research_division_of_wyeth_pharmaceuticals_inc_clinical_reserach_and_development", "name": "wyeth research division of wyeth pharmaceuticals inc., clinical reserach and development", "id": "fe20c8b4-bfb0-4cfc-96be-52e8cd5c61f0"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "dainippon_sumitomo_pharma_america_inc", "name": "dainippon sumitomo pharma america, inc", "id": "df03d0e3-1345-473c-8400-e5dc3e505b99"}, {"": null, "slug": "dainippon_sumitomo_pharma_america_ltd", "name": "dainippon sumitomo pharma america ltd", "id": "dc11babf-a704-11e6-94cb-5656d04fc2b5"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "institut_de_recherches_internationales_servier_i_r_i_s", "name": "institut de recherches internationales servier (i.r.i.s)", "id": "8811d706-c986-44c5-bf01-3f0b35984b6c"}, {"": null, "slug": "institut_de_recherches_internationales_servier_iris", "name": "institut de recherches internationales servier ( iris )", "id": "9b368f8e-02db-4935-a1a6-f2a417efa7ee"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "cardiff_univeristy", "name": "cardiff univeristy", "id": "191a4121-096e-489f-bb60-9d9e340fdcf0"}, {"": null, "slug": "cardiff_university", "name": "cardiff university", "id": "d4686322-3206-4d52-8c13-30229e1d5dc5"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "merck_sharp_dohme_a_subsidiary_of_merck_co_inc", "name": "merck sharp & dohme a subsidiary of merck co., inc", "id": "62f4aae4-7c29-4485-a3bc-17345b365f76"}, {"": null, "slug": "merck_sharp_dohme_corp_a_subsidiary_of_merck_co_inc_n_j_usa", "name": "merck sharp & dohme corp., a subsidiary of merck & co., inc., n.j., usa", "id": "f262b660-2330-429c-8af0-63e5c4fec4b0"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "hull_and_east_yorkshire_nhs_hospital_trust", "name": "hull and east yorkshire nhs hospital trust", "id": "16f70329-dafd-4428-bec6-483f539f52fc"}, {"": null, "slug": "hull_and_east_yorkshire_hospitals_trust", "name": "hull and east yorkshire hospitals trust", "id": "2484bf94-1589-4af7-a881-77e0aae0ece2"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "university_hospital_grenoble", "name": "university hospital grenoble", "id": "55dc3af9-da50-4614-bdeb-59aef351101b"}, {"": null, "slug": "universityhospitalgrenoble", "name": "universityhospitalgrenoble", "id": "d9724382-7654-4ba8-baea-764670d9fe39"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "grupo_espa_ol_de_c_ncer_de_pulm_n", "name": "grupo espanol de cancer de pulmon", "id": "1a0b54ee-3c90-4fb0-a495-d2c36bae4a70"}, {"": null, "slug": "gecp_grupo_espa_ol_de_cancer_de_pulm_n", "name": "gecp(grupo espanol de cancer de pulmon)", "id": "fdeaddfd-d3eb-4620-879e-c7a5ebe251c5"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "pharma_mar_sa", "name": "pharma mar sa", "id": "5821cf6c-5e8a-481f-8ff8-3198a9183cea"}, {"": null, "slug": "pharma_mar_s_a", "name": "pharma mar, s.a", "id": "a004a205-f221-4197-83bd-138ffb93fce0"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "pfizer_limited_ramsgate_road_sandwich_kent_uk_ct13_9nj", "name": "pfizer limited, ramsgate road, sandwich, kent. uk. ct13 9nj", "id": "0ef4ae4f-e284-42c3-b72f-5c46cc59f3e2"}, {"": null, "slug": "pfizer_limited_ramsgate_road_sandwich_kent_ct13_9nj", "name": "pfizer limited, ramsgate road,sandwich,kent ct13 9nj", "id": "3438eb84-9448-480b-915c-1c1511546785"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "vu_univeristy_medical_centre", "name": "vu univeristy medical centre", "id": "b1e7a97f-3f72-43f4-a8f1-ab6902f55179"}, {"": null, "slug": "vu_university_medical_center", "name": "vu university medical center", "id": "a7bf6b0c-2367-49c7-ae4b-7cc850909a14"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "hull_and_east_yorkshire_hospitals_trust_research_and_deveopment_admin_porta_cabin", "name": "hull and east yorkshire hospitals trust, research and deveopment, admin porta cabin", "id": "8b4343ec-fb2f-4fb9-aa9f-044d60e28926"}, {"": null, "slug": "hull_and_east_yorkshire_nhs_trust", "name": "hull and east yorkshire nhs trust", "id": "595e80c5-ebc1-44d4-a496-00b180fea87e"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "hamilton_health_sciences_through_the_population_health_research_institute", "name": "hamilton health sciences through the population health research institute", "id": "ceecdbd1-b90f-4146-83b5-e987dcb62f70"}, {"": null, "slug": "hamilton_health_sciences_corporation_through_its_population_health_research_institute", "name": "hamilton health sciences corporation through its population health research institute", "id": "4ba23efe-04ef-4a5c-853f-235d35ab9a66"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "tolerx_inc", "name": "tolerx, inc", "id": "95db6177-b882-4aa7-8f44-d56c146461cf"}, {"": null, "slug": "tolerrx_inc", "name": "tolerrx, inc", "id": "496acb41-295a-43ee-9136-4d1fef65e2b7"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "domp_s_p_a", "name": "dompe s.p.a", "id": "ca2b02cc-8821-4f3e-899e-69c84148e0c9"}, {"": null, "slug": "dompe_s_p_a", "name": "dompe` s.p.a", "id": "e000d975-f1c4-48db-a8e3-8d6923c66a71"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "boehringer_ingelheim_pharma_ellas_ae", "name": "boehringer ingelheim pharma ellas ae", "id": "55ebc98a-6f9f-11e6-9663-0242ac12000b"}, {"": null, "slug": "boehringer_ingelheim_gmbh", "name": "boehringer ingelheim gmbh", "id": "b8695dd5-485c-422c-8f91-460057752ca2"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "pfizer_ltd_ramsgate_road_sandwich_kent_ct13_9nj", "name": "pfizer ltd,ramsgate road,sandwich,kent ct13 9nj", "id": "5a02f0f0-4b8e-4bcc-b5d9-2bda73ef9a30"}, {"": null, "slug": "pfizer_ltd_ramsgate_road_sandwich_kent_ct13_9nj_uk", "name": "pfizer ltd. ramsgate road, sandwich, kent, ct13 9nj, uk", "id": "773e090f-d432-4405-870d-7cc6cc5ee44e"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "charit_unversit_tsmedizin_berlin", "name": "charite - unversitatsmedizin berlin", "id": "ae7adb7b-c45f-4298-be68-d5dcaa36ea65"}, {"": null, "slug": "charite_universitaetsmedizin_berlin", "name": "charite universitaetsmedizin berlin", "id": "9cc141c9-55dc-4b03-84cc-245b7d5f45e4"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "bial_portela_c_sa", "name": "bial - portela & ca, sa", "id": "5ac2130a-6fbe-11e6-abe8-0242ac12000b"}, {"": null, "slug": "bial_portela_ca_sa", "name": "bial-portela & ca, sa", "id": "e0538c80-0c8f-415a-a43f-f3466054daf1"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "servicio_nefrologia_hospital_bellvitge", "name": "servicio nefrologia hospital bellvitge", "id": "a28cf75f-8b5e-45d6-8a74-cbd575537b0c"}, {"": null, "slug": "servicio_nefrolog_a_hospital_universitario_de_bellvitge", "name": "servicio nefrologia hospital universitario de bellvitge", "id": "90a15d23-27ae-4e52-be99-fc5f7027b6a6"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "bayer_hlthcare", "name": "bayer hlthcare", "id": "72fd5b3a-7191-11e6-a0cd-0242ac12000b"}, {"": null, "slug": "bayer_healthcare", "name": "bayer healthcare", "id": "e8216e75-b447-4252-bb49-7b895bcf4108"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "merck_serono_sa_geneva_an_affiliate_of_merck_kgaa", "name": "merck serono sa - geneva, an affiliate of merck kgaa,", "id": "64386d51-a811-451e-9352-45afeb68a88e"}, {"": null, "slug": "merck_serono_s_p_a", "name": "merck serono s.p.a", "id": "65a74622-52a5-44aa-b68a-4a0614c78010"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "pfizer_inc_235_east_42nd_road_new_york_ny_10017", "name": "pfizer inc., 235 east 42nd road, new york, ny 10017", "id": "93f242c4-1ea8-41a4-962a-cb4b07c428b5"}, {"": null, "slug": "pfizer_inc_235_east_42nd_street_new_york_ny10017_united_states", "name": "pfizer inc, 235 east 42nd street, new york, ny10017, united states", "id": "70a57a39-1462-4239-a163-1968d0d1a9d3"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "boehringer_ingelheim_noway_ks", "name": "boehringer ingelheim noway ks", "id": "1a23c45f-2c11-4e5d-a222-082def03ebc8"}, {"": null, "slug": "boehringer_ingelheim_lda", "name": "boehringer ingelheim, lda", "id": "e35f2092-3b29-4dad-b641-6df7bf1da76d"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "accord_healthcare_ltd", "name": "accord healthcare ltd", "id": "7555d4a5-8b3d-4db8-83d3-5482d0d01d4b"}, {"": null, "slug": "accord_hlthcare", "name": "accord hlthcare", "id": "dd277b94-7141-11e6-a0cd-0242ac12000b"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "boehringer_ingelheim_pharma_gmbh", "name": "boehringer ingelheim pharma gmbh", "id": "5b57c8bd-158c-42a5-bfa1-3323c67e0dd4"}, {"": null, "slug": "boehringer_ingelheim_pharmaceuticals_espa_a_s_a", "name": "boehringer ingelheim pharmaceuticals, espana s.a", "id": "6a82526b-a3f0-4364-b600-bf5233259d47"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "azienda_ospedaliera_senese", "name": "azienda ospedaliera senese", "id": "42587571-741a-4e64-b485-4deee66c414c"}, {"": null, "slug": "azienda_ospedaliera_di_reggio_emilia_arcispedale_s_maria_nuova_irccs", "name": "azienda ospedaliera di reggio emilia arcispedale s.maria nuova/irccs", "id": "95d3949b-2d87-4a68-827e-a08fc3c39978"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "klinikum_der_universit_t_m_nchen_campus_gro_hadern", "name": "klinikum der universitat munchen, campus grosshadern", "id": "42f5b998-ad55-4220-a0e8-55df5cebc81e"}, {"": null, "slug": "klinikum_der_universit_t_m_nchen_gro_hadern", "name": "klinikum der universitat munchen-grosshadern", "id": "089e5628-bb66-4808-818a-a79770a11d54"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "academisch_medisch_centrum_amsterdam", "name": "academisch medisch centrum amsterdam", "id": "974a5c1d-8651-4bb8-8b78-cce4a6850f9c"}, {"": null, "slug": "academisch_medisch_centrum_bij_de_universiteit_van_amsterdam_amc", "name": "academisch medisch centrum bij de universiteit van amsterdam (amc)", "id": "fb80c762-68d3-4689-a142-20a1ee41e6b9"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "merck_kgaa", "name": "merck kgaa", "id": "eb4cc9e6-aaa6-49ad-995f-a9f72f3bdd5d"}, {"": null, "slug": "mmerck_kgaa", "name": "mmerck kgaa", "id": "f28855cb-d949-44f5-8cdb-8d84d1d4ae7f"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "radboudumc", "name": "radboudumc", "id": "cfa82bb6-a7bf-45c7-88d2-6bb1b02661d6"}, {"": null, "slug": "radboud_umc", "name": "radboud umc", "id": "97f181b3-9ad4-48ad-a41d-0342ef33b8c0"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "schering_plough_ce_ag_hungarian_commercial_representative_office", "name": "schering-plough ce ag hungarian commercial representative office", "id": "5a1cfc63-9a21-4db9-a032-f417843cde75"}, {"": null, "slug": "schering_plough_france", "name": "schering-plough france", "id": "c046e1d5-a486-499c-9d6b-8bfc6fd1eeab"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "pulmonary_dept_gentofte_hospital", "name": "pulmonary dept. gentofte hospital", "id": "75672ec9-4f1d-4f39-b0d6-1e3242c52b4f"}, {"": null, "slug": "pulmonary_department_gentofte_hospital", "name": "pulmonary department, gentofte hospital", "id": "85b86a82-e503-4329-88c6-ffdf7328e582"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "nv_organon", "name": "nv organon", "id": "898e1916-aef6-4174-8433-fdb4c05e9c66"}, {"": null, "slug": "n_v_organon", "name": "n v organon", "id": "e853f524-9e97-4579-b8a2-f8c29eca6e92"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "consorci_parc_de_salut_mar", "name": "consorci parc de salut mar", "id": "2e06635b-2945-406f-8fe4-e9afcb19c015"}, {"": null, "slug": "consorci_mar_parc_de_salut_barcelona", "name": "consorci mar parc de salut barcelona", "id": "1d7da9c7-3809-404d-969d-f58cb1e18e25"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "nycomed_us", "name": "nycomed us", "id": "d3bc30d2-72da-11e6-a0cd-0242ac12000b"}, {"": null, "slug": "nycomed_headquarter", "name": "nycomed (headquarter)", "id": "4b7e57a0-5ce0-4447-bcee-a2ee9bb76b6b"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "galderma_research_and_development", "name": "galderma research and development", "id": "7eeea4e4-efc7-45bf-bac7-82d54c1ae894"}, {"": null, "slug": "galderma_research_development_inc", "name": "galderma research & development inc", "id": "65bfdb33-6e94-44cb-98e5-4ec4d0ef60a9"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "the_all_ireland_cooperative_oncology_research_group_icorg", "name": "the all-ireland cooperative oncology research group (icorg)", "id": "47c20cb7-42aa-40d6-b307-102844c8fedb"}, {"": null, "slug": "icorg_all_ireland_cooperative_oncology_research_group", "name": "icorg, all-ireland cooperative oncology research group", "id": "5e33332a-5eab-46e0-84d6-df15c6f32cd7"}]}, {"__class__": "tuple", "__value__": [{"": null, "slug": "pfizer_inc_235_east_42nd_streeet_new_york_ny_10017", "name": "pfizer inc., 235 east 42nd streeet, new york, ny 10017", "id": "0068e377-8873-41b9-baad-e3768b77d0b1"}, {"": null, "slug": "pfizer_inc_235_east_42nd_street_new_york_new_york_10017_usa", "name": "pfizer inc, 235 east 42nd street, new york, new york 10017, usa", "id": "83f77026-9bf3-42d7-bdf1-59a5d35a6b9b"}]}]} \ No newline at end of file diff --git a/processors/base/writers/organisation.py b/processors/base/writers/organisation.py index c8c3dc9..d84a2e1 100644 --- a/processors/base/writers/organisation.py +++ b/processors/base/writers/organisation.py @@ -33,7 +33,7 @@ def write_organisation(conn, organisation, source_id, trial_id=None): timestamp = datetime.datetime.utcnow() # Get name - name = helpers.clean_string(organisation['name']) + name = helpers.get_canonical_organisation_name(organisation['name']) if len(name) <= 1: return None diff --git a/processors/normalization_clusters_updater/__init__.py b/processors/normalization_clusters_updater/__init__.py new file mode 100644 index 0000000..b211373 --- /dev/null +++ b/processors/normalization_clusters_updater/__init__.py @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from .processor import process diff --git a/processors/normalization_clusters_updater/processor.py b/processors/normalization_clusters_updater/processor.py new file mode 100644 index 0000000..a0fb2d0 --- /dev/null +++ b/processors/normalization_clusters_updater/processor.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import processors.base.helpers + + +# Module API + +def process(conf, conn): + """Update clusters used when normalizing trials entities + """ + processors.base.helpers.update_organisation_clusters(conn) diff --git a/requirements.in b/requirements.in index 2d66c1f..d158d27 100644 --- a/requirements.in +++ b/requirements.in @@ -13,3 +13,5 @@ fuzzywuzzy python-Levenshtein iso3166 raven +unidecode +dedupe diff --git a/requirements.txt b/requirements.txt index a4606e5..dd95bf5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,45 +4,60 @@ # # pip-compile --output-file requirements.txt requirements.in # +affinegap==1.10 # via canonicalize, dedupe alembic==0.8.10 # via dataset appdirs==1.4.0 # via setuptools boto3==1.4.4 botocore==1.5.7 # via boto3, s3transfer +btrees==4.4.1 # via dedupe, zope.index +canonicalize==1.3 # via dedupe +categorical-distance==1.9 # via dedupe cchardet==1.1.2 # via tabulator chardet==2.3.0 # via normality click==6.7 # via jsontableschema, python-dotenv, tabulator contextlib2==0.5.4 # via raven datapackage==0.6.1 dataset==0.7.1 +dedupe-hcluster==0.3.2 # via dedupe +dedupe==1.6.1 docutils==0.13.1 # via botocore +doublemetaphone==0.1 # via dedupe et-xmlfile==1.0.1 # via openpyxl ezodf==0.3.2 # via tabulator +fastcluster==1.1.22 # via dedupe functools32==3.2.3.post2 # via jsonschema -future==0.16.0 # via jsontableschema +future==0.16.0 # via dedupe, dedupe-hcluster, jsontableschema, rlr futures==3.0.5 # via s3transfer fuzzywuzzy==0.14.0 +haversine==0.4.5 # via dedupe +highered==0.2.1 # via dedupe ijson==2.3 # via tabulator -iso3166==0.7 +iso3166==0.8 jdcal==1.3 # via openpyxl jmespath==0.9.1 # via boto3, botocore jsonlines==1.1.0 # via tabulator -jsonschema==2.5.1 # via datapackage, jsontableschema +jsonschema==2.6.0 # via datapackage, jsontableschema jsontableschema-sql==0.5.0 jsontableschema==0.9.0 # via datapackage, jsontableschema-sql +levenshtein-search==1.4.2 # via dedupe linear-tsv==1.0.0 # via tabulator lxml==3.7.2 # via tabulator mako==1.0.6 # via alembic markupsafe==0.23 # via mako normality==0.3.9 # via dataset +numpy==1.12.0 # via canonicalize, categorical-distance, dedupe-hcluster, highered, pyhacrf-datamade, pylbfgs, rlr, simplecosine openpyxl==2.4.2 # via tabulator packaging==16.8 # via setuptools +persistent==4.2.2 # via btrees, zope.index psycopg2==2.6.2 pybossa-client==1.1.1 +pyhacrf-datamade==0.2.0 # via highered +pylbfgs==0.2.0.3 # via pyhacrf-datamade, rlr pyparsing==2.1.10 # via packaging pypdf2==1.26.0 python-dateutil==2.6.0 # via botocore, jsontableschema, python-documentcloud python-documentcloud==1.0.4 -python-dotenv==0.6.2 +python-dotenv==0.6.3 python-editor==1.0.3 # via alembic python-levenshtein==0.12.0 pyyaml==3.12 # via dataset @@ -50,12 +65,18 @@ raven==5.32.0 requests==2.13.0 rfc3986==0.4.1 # via jsontableschema rfc3987==1.3.7 # via python-documentcloud +rlr==2.4 # via dedupe s3transfer==0.1.10 # via boto3 -six==1.10.0 # via datapackage, dataset, jsonlines, jsontableschema-sql, linear-tsv, normality, packaging, python-dateutil, python-documentcloud, setuptools, tabulator +simplecosine==1.1 # via dedupe +simplejson==3.10.0 # via dedupe +six==1.10.0 # via datapackage, dataset, jsonlines, jsontableschema-sql, linear-tsv, normality, packaging, python-dateutil, python-documentcloud, setuptools, tabulator, zope.index sqlalchemy==1.1.5 tabulator==0.14.0 # via datapackage, jsontableschema, jsontableschema-sql unicodecsv==0.14.1 # via datapackage, jsontableschema, tabulator +unidecode==0.4.20 xlrd==1.0.0 # via tabulator +zope.index==4.2.0 # via dedupe +zope.interface==4.3.3 # via btrees, persistent, zope.index # The following packages are considered to be unsafe in a requirements file: -# setuptools # via python-levenshtein +# setuptools # via python-levenshtein, zope.index, zope.interface diff --git a/tests/conftest.py b/tests/conftest.py index 9682321..6ebb508 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -45,6 +45,7 @@ def conn(request): api_session = APISession() WarehouseSession = sessionmaker(bind=conn['warehouse'].engine) warehouse_session = WarehouseSession() + def teardown(): truncate_database(conn['database'].engine) truncate_database(conn['warehouse'].engine) diff --git a/tests/dbs/opentrials_api_schema_dump.sql b/tests/dbs/opentrials_api_schema_dump.sql index 5a1353d..050464f 100644 --- a/tests/dbs/opentrials_api_schema_dump.sql +++ b/tests/dbs/opentrials_api_schema_dump.sql @@ -159,7 +159,7 @@ SET default_tablespace = ''; SET default_with_oids = false; -- --- Name: conditions; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: conditions; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE conditions ( @@ -175,7 +175,7 @@ CREATE TABLE conditions ( -- --- Name: document_categories; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: document_categories; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE document_categories ( @@ -186,7 +186,7 @@ CREATE TABLE document_categories ( -- --- Name: documents; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: documents; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE documents ( @@ -202,7 +202,7 @@ CREATE TABLE documents ( -- --- Name: fda_applications; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: fda_applications; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE fda_applications ( @@ -216,7 +216,7 @@ CREATE TABLE fda_applications ( -- --- Name: fda_approvals; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: fda_approvals; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE fda_approvals ( @@ -232,7 +232,7 @@ CREATE TABLE fda_approvals ( -- --- Name: files; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: files; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE files ( @@ -245,7 +245,7 @@ CREATE TABLE files ( -- --- Name: interventions; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: interventions; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE interventions ( @@ -265,7 +265,7 @@ CREATE TABLE interventions ( -- --- Name: knex_migrations; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: knex_migrations; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE knex_migrations ( @@ -296,7 +296,7 @@ ALTER SEQUENCE knex_migrations_id_seq OWNED BY knex_migrations.id; -- --- Name: knex_migrations_lock; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: knex_migrations_lock; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE knex_migrations_lock ( @@ -305,7 +305,7 @@ CREATE TABLE knex_migrations_lock ( -- --- Name: locations; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: locations; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE locations ( @@ -321,7 +321,7 @@ CREATE TABLE locations ( -- --- Name: organisations; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: organisations; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE organisations ( @@ -335,7 +335,7 @@ CREATE TABLE organisations ( -- --- Name: persons; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: persons; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE persons ( @@ -349,7 +349,7 @@ CREATE TABLE persons ( -- --- Name: publications; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: publications; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE publications ( @@ -368,7 +368,7 @@ CREATE TABLE publications ( -- --- Name: records; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: records; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE records ( @@ -406,7 +406,7 @@ CREATE TABLE records ( ); -- --- Name: risk_of_bias_criterias; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: risk_of_bias_criterias; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE risk_of_bias_criterias ( @@ -418,7 +418,7 @@ CREATE TABLE risk_of_bias_criterias ( -- --- Name: risk_of_biases; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: risk_of_biases; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE risk_of_biases ( @@ -433,7 +433,7 @@ CREATE TABLE risk_of_biases ( -- --- Name: risk_of_biases_risk_of_bias_criterias; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: risk_of_biases_risk_of_bias_criterias; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE risk_of_biases_risk_of_bias_criterias ( @@ -445,7 +445,7 @@ CREATE TABLE risk_of_biases_risk_of_bias_criterias ( -- --- Name: sources; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: sources; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE sources ( @@ -460,7 +460,7 @@ CREATE TABLE sources ( ); -- --- Name: trials; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: trials; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE trials ( @@ -495,7 +495,7 @@ CREATE TABLE trials ( -- --- Name: trials_conditions; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: trials_conditions; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE trials_conditions ( @@ -505,7 +505,7 @@ CREATE TABLE trials_conditions ( -- --- Name: trials_documents; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: trials_documents; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE trials_documents ( @@ -515,7 +515,7 @@ CREATE TABLE trials_documents ( -- --- Name: trials_interventions; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: trials_interventions; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE trials_interventions ( @@ -525,7 +525,7 @@ CREATE TABLE trials_interventions ( -- --- Name: trials_locations; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: trials_locations; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE trials_locations ( @@ -537,7 +537,7 @@ CREATE TABLE trials_locations ( -- --- Name: trials_organisations; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: trials_organisations; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE trials_organisations ( @@ -549,7 +549,7 @@ CREATE TABLE trials_organisations ( -- --- Name: trials_persons; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: trials_persons; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE trials_persons ( @@ -561,7 +561,7 @@ CREATE TABLE trials_persons ( -- --- Name: trials_publications; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: trials_publications; Type: TABLE; Schema: public; Owner: -; Tablespace: -- CREATE TABLE trials_publications ( @@ -578,7 +578,7 @@ ALTER TABLE ONLY knex_migrations ALTER COLUMN id SET DEFAULT nextval('knex_migra -- --- Name: document_categories_name_group_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: document_categories_name_group_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY document_categories @@ -586,7 +586,7 @@ ALTER TABLE ONLY document_categories -- --- Name: document_categories_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: document_categories_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY document_categories @@ -594,7 +594,7 @@ ALTER TABLE ONLY document_categories -- --- Name: documents_fda_approval_id_file_id_name_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: documents_fda_approval_id_file_id_name_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY documents @@ -602,7 +602,7 @@ ALTER TABLE ONLY documents -- --- Name: documents_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: documents_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY documents @@ -610,7 +610,7 @@ ALTER TABLE ONLY documents -- --- Name: fda_applications_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: fda_applications_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY fda_applications @@ -618,7 +618,7 @@ ALTER TABLE ONLY fda_applications -- --- Name: fda_approvals_fda_application_id_supplement_number_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: fda_approvals_fda_application_id_supplement_number_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY fda_approvals @@ -626,7 +626,7 @@ ALTER TABLE ONLY fda_approvals -- --- Name: fda_approvals_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: fda_approvals_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY fda_approvals @@ -634,7 +634,7 @@ ALTER TABLE ONLY fda_approvals -- --- Name: files_documentcloud_id_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: files_documentcloud_id_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY files @@ -642,7 +642,7 @@ ALTER TABLE ONLY files -- --- Name: files_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: files_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY files @@ -650,7 +650,7 @@ ALTER TABLE ONLY files -- --- Name: files_sha1_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: files_sha1_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY files @@ -658,7 +658,7 @@ ALTER TABLE ONLY files -- --- Name: files_source_url_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: files_source_url_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY files @@ -666,7 +666,7 @@ ALTER TABLE ONLY files -- --- Name: interventions_name_type_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: interventions_name_type_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY interventions @@ -674,7 +674,7 @@ ALTER TABLE ONLY interventions -- --- Name: interventions_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: interventions_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY interventions @@ -682,7 +682,7 @@ ALTER TABLE ONLY interventions -- --- Name: interventions_slug_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: interventions_slug_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY interventions @@ -690,7 +690,7 @@ ALTER TABLE ONLY interventions -- --- Name: knex_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: knex_migrations_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY knex_migrations @@ -698,7 +698,7 @@ ALTER TABLE ONLY knex_migrations -- --- Name: locations_name_type_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: locations_name_type_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY locations @@ -706,7 +706,7 @@ ALTER TABLE ONLY locations -- --- Name: locations_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: locations_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY locations @@ -714,7 +714,7 @@ ALTER TABLE ONLY locations -- --- Name: locations_slug_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: locations_slug_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY locations @@ -722,7 +722,7 @@ ALTER TABLE ONLY locations -- --- Name: organisations_name_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: organisations_name_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY organisations @@ -730,7 +730,7 @@ ALTER TABLE ONLY organisations -- --- Name: organisations_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: organisations_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY organisations @@ -738,7 +738,7 @@ ALTER TABLE ONLY organisations -- --- Name: organisations_slug_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: organisations_slug_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY organisations @@ -746,7 +746,7 @@ ALTER TABLE ONLY organisations -- --- Name: persons_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: persons_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY persons @@ -754,7 +754,7 @@ ALTER TABLE ONLY persons -- --- Name: persons_slug_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: persons_slug_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY persons @@ -762,7 +762,7 @@ ALTER TABLE ONLY persons -- --- Name: problems_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: problems_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY conditions @@ -770,7 +770,7 @@ ALTER TABLE ONLY conditions -- --- Name: problems_slug_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: problems_slug_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY conditions @@ -778,7 +778,7 @@ ALTER TABLE ONLY conditions -- --- Name: publications_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: publications_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY publications @@ -786,7 +786,7 @@ ALTER TABLE ONLY publications -- --- Name: publications_slug_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: publications_slug_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY publications @@ -794,7 +794,7 @@ ALTER TABLE ONLY publications -- --- Name: records_source_url_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: records_source_url_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY records @@ -802,7 +802,7 @@ ALTER TABLE ONLY records -- --- Name: risk_of_bias_criterias_name_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: risk_of_bias_criterias_name_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY risk_of_bias_criterias @@ -810,7 +810,7 @@ ALTER TABLE ONLY risk_of_bias_criterias -- --- Name: risk_of_bias_criterias_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: risk_of_bias_criterias_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY risk_of_bias_criterias @@ -818,7 +818,7 @@ ALTER TABLE ONLY risk_of_bias_criterias -- --- Name: risk_of_biases_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: risk_of_biases_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY risk_of_biases @@ -826,7 +826,7 @@ ALTER TABLE ONLY risk_of_biases -- --- Name: risk_of_biases_risk_of_bias_criterias_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: risk_of_biases_risk_of_bias_criterias_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY risk_of_biases_risk_of_bias_criterias @@ -834,7 +834,7 @@ ALTER TABLE ONLY risk_of_biases_risk_of_bias_criterias -- --- Name: risk_of_biases_study_id_source_url_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: risk_of_biases_study_id_source_url_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY risk_of_biases @@ -842,7 +842,7 @@ ALTER TABLE ONLY risk_of_biases -- --- Name: sources_name_type_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: sources_name_type_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY sources @@ -850,7 +850,7 @@ ALTER TABLE ONLY sources -- --- Name: sources_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: sources_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY sources @@ -858,7 +858,7 @@ ALTER TABLE ONLY sources -- --- Name: trialrecords_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: trialrecords_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY records @@ -866,7 +866,7 @@ ALTER TABLE ONLY records -- --- Name: trials_documents_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: trials_documents_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY trials_documents @@ -874,7 +874,7 @@ ALTER TABLE ONLY trials_documents -- --- Name: trials_interventions_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: trials_interventions_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY trials_interventions @@ -882,7 +882,7 @@ ALTER TABLE ONLY trials_interventions -- --- Name: trials_locations_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: trials_locations_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY trials_locations @@ -890,7 +890,7 @@ ALTER TABLE ONLY trials_locations -- --- Name: trials_organisations_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: trials_organisations_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY trials_organisations @@ -898,7 +898,7 @@ ALTER TABLE ONLY trials_organisations -- --- Name: trials_persons_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: trials_persons_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY trials_persons @@ -906,7 +906,7 @@ ALTER TABLE ONLY trials_persons -- --- Name: trials_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: trials_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY trials @@ -914,7 +914,7 @@ ALTER TABLE ONLY trials -- --- Name: trials_problems_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: trials_problems_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY trials_conditions @@ -922,7 +922,7 @@ ALTER TABLE ONLY trials_conditions -- --- Name: trials_publications_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: trials_publications_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: -- ALTER TABLE ONLY trials_publications @@ -930,35 +930,35 @@ ALTER TABLE ONLY trials_publications -- --- Name: fda_approvals_type_index; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: fda_approvals_type_index; Type: INDEX; Schema: public; Owner: -; Tablespace: -- CREATE INDEX fda_approvals_type_index ON fda_approvals USING btree (type); -- --- Name: interventions_fda_application_number_index; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: interventions_fda_application_number_index; Type: INDEX; Schema: public; Owner: -; Tablespace: -- CREATE INDEX interventions_fda_application_number_index ON interventions USING btree (fda_application_id); -- --- Name: records_identifiers_index; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: records_identifiers_index; Type: INDEX; Schema: public; Owner: -; Tablespace: -- CREATE INDEX records_identifiers_index ON records USING gin (identifiers); -- --- Name: trialrecords_trial_id_index; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: trialrecords_trial_id_index; Type: INDEX; Schema: public; Owner: -; Tablespace: -- CREATE INDEX trialrecords_trial_id_index ON records USING btree (trial_id); -- --- Name: trials_documents_document_id_index; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: trials_documents_document_id_index; Type: INDEX; Schema: public; Owner: -; Tablespace: -- CREATE INDEX trials_documents_document_id_index ON trials_documents USING btree (document_id); diff --git a/tests/dbs/warehouse_schema_dump.sql b/tests/dbs/warehouse_schema_dump.sql index fd4ecdc..6c57565 100644 --- a/tests/dbs/warehouse_schema_dump.sql +++ b/tests/dbs/warehouse_schema_dump.sql @@ -2,15 +2,22 @@ -- PostgreSQL database dump -- +-- Dumped from database version 9.5.5 +-- Dumped by pg_dump version 9.5.5 + SET statement_timeout = 0; SET lock_timeout = 0; SET client_encoding = 'UTF8'; SET standard_conforming_strings = on; SET check_function_bodies = false; SET client_min_messages = warning; +SET row_security = off; SET search_path = public, pg_catalog; +DROP INDEX IF EXISTS public.ix_takeda_f580ee1a7559876f; +DROP INDEX IF EXISTS public.ix_pubmed_ec44fdfeccd66661; +DROP INDEX IF EXISTS public.ix_pfizer_9502f58393a4fc6d; DROP INDEX IF EXISTS public.ix_nct_9502f58393a4fc6d; DROP INDEX IF EXISTS public.ix_jprn_4e2a0ce8f4508948; DROP INDEX IF EXISTS public.ix_isrctn_5630355bbd51c4b7; @@ -50,6 +57,7 @@ ALTER TABLE IF EXISTS ONLY public.actrn DROP CONSTRAINT IF EXISTS actrn_meta_id_ DROP TABLE IF EXISTS public.takeda; DROP TABLE IF EXISTS public.pubmed; DROP TABLE IF EXISTS public.pfizer; +DROP TABLE IF EXISTS public.organisation_clusters; DROP TABLE IF EXISTS public.nct; DROP TABLE IF EXISTS public.jprn; DROP TABLE IF EXISTS public.isrctn; @@ -101,7 +109,7 @@ SET default_tablespace = ''; SET default_with_oids = false; -- --- Name: actrn; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: actrn; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE actrn ( @@ -176,7 +184,7 @@ CREATE TABLE actrn ( -- --- Name: alembic_version; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: alembic_version; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE alembic_version ( @@ -185,7 +193,7 @@ CREATE TABLE alembic_version ( -- --- Name: cochrane_reviews; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: cochrane_reviews; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE cochrane_reviews ( @@ -204,7 +212,7 @@ CREATE TABLE cochrane_reviews ( -- --- Name: euctr; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: euctr; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE euctr ( @@ -335,7 +343,7 @@ CREATE TABLE euctr ( -- --- Name: fda_dap; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: fda_dap; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE fda_dap ( @@ -357,7 +365,7 @@ CREATE TABLE fda_dap ( -- --- Name: fdadl; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: fdadl; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE fdadl ( @@ -375,7 +383,7 @@ CREATE TABLE fdadl ( -- --- Name: gsk; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: gsk; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE gsk ( @@ -471,7 +479,7 @@ CREATE TABLE gsk ( -- --- Name: hra; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: hra; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE hra ( @@ -524,7 +532,7 @@ CREATE TABLE hra ( -- --- Name: icdcm; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: icdcm; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE icdcm ( @@ -541,7 +549,7 @@ CREATE TABLE icdcm ( -- --- Name: icdpcs; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: icdpcs; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE icdpcs ( @@ -559,7 +567,7 @@ CREATE TABLE icdpcs ( -- --- Name: ictrp; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: ictrp; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE ictrp ( @@ -595,7 +603,7 @@ CREATE TABLE ictrp ( -- --- Name: isrctn; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: isrctn; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE isrctn ( @@ -661,7 +669,7 @@ CREATE TABLE isrctn ( -- --- Name: jprn; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: jprn; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE jprn ( @@ -756,7 +764,7 @@ CREATE TABLE jprn ( -- --- Name: nct; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: nct; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE nct ( @@ -823,13 +831,22 @@ CREATE TABLE nct ( clinical_results jsonb, nct_aliases text[], removed_countries text[], - biospec_desrc text, - results_exemption_date date + biospec_desrc text ); -- --- Name: pfizer; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: organisation_clusters; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE organisation_clusters ( + canonical text, + variations text[] +); + + +-- +-- Name: pfizer; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE pfizer ( @@ -852,7 +869,7 @@ CREATE TABLE pfizer ( -- --- Name: pubmed; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: pubmed; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE pubmed ( @@ -885,7 +902,7 @@ CREATE TABLE pubmed ( -- --- Name: takeda; Type: TABLE; Schema: public; Owner: -; Tablespace: +-- Name: takeda; Type: TABLE; Schema: public; Owner: - -- CREATE TABLE takeda ( @@ -924,7 +941,7 @@ CREATE TABLE takeda ( -- --- Name: actrn_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: actrn_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY actrn @@ -932,7 +949,7 @@ ALTER TABLE ONLY actrn -- --- Name: actrn_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: actrn_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY actrn @@ -940,7 +957,7 @@ ALTER TABLE ONLY actrn -- --- Name: cochrane_reviews_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: cochrane_reviews_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY cochrane_reviews @@ -948,7 +965,7 @@ ALTER TABLE ONLY cochrane_reviews -- --- Name: euctr_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: euctr_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY euctr @@ -956,7 +973,7 @@ ALTER TABLE ONLY euctr -- --- Name: euctr_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: euctr_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY euctr @@ -964,7 +981,7 @@ ALTER TABLE ONLY euctr -- --- Name: fda_dap_id_key; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: fda_dap_id_key; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY fda_dap @@ -972,7 +989,7 @@ ALTER TABLE ONLY fda_dap -- --- Name: fda_dap_meta_id_key; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: fda_dap_meta_id_key; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY fda_dap @@ -980,7 +997,7 @@ ALTER TABLE ONLY fda_dap -- --- Name: fda_meta_id_key; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: fda_meta_id_key; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY fdadl @@ -988,7 +1005,7 @@ ALTER TABLE ONLY fdadl -- --- Name: fda_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: fda_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY fdadl @@ -996,7 +1013,7 @@ ALTER TABLE ONLY fdadl -- --- Name: gsk_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: gsk_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY gsk @@ -1004,7 +1021,7 @@ ALTER TABLE ONLY gsk -- --- Name: gsk_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: gsk_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY gsk @@ -1012,7 +1029,7 @@ ALTER TABLE ONLY gsk -- --- Name: hra_meta_id_key; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: hra_meta_id_key; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY hra @@ -1020,7 +1037,7 @@ ALTER TABLE ONLY hra -- --- Name: icdcm_meta_id_key; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: icdcm_meta_id_key; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY icdcm @@ -1028,7 +1045,7 @@ ALTER TABLE ONLY icdcm -- --- Name: icdcm_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: icdcm_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY icdcm @@ -1036,7 +1053,7 @@ ALTER TABLE ONLY icdcm -- --- Name: icdpcs_meta_id_key; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: icdpcs_meta_id_key; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY icdpcs @@ -1044,7 +1061,7 @@ ALTER TABLE ONLY icdpcs -- --- Name: icdpcs_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: icdpcs_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY icdpcs @@ -1052,7 +1069,7 @@ ALTER TABLE ONLY icdpcs -- --- Name: ictrp_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: ictrp_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY ictrp @@ -1060,7 +1077,7 @@ ALTER TABLE ONLY ictrp -- --- Name: ictrp_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: ictrp_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY ictrp @@ -1068,7 +1085,7 @@ ALTER TABLE ONLY ictrp -- --- Name: isrctn_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: isrctn_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY isrctn @@ -1076,7 +1093,7 @@ ALTER TABLE ONLY isrctn -- --- Name: isrctn_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: isrctn_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY isrctn @@ -1084,7 +1101,7 @@ ALTER TABLE ONLY isrctn -- --- Name: jprn_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: jprn_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY jprn @@ -1092,7 +1109,7 @@ ALTER TABLE ONLY jprn -- --- Name: jprn_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: jprn_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY jprn @@ -1100,7 +1117,7 @@ ALTER TABLE ONLY jprn -- --- Name: nct_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: nct_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY nct @@ -1108,7 +1125,7 @@ ALTER TABLE ONLY nct -- --- Name: nct_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: nct_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY nct @@ -1116,7 +1133,7 @@ ALTER TABLE ONLY nct -- --- Name: pfizer_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: pfizer_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY pfizer @@ -1124,7 +1141,7 @@ ALTER TABLE ONLY pfizer -- --- Name: pfizer_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: pfizer_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY pfizer @@ -1132,7 +1149,7 @@ ALTER TABLE ONLY pfizer -- --- Name: pubmed_meta_id_key; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: pubmed_meta_id_key; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY pubmed @@ -1140,7 +1157,7 @@ ALTER TABLE ONLY pubmed -- --- Name: pubmed_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: pubmed_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY pubmed @@ -1148,7 +1165,7 @@ ALTER TABLE ONLY pubmed -- --- Name: takeda_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: takeda_meta_id_unique; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY takeda @@ -1156,7 +1173,7 @@ ALTER TABLE ONLY takeda -- --- Name: takeda_pkey; Type: CONSTRAINT; Schema: public; Owner: -; Tablespace: +-- Name: takeda_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- ALTER TABLE ONLY takeda @@ -1164,47 +1181,68 @@ ALTER TABLE ONLY takeda -- --- Name: ix_actrn_9b88700fa823eea2; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: ix_actrn_9b88700fa823eea2; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX ix_actrn_9b88700fa823eea2 ON actrn USING btree (trial_id); -- --- Name: ix_euctr_86a28cd2542cd0c4; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: ix_euctr_86a28cd2542cd0c4; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX ix_euctr_86a28cd2542cd0c4 ON euctr USING btree (eudract_number_with_country); -- --- Name: ix_gsk_ae14dbea0172b852; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: ix_gsk_ae14dbea0172b852; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX ix_gsk_ae14dbea0172b852 ON gsk USING btree (study_id); -- --- Name: ix_isrctn_5630355bbd51c4b7; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: ix_isrctn_5630355bbd51c4b7; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX ix_isrctn_5630355bbd51c4b7 ON isrctn USING btree (isrctn_id); -- --- Name: ix_jprn_4e2a0ce8f4508948; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: ix_jprn_4e2a0ce8f4508948; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX ix_jprn_4e2a0ce8f4508948 ON jprn USING btree (unique_trial_number); -- --- Name: ix_nct_9502f58393a4fc6d; Type: INDEX; Schema: public; Owner: -; Tablespace: +-- Name: ix_nct_9502f58393a4fc6d; Type: INDEX; Schema: public; Owner: - -- CREATE INDEX ix_nct_9502f58393a4fc6d ON nct USING btree (nct_id); +-- +-- Name: ix_pfizer_9502f58393a4fc6d; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_pfizer_9502f58393a4fc6d ON pfizer USING btree (nct_id); + + +-- +-- Name: ix_pubmed_ec44fdfeccd66661; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_pubmed_ec44fdfeccd66661 ON pubmed USING btree (pmid); + + +-- +-- Name: ix_takeda_f580ee1a7559876f; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX ix_takeda_f580ee1a7559876f ON takeda USING btree (takeda_trial_id); + + -- -- PostgreSQL database dump complete -- diff --git a/tests/processors/base/helpers/test_helpers.py b/tests/processors/base/helpers/test_helpers.py index 3754e85..8f554d1 100644 --- a/tests/processors/base/helpers/test_helpers.py +++ b/tests/processors/base/helpers/test_helpers.py @@ -7,7 +7,6 @@ import pytest import processors.base.helpers as helpers - class TestValidateIdentifier(object): @pytest.mark.parametrize('identifier', [ 'NCT00020500', @@ -130,22 +129,48 @@ class TestLocationNormalizer(object): @pytest.mark.parametrize("test_input,expected", [ # Locations normalized by ISO-3166 name standards ("Brazil", "Brazil"), - ("United States of America","United States"), + ("United States of America", "United States"), # Locations normalized by ISO-3166 acronym standards - ("US","United States"), - ("UK","United Kingdom"), + ("US", "United States"), + ("UK", "United Kingdom of Great Britain and Northern Ireland"), # Locations normalized by Levenshtein distance (name) - ("Chnia","China"), - ("The Netherlands","Netherlands"), - ("thauland","Thailand"), + ("Chnia", "China"), + ("The Netherlands", "Netherlands"), + ("thauland", "Thailand"), # Locations not normalized (corner cases) - ("Asia(except Japan)","Asia(except Japan)"), - ("Outside","Outside"), + ("Asia(except Japan)", "Asia(except Japan)"), + ("Outside", "Outside"), (None, None)]) def test_location_normalizer(self, test_input, expected): assert helpers.get_canonical_location_name(test_input) == expected + +class TestOrganisationNormalizer(object): + def _create_organisation_cluster(self, conn, canonical_name, variations): + cluster = { + 'canonical': canonical_name, + 'variations': [canonical_name] + [variations] + } + conn['warehouse']['organisation_clusters'].insert(cluster) + + @pytest.mark.parametrize("test_input,expected", [ + # Organisation normalized with a longer equivalent + ("VERTEX PHARMS INC", "VERTEX PHARMS"), + ("Alliance Pharmaceuticals", "Alliance Pharmaceuticals Ltd"), + + # Organisation normalized as it is (the longest available form) + ("Justus Liebig University of Giessen", + "Justus Liebig University of Giessen"), + + # New organisation non-normalized + ("Federal University of Campina Grande", + "Federal University of Campina Grande"), + ]) + + def test_organisation_normalizer(self, conn, test_input, expected): + self._create_organisation_cluster(conn, expected, test_input) + assert helpers.get_canonical_organisation_name(conn, test_input) == expected \ No newline at end of file diff --git a/tox.ini b/tox.ini index e362287..c856cea 100644 --- a/tox.ini +++ b/tox.ini @@ -6,7 +6,7 @@ skipsdist = True [testenv] deps = - -r{toxinidir}/requirements.txt + numpy pytest mock setenv = @@ -14,7 +14,10 @@ setenv = passenv = TEST_WAREHOUSE_URL TEST_DATABASE_URL +# Installing requirements after tox deps for depedency reasons +# See: https://github.com/opentrials/processors/pull/116 commands = + pip install -r {toxinidir}/requirements.txt py.test {posargs} [testenv:lint]