From 47470c5f251a62b55c6d7091f832b555966d1cd5 Mon Sep 17 00:00:00 2001 From: yuliujpl <15621084+yuliujpl@users.noreply.github.com> Date: Sun, 12 Apr 2026 21:36:39 -0700 Subject: [PATCH 1/2] Add Docker deployment for KSDB --- .dockerignore | 16 ++ Dockerfile | 23 +++ README.md | 61 +++--- deploy.sh | 60 ++++++ django_datatables_view/__init__.py | 1 + django_datatables_view/base_datatable_view.py | 81 ++++++++ docker-compose.yml | 49 +++++ docker/entrypoint.sh | 76 +++++++ docker/postgres-init/10-load-schema.sh | 22 +++ docker/postgres-init/20-schema-fixes.sql | 30 +++ ksdb/forms.py | 7 +- ksdb/ksdbviews.py | 25 ++- ksdb/labcas_assaytype.py | 103 ++++++++++ ksdb/management/commands/ingestorgans.py | 73 +++---- ksdb/management/commands/ingestpersons.py | 186 ++++++++---------- ksdb/management/commands/ingestprotocols.py | 134 +++++++++---- .../management/commands/ingestpublications.py | 151 +++++++------- ksdb/management/commands/preloadksdbdata.py | 45 +++++ ksdb/models.py | 61 +++++- ksdb/publishRDF.py | 9 +- ksdb/rdf_ingest_utils.py | 68 +++++++ ksdb/urls.py | 5 +- ksdb/view.py | 9 +- requirements-docker.txt | 8 + sitemain/settings_docker.py | 128 ++++++++++++ sitemain/wsgi_docker.py | 8 + templates/assaytypeinput.html | 89 +++++++++ templates/ksdb.html | 5 + tinymce/__init__.py | 1 + tinymce/models.py | 6 + 30 files changed, 1215 insertions(+), 325 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100755 deploy.sh create mode 100644 django_datatables_view/__init__.py create mode 100644 django_datatables_view/base_datatable_view.py create mode 100644 docker-compose.yml create mode 100755 docker/entrypoint.sh create mode 100755 docker/postgres-init/10-load-schema.sh create mode 100644 docker/postgres-init/20-schema-fixes.sql create mode 100755 ksdb/labcas_assaytype.py create mode 100644 ksdb/management/commands/preloadksdbdata.py create mode 100644 ksdb/rdf_ingest_utils.py create mode 100644 requirements-docker.txt create mode 100644 sitemain/settings_docker.py create mode 100644 sitemain/wsgi_docker.py create mode 100755 templates/assaytypeinput.html create mode 100644 tinymce/__init__.py create mode 100644 tinymce/models.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..554a40a --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.git +.gitignore +__pycache__/ +*.pyc +*.pyo +*.pyd +.Python +.pytest_cache/ +.mypy_cache/ +htmlcov/ +dist/ +build/ +staticfiles/ +ksdb/static/tmp/ +test + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3ef435b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.7-slim-bullseye + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +RUN apt-get update \ + && apt-get install -y --no-install-recommends build-essential ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements-docker.txt /app/requirements-docker.txt +RUN pip install --no-cache-dir --upgrade pip setuptools wheel \ + && pip install --no-cache-dir -r /app/requirements-docker.txt + +COPY . /app +COPY docker/entrypoint.sh /usr/local/bin/ksdb-entrypoint +RUN chmod +x /usr/local/bin/ksdb-entrypoint + +EXPOSE 8000 + +ENTRYPOINT ["/usr/local/bin/ksdb-entrypoint"] +CMD ["gunicorn", "sitemain.wsgi_docker:application", "--bind", "0.0.0.0:8000", "--workers", "2"] diff --git a/README.md b/README.md index 6f5c447..8839812 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,8 @@ Knowledge System DB: The goal of this system is to maintain generalized project information to be used for any bioinformatics knowledge system that the JPL Informatics Center supports. For example, EDRN or MCL. -# DEPENDENCIES +#DEPENDENCIES -``` sudo yum install postgresql-server postgresql-devel gcc openldap-devel openssl openssl-devel python3: curl -O https://www.python.org/ftp/python/3.5.0/Python-3.5.0.tgz @@ -16,13 +15,10 @@ python3: pip3: curl -O https://bootstrap.pypa.io/get-pip.py sudo python3 get-pip.py -``` -# SETUP POSTGRES +#SETUP POSTGRES -## setup postgres - -``` +#setup postgres sudo service postgresql initdb sudo chkconfig postgresql on sudo service postgresql start @@ -32,53 +28,38 @@ pip3: create database ksdb; alter user edrn password 'edrn'; GRANT ALL ON database ksdb TO edrn; -``` - -## insert tables -``` +#insert tables ctrl^D once to exit out of database sudo service postgresql restart export PGPASSWORD=edrn psql -U edrn -h 127.0.0.1 -d ksdb -a -f conf/createtables.sql -``` # SETUP KSDB -## Make sure to create a virtual environment - -``` +# Make sure to create a virtual environment sudo pip install setuptools --upgrade sudo pip install -r conf/dependencies.cfg -``` - -## Configure KSDB -``` +# Configure KSDB modify sitemain/settings.py to update DATABASES parameter. Make sure you updated 'name' to ksdb, 'user' to edrn, 'password' to edrn, and host to localhost. *You can use support/set_settings.py to automatically apply a local settings file to the template settings file. The script will write a settings.py in the same directory as the template settings file: **Example: python support/set_settings.py ~/KSDB/settings.tumor.py sitemain/settings.py.in python manage.py migrate auth python manage.py migrate python manage.py createsuperuser (create the user you will be using to login) -``` - -## Running the project - -``` + +# Running the project To run this project: #ingest knowledge objects from CancerDataExpo into database python manage.py ingestorgans python manage.py ingestpublications python manage.py ingestpersons -``` -## Run the server -``` +# Run the server python manage.py runserver 0.0.0.0:8000 -``` You can now visit the following URLS: @@ -89,6 +70,28 @@ You can now visit the following URLS: Run the test suite: -``` python manage.py test ksdb + + +# Docker deployment + +For the containerized setup, use the single wrapper script from the project root: + +```bash +./deploy.sh ``` + +Useful variants: + +```bash +./deploy.sh fresh +./deploy.sh status +./deploy.sh logs +./deploy.sh down +``` + +Notes: + +- `./deploy.sh` builds the image, starts Postgres and Django, loads the schema, preloads supported KSDB data, and exposes the app on `http://localhost:8000/` +- `./deploy.sh fresh` also deletes the Docker volume and rebuilds the database from scratch +- The default login is `admin` / `admin` unless you override the compose environment variables diff --git a/deploy.sh b/deploy.sh new file mode 100755 index 0000000..30c81b0 --- /dev/null +++ b/deploy.sh @@ -0,0 +1,60 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +cd "${SCRIPT_DIR}" + +ACTION="${1:-up}" + +print_summary() { + port="${KSDB_PORT:-8000}" + user="${DJANGO_SUPERUSER_USERNAME:-admin}" + password="${DJANGO_SUPERUSER_PASSWORD:-admin}" + + cat <&2 + echo "Run ./deploy.sh --help for usage." >&2 + exit 1 + ;; +esac diff --git a/django_datatables_view/__init__.py b/django_datatables_view/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/django_datatables_view/__init__.py @@ -0,0 +1 @@ + diff --git a/django_datatables_view/base_datatable_view.py b/django_datatables_view/base_datatable_view.py new file mode 100644 index 0000000..4e44082 --- /dev/null +++ b/django_datatables_view/base_datatable_view.py @@ -0,0 +1,81 @@ +from django.core.exceptions import FieldError +from django.http import JsonResponse +from django.views.generic.base import View + + +class BaseDatatableView(View): + model = None + columns = [] + order_columns = [] + max_display_length = 100 + + def get_initial_queryset(self): + return self.model.objects.all() + + def filter_queryset(self, queryset): + return queryset + + def render_column(self, row, column): + return getattr(row, column) + + def ordering(self, queryset): + column_index = self.request.GET.get("order[0][column]") + direction = self.request.GET.get("order[0][dir]", "asc") + + if column_index is None: + return queryset + + try: + order_column = self.order_columns[int(column_index)] + except (IndexError, TypeError, ValueError): + return queryset + + if not order_column or order_column == "Select": + return queryset + + prefix = "-" if direction == "desc" else "" + try: + return queryset.order_by(prefix + order_column) + except FieldError: + return queryset + + def paging(self, queryset): + try: + start = int(self.request.GET.get("start", 0)) + except (TypeError, ValueError): + start = 0 + + try: + length = int(self.request.GET.get("length", 10)) + except (TypeError, ValueError): + length = 10 + + if length < 0: + length = self.max_display_length + length = min(length, self.max_display_length) + + return queryset[start:start + length] + + def prepare_results(self, queryset): + data = [] + for row in queryset: + data.append([self.render_column(row, column) for column in self.columns]) + return data + + def get(self, request, *args, **kwargs): + queryset = self.get_initial_queryset() + records_total = queryset.count() + + filtered_queryset = self.filter_queryset(queryset) + records_filtered = filtered_queryset.count() + ordered_queryset = self.ordering(filtered_queryset) + page_queryset = self.paging(ordered_queryset) + + return JsonResponse( + { + "draw": int(request.GET.get("draw", 1)), + "recordsTotal": records_total, + "recordsFiltered": records_filtered, + "data": self.prepare_results(page_queryset), + } + ) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4be72d2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,49 @@ +services: + db: + image: postgres:14 + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB:-ksdb} + POSTGRES_USER: ${POSTGRES_USER:-ksdb} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-ksdb} + volumes: + - postgres_data:/var/lib/postgresql/data + - ./conf/createtables.sql:/schema/createtables.sql:ro + - ./docker/postgres-init/10-load-schema.sh:/docker-entrypoint-initdb.d/10-load-schema.sh:ro + - ./docker/postgres-init/20-schema-fixes.sql:/docker-entrypoint-initdb.d/20-schema-fixes.sql:ro + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 5s + + web: + build: + context: . + dockerfile: Dockerfile + restart: unless-stopped + depends_on: + db: + condition: service_healthy + environment: + DJANGO_SETTINGS_MODULE: sitemain.settings_docker + DATABASE_NAME: ${POSTGRES_DB:-ksdb} + DATABASE_USER: ${POSTGRES_USER:-ksdb} + DATABASE_PASSWORD: ${POSTGRES_PASSWORD:-ksdb} + DATABASE_HOST: db + DATABASE_PORT: 5432 + DJANGO_DEBUG: ${DJANGO_DEBUG:-1} + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY:-ksdb-local-dev-secret-key} + DJANGO_ALLOWED_HOSTS: ${DJANGO_ALLOWED_HOSTS:-*} + RDF_URL: ${RDF_URL:-http://localhost:8000/} + MCL_URL: ${MCL_URL:-http://localhost:8000/} + DJANGO_SUPERUSER_USERNAME: ${DJANGO_SUPERUSER_USERNAME:-admin} + DJANGO_SUPERUSER_EMAIL: ${DJANGO_SUPERUSER_EMAIL:-admin@example.com} + DJANGO_SUPERUSER_PASSWORD: ${DJANGO_SUPERUSER_PASSWORD:-admin} + KSDB_PRELOAD_DATA: ${KSDB_PRELOAD_DATA:-1} + ports: + - "${KSDB_PORT:-8000}:8000" + +volumes: + postgres_data: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 0000000..4876d9b --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,76 @@ +#!/bin/sh +set -eu + +cp /app/sitemain/settings_docker.py /app/sitemain/settings.py +cp /app/sitemain/wsgi_docker.py /app/sitemain/wsgi.py + +python - <<'PY' +import os +import time + +import psycopg2 + +settings = { + "dbname": os.environ.get("DATABASE_NAME", "ksdb"), + "user": os.environ.get("DATABASE_USER", "ksdb"), + "password": os.environ.get("DATABASE_PASSWORD", "ksdb"), + "host": os.environ.get("DATABASE_HOST", "db"), + "port": os.environ.get("DATABASE_PORT", "5432"), +} + +for attempt in range(60): + try: + connection = psycopg2.connect(**settings) + except Exception as exc: + print("Waiting for database ({}/60): {}".format(attempt + 1, exc)) + time.sleep(2) + else: + connection.close() + print("Database is ready.") + break +else: + raise SystemExit("Database never became ready.") +PY + +python manage.py migrate --noinput --run-syncdb + +case "${KSDB_PRELOAD_DATA:-1}" in + 1|true|TRUE|yes|YES|on|ON) + python manage.py preloadksdbdata + ;; +esac + +python manage.py collectstatic --noinput + +if [ -n "${DJANGO_SUPERUSER_USERNAME:-}" ] && [ -n "${DJANGO_SUPERUSER_PASSWORD:-}" ]; then +python - <<'PY' +import os + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", os.environ.get("DJANGO_SETTINGS_MODULE", "sitemain.settings_docker")) + +import django + +django.setup() + +from django.contrib.auth import get_user_model + +User = get_user_model() +username = os.environ["DJANGO_SUPERUSER_USERNAME"] +email = os.environ.get("DJANGO_SUPERUSER_EMAIL", "") +password = os.environ["DJANGO_SUPERUSER_PASSWORD"] + +user, created = User.objects.get_or_create( + username=username, + defaults={"email": email, "is_staff": True, "is_superuser": True}, +) +user.email = email +user.is_staff = True +user.is_superuser = True +user.set_password(password) +user.save() + +print("{} Django superuser '{}'.".format("Created" if created else "Updated", username)) +PY +fi + +exec "$@" diff --git a/docker/postgres-init/10-load-schema.sh b/docker/postgres-init/10-load-schema.sh new file mode 100755 index 0000000..ddf9a2f --- /dev/null +++ b/docker/postgres-init/10-load-schema.sh @@ -0,0 +1,22 @@ +#!/bin/sh +set -eu + +cleaned_schema="/tmp/ksdb-schema.sql" + +awk ' + /^[[:space:]]*#/ { next } + /CREATE SEQUENCE program_seq START 3;/ { next } + /alter table fundedsite_project_link rename to fundedsite_program_link;/ { next } + /alter table project rename to program;/ { next } + /alter table fundedsite_program_link rename column projectid to programid;/ { next } + /alter table publication alter column projectid type varchar\(2000\)/ { next } + /alter table publication add column title type varchar\(2000\)/ { next } + /alter table publication alter column title type varchar\(2000\)/ { + print "alter table publication alter column title type varchar(2000);" + next + } + { print } +' /schema/createtables.sql > "${cleaned_schema}" + +psql -v ON_ERROR_STOP=0 -U "${POSTGRES_USER}" -d "${POSTGRES_DB}" -f "${cleaned_schema}" + diff --git a/docker/postgres-init/20-schema-fixes.sql b/docker/postgres-init/20-schema-fixes.sql new file mode 100644 index 0000000..d07de44 --- /dev/null +++ b/docker/postgres-init/20-schema-fixes.sql @@ -0,0 +1,30 @@ +ALTER TABLE IF EXISTS protocol + ADD COLUMN IF NOT EXISTS edrn_id text; + +ALTER TABLE IF EXISTS protocol + ADD COLUMN IF NOT EXISTS institutions text; + +ALTER TABLE IF EXISTS knowledge_linkage + ADD COLUMN IF NOT EXISTS id bigserial PRIMARY KEY; + +ALTER TABLE IF EXISTS labcas_user_profiles + ADD COLUMN IF NOT EXISTS id bigserial PRIMARY KEY; + +ALTER TABLE IF EXISTS labcas_user_profiles + ADD COLUMN IF NOT EXISTS search_name text NOT NULL DEFAULT ''; + +ALTER TABLE IF EXISTS labcas_user_profiles + DROP CONSTRAINT IF EXISTS labcas_user_profiles_username_key; + +ALTER TABLE IF EXISTS labcas_user_profiles + ADD CONSTRAINT labcas_user_profiles_username_search_name_key + UNIQUE (username, search_name); + +CREATE TABLE IF NOT EXISTS fundedsite_institution_visible_id ( + id bigserial PRIMARY KEY, + fundedsiteid int, + institutionid int, + fundedsite_institution_visible_id int, + UNIQUE (fundedsiteid, institutionid, fundedsite_institution_visible_id) +); + diff --git a/ksdb/forms.py b/ksdb/forms.py index a09a551..244ce08 100755 --- a/ksdb/forms.py +++ b/ksdb/forms.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from django.contrib.auth.forms import AuthenticationForm from django import forms -from ksdb.models import protocol, person, program, institution, publication, fundedsite, organ, degree, committee, group, disease, species, specimentype, discipline +from ksdb.models import protocol, person, program, institution, publication, fundedsite, organ, degree, committee, group, disease, species, specimentype, discipline, labcas_assaytype class ProtocolForm(forms.ModelForm): @@ -39,6 +39,11 @@ class Meta: model = degree fields = '__all__' +class LabcasAssayTypeForm(forms.ModelForm): + class Meta: + model = labcas_assaytype + fields = '__all__' + class OrganForm(forms.ModelForm): class Meta: model = organ diff --git a/ksdb/ksdbviews.py b/ksdb/ksdbviews.py index 8b22fde..d8c2f43 100644 --- a/ksdb/ksdbviews.py +++ b/ksdb/ksdbviews.py @@ -1,5 +1,5 @@ from django.db.models import Q -from ksdb.models import person, publication, program, institution, fundedsite, protocol, organ, degree, group, disease, committee, discipline, species, specimentype +from ksdb.models import person, publication, program, institution, fundedsite, protocol, organ, labcas_assaytype, degree, group, disease, committee, discipline, species, specimentype from ksdb.ekeutils import _KSDBhref, getPersonNameByID, getProgramTitleByID from django_datatables_view.base_datatable_view import BaseDatatableView @@ -279,6 +279,29 @@ def filter_queryset(self, qs): Q(id__icontains=search)) return qs +class AssayTypeView(BaseDatatableView): + model = labcas_assaytype + objtype = "assaytype" + # define the columns that will be returned + columns = ['Select', 'id', 'name'] + + order_columns = ['id', 'id', 'name'] + max_display_length = 500 + + def render_column(self, row, column): + # We want to render user as a custom column + if column == 'Select': + return ''.format(self.objtype, row.id) + else: + obj = super(AssayTypeView, self).render_column(row, column) + return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + def filter_queryset(self, qs): + search = self.request.GET.get(u'search[value]', None) + if search: + qs = qs.filter(Q(name__icontains=search) | + Q(id__icontains=search)) + return qs + class OrganView(BaseDatatableView): model = organ objtype = "organ" diff --git a/ksdb/labcas_assaytype.py b/ksdb/labcas_assaytype.py new file mode 100755 index 0000000..4c9700c --- /dev/null +++ b/ksdb/labcas_assaytype.py @@ -0,0 +1,103 @@ +# protocols.py +from django.db import connection, transaction +from django.shortcuts import render_to_response +from django.template import RequestContext +import copy, simplejson + +# Create your views here. +from ksdb.models import IdSeq +from ksdb.models import labcas_assaytype + +# Allow external command processing +from django.http import JsonResponse +from ksdb.forms import LabcasAssayTypeForm + +#import settings +import logging +logger = logging.getLogger(__name__) + +def gen_assaytype_data(request): + data = {"action" : "New" , + } + if request.method == 'GET': + assaytypeid = request.GET.get('id') + if assaytypeid: + obj = labcas_assaytype.objects.get(pk=int(assaytypeid)) + data = { "action" : "Edit", + "id" : obj.id, + "name" : obj.name, + "alias" : obj.alias, + } + return data + +def delete_assaytype(request): + message = None + success = False + + if request.method == 'POST': + ids = request.POST.get("id").split(",") + if len(ids) > 0: + for assaytype_id in ids: + #delete assaytype itself + labcas_assaytype.objects.filter(id=assaytype_id).delete() + + message = "Successfully deleted assaytype id(s): "+request.POST.get("id") + success = True + else: + success = False + message = "No assaytypes selected, please select assaytype for deletion." + else: + message = "Not a post method, has to be post in order to delete object." + return JsonResponse({'Success':success, + 'Message':message}) + +def assaytype_input(request): + if request.method == 'POST': + + assaytype_id = None + message = "You have successfully added a assaytype." + success = True + parameters = copy.copy(request.POST) + + if request.POST.get('action') == "edit": + assaytype_id = int(request.POST.get('assaytypeid')) + message = "You have successfull edited assaytype "+str(assaytype_id)+"." + parameters["id"] = assaytype_id + assaytypei = labcas_assaytype.objects.get(id=assaytype_id) + assaytypem = LabcasAssayTypeForm(parameters or None, instance=assaytypei) + else: + if (request.POST.get('duplicate') == 'false'): + try: + labcas_assaytype.objects.get(name=parameters['name']) + return JsonResponse({'Success':False, + 'Message':'{"name":["This assaytype name has already been registered."]}'}) + except labcas_assaytype.DoesNotExist: + pass + except labcas_assaytype.MultipleObjectsReturned: + return JsonResponse({'Success':False, + 'Message':'{"name":["This assaytype name has already been registered."]}'}) + cursor = connection.cursor() + + result = cursor.execute("select nextval('labcas_assaytype_seq') from labcas_assaytype_seq") + assaytype_id = cursor.fetchone()[0] + #assaytype_id = IdSeq.objects.raw("select sequence_name, nextval('assaytype_seq') from assaytype_seq")[0].nextval + parameters["id"] = assaytype_id + assaytypem = LabcasAssayTypeForm(parameters) + + if assaytypem.is_valid(): + assaytypem.save() + else: + message = simplejson.dumps(assaytypem.errors) + success = False + return JsonResponse({'Success':success, + 'Message':message}) + + #generate assaytype data from db + data = gen_assaytype_data(request) + + # Render input page with the documents and the form + return render_to_response( + 'assaytypeinput.html', + data, + context_instance=RequestContext(request) + ) diff --git a/ksdb/management/commands/ingestorgans.py b/ksdb/management/commands/ingestorgans.py index 42e6ff9..0498977 100755 --- a/ksdb/management/commands/ingestorgans.py +++ b/ksdb/management/commands/ingestorgans.py @@ -1,60 +1,37 @@ -# ingestorgans.py -from django.core.management.base import BaseCommand, CommandError -from ksdb.models import organ, IdSeq -from ksdb.forms import OrganForm +from django.core.management.base import BaseCommand +from rdflib.term import URIRef -import rdflib -from rdflib.term import URIRef, Literal +from ksdb.models import organ +from ksdb.rdf_ingest_utils import fetch_rdf_graph, next_sequence_value, parse_statements, text_value -#import settings import logging -import json + + logger = logging.getLogger(__name__) + class Command(BaseCommand): - help = 'Ingest publications' + help = "Ingest organs from the EDRN RDF feed." organurl = "https://edrn.jpl.nasa.gov/cancerdataexpo/rdf-data/body-systems/@@rdf" - _bodySystemTitle = URIRef("http://purl.org/dc/terms/title") + _body_system_title = URIRef("http://purl.org/dc/terms/title") def handle(self, *args, **options): - g = rdflib.Graph() - result = g.parse(self.organurl) + graph = fetch_rdf_graph(self.organurl) + statements = parse_statements(graph) imported_count = 0 - organStatements = self._parseRDF(g) - for body in organStatements: - for title in organStatements[body][self._bodySystemTitle]: - if not self._exists(title): - parameters = {} - org_id = IdSeq.objects.raw("select sequence_name, nextval('organ_seq') from organ_seq")[0].nextval - parameters["id"] = org_id - parameters["name"] = title - parameters["description"] = "" - organm = OrganForm(parameters) - - if organm.is_valid(): - organm.save() - imported_count += 1 - logger.info("Successfully imported {} organs from cancerdataexpo rdf.".format(imported_count)) - def _exists(self, name): - #returns true if no object exist, returns false if one or more object exist - exist = True - try: - organ.objects.get(name=name) - except organ.DoesNotExist: - exist = False - pass - except organ.MultipleObjectsReturned: - pass - return exist - def _parseRDF(self, graph): - statements = {} - for s, p, o in graph: - if s not in statements: - statements[s] = {} - predicates = statements[s] - if p not in predicates: - predicates[p] = [] - predicates[p].append(o) - return statements + for subject in statements: + title = text_value(statements[subject].get(self._body_system_title, [""])[0]) + if not title: + continue + if organ.objects.filter(name=title).exists(): + continue + + organ.objects.create( + id=next_sequence_value("organ_seq"), + name=title, + description="", + ) + imported_count += 1 + logger.info("Successfully imported %s organs from cancerdataexpo rdf.", imported_count) diff --git a/ksdb/management/commands/ingestpersons.py b/ksdb/management/commands/ingestpersons.py index bd4054a..5ccf06f 100755 --- a/ksdb/management/commands/ingestpersons.py +++ b/ksdb/management/commands/ingestpersons.py @@ -1,126 +1,94 @@ -# ingestpersons.py -from django.core.management.base import BaseCommand, CommandError -from ksdb.models import person, degree, person_degree_link, IdSeq -from ksdb.forms import PersonForm, DegreeForm +from django.core.management.base import BaseCommand +from rdflib.term import URIRef -import rdflib -from rdflib.term import URIRef, Literal +from ksdb.models import degree, person, person_degree_link +from ksdb.rdf_ingest_utils import fetch_rdf_graph, next_sequence_value, parse_statements, text_value -#import settings import logging -import json -logger = logging.getLogger(__name__) -class Command(BaseCommand): - help = 'Ingest Registered person' - personurl = "https://edrn.jpl.nasa.gov/cancerdataexpo/rdf-data/registered-person/@@rdf" - #Well-known URI refs - _personTypeURI = URIRef('http://edrn.nci.nih.gov/rdf/types.rdf#Person') - _firstnameURI = URIRef('http://xmlns.com/foaf/0.1/givenname') - _lastnameURI = URIRef('http://xmlns.com/foaf/0.1/surname') - _phoneURI = URIRef('http://xmlns.com/foaf/0.1/phone') - _mboxURI = URIRef('http://xmlns.com/foaf/0.1/mbox') - _degree1URI = URIRef('http://edrn.nci.nih.gov/rdf/schema.rdf#degree1') - _degree2URI = URIRef('http://edrn.nci.nih.gov/rdf/schema.rdf#degree2') - _degree3URI = URIRef('http://edrn.nci.nih.gov/rdf/schema.rdf#degree3') - def checkdegree(self, deg): - degids = [] - deglist = list(degree.objects.filter(title=deg)) - logger.warn("DEGREE") - logger.warn(len(deglist)) - if len(deglist) > 0: - logger.warn("GOT HERE") - for d in deglist: - degids.append(d.id) - else: - logger.warn("NEW") - newid = self.adddegree(deg) - degids.append(newid) - logger.warn("HERE") - logger.warn(degids) - return degids +logger = logging.getLogger(__name__) - def adddegree(self, degree): - parameters = {} - deg_id = IdSeq.objects.raw("select sequence_name, nextval('degree_seq') from degree_seq")[0].nextval - parameters["id"] = deg_id - parameters["title"] = degree.encode('utf-8').strip() - degreem = DegreeForm(parameters) - if degreem.is_valid(): - degreem.save() - return deg_id +class Command(BaseCommand): + help = "Ingest registered people from the EDRN RDF feed." + personurl = "https://edrn.jpl.nasa.gov/cancerdataexpo/rdf-data/registered-person/@@rdf" + _firstname_uri = URIRef("http://xmlns.com/foaf/0.1/givenname") + _lastname_uri = URIRef("http://xmlns.com/foaf/0.1/surname") + _phone_uri = URIRef("http://xmlns.com/foaf/0.1/phone") + _mbox_uri = URIRef("http://xmlns.com/foaf/0.1/mbox") + _degree_uris = ( + URIRef("http://edrn.nci.nih.gov/rdf/schema.rdf#degree1"), + URIRef("http://edrn.nci.nih.gov/rdf/schema.rdf#degree2"), + URIRef("http://edrn.nci.nih.gov/rdf/schema.rdf#degree3"), + ) def handle(self, *args, **options): - g = rdflib.Graph() - result = g.parse(self.personurl) + graph = fetch_rdf_graph(self.personurl) + statements = parse_statements(graph) imported_count = 0 - personStatements = self._parseRDF(g) - for per in personStatements: - if self._firstnameURI not in personStatements[per] or self._lastnameURI not in personStatements[per] or self._mboxURI not in personStatements[per]: - continue - parameters = {} - firstname = personStatements[per][self._firstnameURI] - lastname = personStatements[per][self._lastnameURI] - email = personStatements[per][self._mboxURI] - per_id = None + updated_count = 0 - if len(firstname) > 0: - parameters["firstname"] = firstname[0].encode('utf-8').strip() - if len(lastname) > 0: - parameters["lastname"] = lastname[0].encode('utf-8').strip() - if len(email) > 0: - parameters["email"] = str(email[0].encode('utf-8').strip()).replace("mailto:","") + for subject in statements: + firstname = text_value(statements[subject].get(self._firstname_uri, [""])[0]) + lastname = text_value(statements[subject].get(self._lastname_uri, [""])[0]) + email = text_value(statements[subject].get(self._mbox_uri, [""])[0]).replace("mailto:", "") + telephone = text_value(statements[subject].get(self._phone_uri, [""])[0]).replace("tel:", "") - existingpers = person.objects.filter(firstname=firstname, lastname=lastname) - if len(list(existingpers)) > 0: - per_id = existingpers[0] - else: - per_id = IdSeq.objects.raw("select sequence_name, nextval('person_seq') from person_seq")[0].nextval + if not (firstname and lastname and email): + continue - if self._phoneURI in personStatements[per]: - if len(personStatements[per][self._phoneURI]) > 0: - parameters["telephone"] = personStatements[per][self._phoneURI][0].encode('utf-8').strip() - degids = [] - if self._degree1URI in personStatements[per]: - if len(personStatements[per][self._degree1URI]) > 0: - deg = str(personStatements[per][self._degree1URI][0]) - dlist = self.checkdegree(deg) - degids += dlist - if self._degree2URI in personStatements[per]: - if len(personStatements[per][self._degree2URI]) > 0: - deg = str(personStatements[per][self._degree2URI][0]) - dlist = self.checkdegree(deg) - degids += dlist - if self._degree3URI in personStatements[per]: - if len(personStatements[per][self._degree3URI]) > 0: - deg = str(personStatements[per][self._degree3URI][0]) - dlist = self.checkdegree(deg) - degids += dlist + person_obj = person.objects.filter(firstname=firstname, lastname=lastname).first() + if person_obj is None and email: + person_obj = person.objects.filter(email=email).first() - parameters["id"] = per_id - personm = PersonForm(parameters) - if personm.is_valid(): - personm.save() - for degid in degids: - logger.warn(per_id) - logger.warn(degid) - person_degree_linkm = person_degree_link(personid = per_id, degreeid = int(degid)) - person_degree_linkm.save() + if person_obj is None: + person_obj = person.objects.create( + id=next_sequence_value("person_seq"), + firstname=firstname, + lastname=lastname, + email=email, + telephone=telephone or None, + extension=None, + description="", + degrees=None, + dcp=False, + dcb=False, + ) imported_count += 1 + else: + person_obj.firstname = firstname + person_obj.lastname = lastname + person_obj.email = email + if telephone: + person_obj.telephone = telephone + if person_obj.description is None: + person_obj.description = "" + if person_obj.dcp is None: + person_obj.dcp = False + if person_obj.dcb is None: + person_obj.dcb = False + person_obj.save() + updated_count += 1 - logger.info("Successfully imported {} persons from cancerdataexpo rdf into KSDB.".format(imported_count)) - - def _parseRDF(self, graph): - statements = {} - for s, p, o in graph: - if s not in statements: - statements[s] = {} - predicates = statements[s] - if p not in predicates: - predicates[p] = [] - predicates[p].append(o) - return statements - + for degree_uri in self._degree_uris: + degree_title = text_value(statements[subject].get(degree_uri, [""])[0]) + if not degree_title: + continue + degree_obj = degree.objects.filter(title=degree_title).first() + if degree_obj is None: + degree_obj = degree.objects.create( + id=next_sequence_value("degree_seq"), + title=degree_title, + description="", + ) + person_degree_link.objects.get_or_create( + personid=person_obj.id, + degreeid=degree_obj.id, + ) + logger.info( + "Successfully imported %s persons and updated %s persons from cancerdataexpo rdf into KSDB.", + imported_count, + updated_count, + ) diff --git a/ksdb/management/commands/ingestprotocols.py b/ksdb/management/commands/ingestprotocols.py index d1fedf7..081c729 100755 --- a/ksdb/management/commands/ingestprotocols.py +++ b/ksdb/management/commands/ingestprotocols.py @@ -1,48 +1,106 @@ -# ingestorgans.py -from django.core.management.base import BaseCommand, CommandError -from ksdb.models import organ, IdSeq -from ksdb.forms import OrganForm +from django.core.management.base import BaseCommand +from rdflib.term import URIRef -import rdflib -from rdflib.term import URIRef, Literal +from ksdb.models import protocol, protocol_program_link +from ksdb.rdf_ingest_utils import ( + ensure_default_programs, + fetch_rdf_graph, + next_sequence_value, + parse_protocol_datetime, + parse_statements, + text_value, +) -#import settings import logging -import json + + logger = logging.getLogger(__name__) + class Command(BaseCommand): - help = 'Closes the specified poll for voting' - organurl = "https://edrn.jpl.nasa.gov/cancerdataexpo/rdf-data/body-systems/@@rdf" - _bodySystemTitle = URIRef("http://purl.org/dc/terms/title") + help = "Ingest protocols from the EDRN RDF feed." + protocolurl = "https://edrn.jpl.nasa.gov/cancerdataexpo/rdf-data/protocols/@@rdf" + _rdf_type = URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type") + _protocol_type_uri = URIRef("http://edrn.nci.nih.gov/rdf/types.rdf#Protocol") + _aims_uri = URIRef("http://edrn.nci.nih.gov/rdf/schema.rdf#aims") + _abstract_uri = URIRef("http://edrn.nci.nih.gov/rdf/schema.rdf#objective") + _finish_date_uri = URIRef("http://edrn.nci.nih.gov/rdf/schema.rdf#estimatedFinishDate") + _start_date_uri = URIRef("http://edrn.nci.nih.gov/rdf/schema.rdf#startDate") + _ab_name_uri = URIRef("http://edrn.nci.nih.gov/rdf/schema.rdf#abbreviatedName") + _author_uri = URIRef("http://purl.org/dc/terms/author") + _title_uri = URIRef("http://purl.org/dc/terms/title") def handle(self, *args, **options): - g = rdflib.Graph() - result = g.parse(self.organurl) - - organStatements = self._parseRDF(g) - for body in organStatements: - for title in organStatements[body][self._bodySystemTitle]: - parameters = {} - org_id = IdSeq.objects.raw("select sequence_name, nextval('organ_seq') from organ_seq")[0].nextval - parameters["id"] = org_id - parameters["name"] = title - parameters["description"] = "" - organm = OrganForm(parameters) - - if organm.is_valid(): - organm.save() - print "Successfully imported organs from dmcc rdf." - - def _parseRDF(self, graph): - statements = {} - for s, p, o in graph: - if s not in statements: - statements[s] = {} - predicates = statements[s] - if p not in predicates: - predicates[p] = [] - predicates[p].append(o) - return statements + ensure_default_programs() + graph = fetch_rdf_graph(self.protocolurl) + statements = parse_statements(graph) + imported_count = 0 + updated_count = 0 + + for subject in statements: + if self._protocol_type_uri not in statements[subject].get(self._rdf_type, []): + continue + + title = text_value(statements[subject].get(self._title_uri, [""])[0]) + if not title: + continue + + edrn_id = text_value(subject) + aims = text_value(statements[subject].get(self._aims_uri, [""])[0]) + abstract = text_value(statements[subject].get(self._abstract_uri, [""])[0]) + shortname = text_value(statements[subject].get(self._ab_name_uri, [""])[0]) + author = text_value(statements[subject].get(self._author_uri, [""])[0]) + end_date = parse_protocol_datetime(statements[subject].get(self._finish_date_uri, [""])[0]) + start_date = parse_protocol_datetime(statements[subject].get(self._start_date_uri, [""])[0]) + + defaults = { + "title": title, + "edrn_id": edrn_id, + "shortname": shortname or None, + "organs": "", + "fundedsites": "", + "institutions": "", + "programs": "2", + "pis": author or "", + "cis": "", + "start_date": start_date, + "end_date": end_date, + "site_contact": None, + "site_contact_email": None, + "irb_approval": None, + "irb_approval_num": None, + "irb_contact": None, + "irb_contact_email": None, + "hum_sub_train": None, + "abstract": abstract or "", + "aims": aims or "", + "publications": "", + "data_custodians": "", + } + + protocol_obj = protocol.objects.filter(edrn_id=edrn_id).first() + if protocol_obj is None: + protocol_obj = protocol.objects.filter(title=title).first() + + if protocol_obj is None: + protocol_obj = protocol.objects.create( + id=next_sequence_value("protocol_seq"), + **defaults + ) + imported_count += 1 + else: + for field, value in defaults.items(): + setattr(protocol_obj, field, value) + protocol_obj.save() + updated_count += 1 + protocol_program_link.objects.get_or_create( + protocolid=protocol_obj.id, + programid=2, + ) + logger.info( + "Successfully imported %s protocols and updated %s protocols from cancerdataexpo rdf.", + imported_count, + updated_count, + ) diff --git a/ksdb/management/commands/ingestpublications.py b/ksdb/management/commands/ingestpublications.py index 93a7269..729f692 100755 --- a/ksdb/management/commands/ingestpublications.py +++ b/ksdb/management/commands/ingestpublications.py @@ -1,101 +1,82 @@ -# ingestpublications.py -from django.core.management.base import BaseCommand, CommandError -from ksdb.models import publication, IdSeq -from ksdb.forms import PublicationForm +from django.core.management.base import BaseCommand +from rdflib.term import URIRef -import rdflib -from rdflib.term import URIRef, Literal +from ksdb.models import publication, publication_program_link +from ksdb.rdf_ingest_utils import ( + ensure_default_programs, + fetch_rdf_graph, + next_sequence_value, + parse_statements, + text_value, +) -#import settings import logging -import json + + logger = logging.getLogger(__name__) + class Command(BaseCommand): - help = 'Ingest publications' + help = "Ingest publications from the EDRN RDF feed." publicationurl = "https://edrn.jpl.nasa.gov/cancerdataexpo/rdf-data/publications/@@rdf" - #Well-known URI refs - _publicationTypeURI = URIRef('http://edrn.nci.nih.gov/rdf/types.rdf#Publication') - _typeURI = URIRef('http://www.w3.org/1999/02/22-rdf-syntax-ns#type') - _pmidURI = URIRef('http://edrn.nci.nih.gov/rdf/schema.rdf#pmid') - _yearURI = URIRef('http://edrn.nci.nih.gov/rdf/schema.rdf#year') - _journalURI = URIRef('http://edrn.nci.nih.gov/rdf/schema.rdf#journal') - _authorURI = URIRef('http://purl.org/dc/terms/author') - _titleURI = URIRef('http://purl.org/dc/terms/title') - _volumeURI = URIRef('http://edrn.nci.nih.gov/rdf/schema.rdf#volume') + _pmid_uri = URIRef("http://edrn.nci.nih.gov/rdf/schema.rdf#pmid") + _year_uri = URIRef("http://edrn.nci.nih.gov/rdf/schema.rdf#year") + _journal_uri = URIRef("http://edrn.nci.nih.gov/rdf/schema.rdf#journal") + _author_uri = URIRef("http://purl.org/dc/terms/author") + _title_uri = URIRef("http://purl.org/dc/terms/title") def handle(self, *args, **options): - g = rdflib.Graph() - result = g.parse(self.publicationurl) + ensure_default_programs() + graph = fetch_rdf_graph(self.publicationurl) + statements = parse_statements(graph) imported_count = 0 - pubStatements = self._parseRDF(g) - for pub in pubStatements: - if self._titleURI not in pubStatements[pub] or self._pmidURI not in pubStatements[pub] or self._yearURI not in pubStatements[pub] or self._journalURI not in pubStatements[pub] or self._authorURI not in pubStatements[pub]: - continue - titles = pubStatements[pub][self._titleURI] - pmids = pubStatements[pub][self._pmidURI] - years = pubStatements[pub][self._yearURI] - journals = pubStatements[pub][self._journalURI] - authors = pubStatements[pub][self._authorURI] - title = None - pmid = None - year = None - journal = None - if len(titles) < 1: - continue - else: - title = titles[0] - if len(pmids) < 1: - continue - else: - pmid = pmids[0] - if len(years) < 1: + updated_count = 0 + + for subject in statements: + title = text_value(statements[subject].get(self._title_uri, [""])[0]) + pmid = text_value(statements[subject].get(self._pmid_uri, [""])[0]) + year = text_value(statements[subject].get(self._year_uri, [""])[0]) + journal = text_value(statements[subject].get(self._journal_uri, [""])[0]) + authors = [text_value(author) for author in statements[subject].get(self._author_uri, [])] + + if not (title and pmid and year and journal and authors): continue - else: - year = years[0] - if len(journals) < 1: + + try: + pubyear = int(year) + except (TypeError, ValueError): continue - else: - journal = journals[0] - - if not self._exists(pmid.encode('utf-8').strip()): - parameters = {} - pub_id = IdSeq.objects.raw("select sequence_name, nextval('publication_seq') from publication_seq")[0].nextval - - parameters["id"] = pub_id - parameters["title"] = title.encode('utf-8').strip() - parameters["authors"] = ", ".join(authors).encode('utf-8').strip() - parameters["pubmedid"] = pmid.encode('utf-8').strip() - parameters["pubyear"] = year.encode('utf-8').strip() - parameters["journal"] = journal.encode('utf-8').strip() - - publicationm = PublicationForm(parameters) - if publicationm.is_valid(): - publicationm.save() - imported_count += 1 + defaults = { + "title": title, + "authors": ", ".join([author for author in authors if author]), + "journal": journal, + "pubyear": pubyear, + "programs": "2", + } + existing = publication.objects.filter(pubmedid=pmid).first() - logger.info("Successfully imported {} publication from cancerdataexpo rdf.".format(imported_count)) - def _exists(self, pubid): - #returns true if no object exist, returns false if one or more object exist - exist = True - try: - publication.objects.get(pubmedid=pubid) - except publication.DoesNotExist: - exist = False - pass - except publication.MultipleObjectsReturned: - pass - return exist - def _parseRDF(self, graph): - statements = {} - for s, p, o in graph: - if s not in statements: - statements[s] = {} - predicates = statements[s] - if p not in predicates: - predicates[p] = [] - predicates[p].append(o) - return statements + if existing is None: + publication_obj = publication.objects.create( + id=next_sequence_value("publication_seq"), + pubmedid=pmid, + **defaults + ) + imported_count += 1 + else: + for field, value in defaults.items(): + setattr(existing, field, value) + existing.save(update_fields=list(defaults.keys())) + publication_obj = existing + updated_count += 1 + publication_program_link.objects.get_or_create( + publicationid=publication_obj.id, + programid=2, + ) + logger.info( + "Successfully imported %s publications and updated %s publications from cancerdataexpo rdf.", + imported_count, + updated_count, + ) diff --git a/ksdb/management/commands/preloadksdbdata.py b/ksdb/management/commands/preloadksdbdata.py new file mode 100644 index 0000000..88c5fee --- /dev/null +++ b/ksdb/management/commands/preloadksdbdata.py @@ -0,0 +1,45 @@ +from django.core.management import call_command +from django.core.management.base import BaseCommand + +from ksdb.models import organ, person, program, protocol, publication +from ksdb.rdf_ingest_utils import ensure_default_programs + + +class Command(BaseCommand): + help = "Preload KSDB data into an empty database." + + def add_arguments(self, parser): + parser.add_argument( + "--force", + action="store_true", + help="Run the preload even when core tables already contain rows.", + ) + + def handle(self, *args, **options): + force = options.get("force", False) + core_counts = { + "program": program.objects.count(), + "organ": organ.objects.count(), + "person": person.objects.count(), + "publication": publication.objects.count(), + "protocol": protocol.objects.count(), + } + + if not force and any(core_counts.values()): + self.stdout.write("Skipping KSDB preload because the database already contains data: {}".format(core_counts)) + return + + ensure_default_programs() + call_command("ingestorgans") + call_command("ingestpersons") + call_command("ingestpublications") + call_command("ingestprotocols") + + final_counts = { + "program": program.objects.count(), + "organ": organ.objects.count(), + "person": person.objects.count(), + "publication": publication.objects.count(), + "protocol": protocol.objects.count(), + } + self.stdout.write("Completed KSDB preload: {}".format(final_counts)) diff --git a/ksdb/models.py b/ksdb/models.py index 00b10da..e4b2831 100755 --- a/ksdb/models.py +++ b/ksdb/models.py @@ -5,19 +5,20 @@ #Protocol related models class protocol(models.Model): id = models.IntegerField(primary_key=True) - title = models.CharField(max_length=128) - shortname = models.CharField(max_length=128) + title = models.CharField(max_length=1000) + edrn_id = models.CharField(max_length=1000, blank=True, null=True) + shortname = models.CharField(max_length=128, blank=True, null=True) organs = models.CharField(max_length=1000, blank=True, null=True) fundedsites = models.CharField(max_length=1000, blank=True, null=True) - institutions = models.CharField(max_length=1000) + institutions = models.CharField(max_length=1000, blank=True, null=True) programs = models.CharField(max_length=1000) - pis = models.CharField(max_length=1000) + pis = models.CharField(max_length=1000, blank=True, null=True) cis = models.CharField(max_length=1000, blank=True, null=True) start_date = models.DateTimeField(blank=True, null=True) end_date = models.DateTimeField(blank=True, null=True) site_contact = models.CharField(max_length=500, blank=True, null=True) site_contact_email = models.EmailField(max_length=500, blank=True, null=True) - irb_approval = models.CharField(max_length=128) + irb_approval = models.CharField(max_length=128, blank=True, null=True) irb_approval_num = models.CharField(max_length=128, blank=True, null=True) irb_contact = models.CharField(max_length=500, blank=True, null=True) irb_contact_email = models.EmailField(max_length=500, blank=True, null=True) @@ -236,6 +237,15 @@ class Meta: db_table = u'fundedsite_program_link' unique_together = (("programid", "fundedsiteid"),) +class fundedsite_institution_visible_id(models.Model): + fundedsiteid = models.IntegerField() + institutionid = models.IntegerField() + fundedsite_institution_visible_id = models.IntegerField() + + class Meta: + db_table = u'fundedsite_institution_visible_id' + unique_together = (("fundedsiteid", "institutionid", "fundedsite_institution_visible_id"),) + class fundedsite_institution_link(models.Model): fundedsiteid = models.IntegerField() institutionid = models.IntegerField() @@ -475,3 +485,44 @@ class IdSeq(models.Model): log_cnt = models.IntegerField() is_cycled = models.BooleanField() is_called = models.BooleanField() + +class labcas_user_profiles(models.Model): + id = models.AutoField(primary_key=True) + username = models.CharField(max_length=1024) + search_profile = models.TextField(blank=True, null=True) + search_name = models.CharField(max_length=1024) + + class Meta: + db_table = u'labcas_user_profiles' + unique_together = (("username", "search_name"),) + +class labcas_user_dicom_states(models.Model): + id = models.AutoField(primary_key=True) + username = models.CharField(max_length=1024) + dicom_state = models.TextField(blank=True, null=True) + state_name = models.CharField(max_length=1024) + + class Meta: + db_table = u'labcas_user_dicom_states' + unique_together = (("username", "state_name"),) + +class labcas_person(models.Model): + id = models.AutoField(primary_key=True) + username = models.CharField(max_length=1024) + firstname = models.CharField(max_length=1024) + lastname = models.CharField(max_length=1024) + email = models.CharField(max_length=1024) + accepted = models.BooleanField() + firsttime = models.BooleanField() + time_accepted = models.DateTimeField(blank=True, null=True) + + class Meta: + db_table = u'labcas_person' + +class labcas_assaytype(models.Model): + id = models.IntegerField(primary_key=True) + name = models.CharField(max_length=2048) + alias = models.CharField(max_length=2048) + + class Meta: + db_table = u'labcas_assaytype' diff --git a/ksdb/publishRDF.py b/ksdb/publishRDF.py index 63fcfb9..fa9b149 100755 --- a/ksdb/publishRDF.py +++ b/ksdb/publishRDF.py @@ -1,7 +1,7 @@ # publishRDF.py from django.http import HttpResponse from django.core.management import call_command -from io import StringIO +from io import BytesIO, StringIO #import settings import logging @@ -9,7 +9,6 @@ logger = logging.getLogger(__name__) from openpyxl import Workbook -from openpyxl.writer.excel import save_virtual_workbook def publishrdf(request): if request.method == 'GET': @@ -42,7 +41,7 @@ def publishhtml(request): html_resp = html_resp[2:] if html_resp.endswith("'"): html_resp = html_resp[:-1] - html_resp = html_resp.replace("\\n", "\n") + html_resp = html_resp.replace("\\n", "\n").encode('utf-8') return HttpResponse(html_resp.strip(), content_type='text/plain; charset=us-ascii') def publishexcel(request): @@ -70,6 +69,8 @@ def publishexcel(request): for row in excel_resp: worksheet.append(row) - response = HttpResponse(content=save_virtual_workbook(workbook), content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') + excel_buffer = BytesIO() + workbook.save(excel_buffer) + response = HttpResponse(content=excel_buffer.getvalue(), content_type='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet') response['Content-Disposition'] = 'attachment; filename=ksdb.xlsx' return response diff --git a/ksdb/rdf_ingest_utils.py b/ksdb/rdf_ingest_utils.py new file mode 100644 index 0000000..123f075 --- /dev/null +++ b/ksdb/rdf_ingest_utils.py @@ -0,0 +1,68 @@ +from datetime import datetime + +import requests +import rdflib +from django.db import connection +from django.utils.timezone import make_aware, utc + +from ksdb.models import program + + +def fetch_rdf_graph(url, timeout=120): + response = requests.get(url, timeout=timeout) + response.raise_for_status() + graph = rdflib.Graph() + graph.parse(data=response.text, format="xml") + return graph + + +def parse_statements(graph): + statements = {} + for subject, predicate, obj in graph: + predicates = statements.setdefault(subject, {}) + predicates.setdefault(predicate, []).append(obj) + return statements + + +def text_value(value): + if value is None: + return "" + return " ".join(str(value).strip().split()) + + +def next_sequence_value(sequence_name): + with connection.cursor() as cursor: + cursor.execute("select nextval(%s)", [sequence_name]) + return cursor.fetchone()[0] + + +def parse_protocol_datetime(value): + cleaned = text_value(value) + if not cleaned: + return None + for fmt in ("%b %d %Y %I:%M%p", "%b %d %Y %I:%M %p"): + try: + parsed = datetime.strptime(cleaned, fmt) + return make_aware(parsed, utc) + except ValueError: + continue + return None + + +def ensure_default_programs(): + program.objects.update_or_create( + id=1, + defaults={ + "title": "MCL", + "abbreviation": "MCL", + "description": "Seeded default program for local KSDB bootstrap.", + }, + ) + program.objects.update_or_create( + id=2, + defaults={ + "title": "EDRN", + "abbreviation": "EDRN", + "description": "Seeded default program for EDRN RDF imports.", + }, + ) diff --git a/ksdb/urls.py b/ksdb/urls.py index aa5074d..7350a70 100755 --- a/ksdb/urls.py +++ b/ksdb/urls.py @@ -2,7 +2,7 @@ from django.conf.urls import url -from ksdb import protocols, program, person, institution, publication, fundedsite, group, committee, view, organ, discipline, specimentype, species, disease, degree, querypubmed, publishRDF, ingestRDF, ModalView, ekeutils, ksdbviews +from ksdb import protocols, program, person, institution, publication, fundedsite, group, committee, view, organ, labcas_assaytype, discipline, specimentype, species, disease, degree, querypubmed, publishRDF, ingestRDF, ModalView, ekeutils, ksdbviews from sitemain import settings urlpatterns = [ @@ -16,6 +16,7 @@ url(r'^groupinput/$', group.group_input, name='groupinput'), url(r'^committeeinput/$', committee.committee_input, name='committeeinput'), url(r'^organinput/$', organ.organ_input, name='organinput'), + url(r'^assaytypeinput/$', labcas_assaytype.assaytype_input, name='assaytypeinput'), url(r'^disciplineinput/$', discipline.discipline_input, name='disciplineinput'), url(r'^speciesinput/$', species.species_input, name='speciesinput'), url(r'^specimentypeinput/$', specimentype.specimentype_input, name='specimentypeinput'), @@ -35,6 +36,7 @@ url(r'^deleteperson/$', person.delete_person, name='deleteperson'), url(r'^deletepublication/$', publication.delete_publication, name='deletepublication'), url(r'^deleteorgan/$', organ.delete_organ, name='deleteorgan'), + url(r'^deleteassaytype/$', labcas_assaytype.delete_assaytype, name='deleteassaytype'), url(r'^deletespecimentype/$', specimentype.delete_specimentype, name='deletespecimentype'), url(r'^deletespecies/$', species.delete_species, name='deletespecies'), url(r'^deletediscipline/$', discipline.delete_discipline, name='deletediscipline'), @@ -51,6 +53,7 @@ url(r'^committeeview/$', ksdbviews.CommitteeView.as_view(), name='committeeview'), url(r'^protocolview/$', ksdbviews.ProtocolView.as_view(), name='protocolview'), url(r'^organview/$', ksdbviews.OrganView.as_view(), name='organview'), + url(r'^assaytypeview/$', ksdbviews.AssayTypeView.as_view(), name='assaytypeview'), url(r'^disciplineview/$', ksdbviews.DisciplineView.as_view(), name='disciplineview'), url(r'^speciesview/$', ksdbviews.SpeciesView.as_view(), name='speciesview'), url(r'^specimentypeview/$', ksdbviews.SpecimenTypeView.as_view(), name='specimentypeview'), diff --git a/ksdb/view.py b/ksdb/view.py index 0980598..236f8af 100755 --- a/ksdb/view.py +++ b/ksdb/view.py @@ -17,9 +17,9 @@ def setupTable(frame, objtype): objid = frameline[0] frameline.insert(0, "") for j in range(1, len(frameline)): - #ignore unicode characters - if isinstance(frameline[j], basestring): - frameline[j] = frameline[j].encode('ascii', 'ignore') + # Normalize display text for Python 3 without leaking byte reprs. + if isinstance(frameline[j], str): + frameline[j] = frameline[j].encode('ascii', 'ignore').decode('ascii') frameline[j] = ""+str(frameline[j])+"" newframe.append(frameline) return newframe @@ -35,6 +35,7 @@ def view_service(request): institutionheaders = ["Select","Institution ID", "Name", "Abbreviation"] programheaders = ["Select","Program ID", "Title", "Abbreviation"] organheaders = ["Select","Organ ID", "Name"] + assaytypeheaders = ["Select","Assay ID", "Name"] disciplineheaders = ["Select","Discipline ID", "Name"] speciesheaders = ["Select","Species ID", "Name"] specimentypeheaders = ["Select","Specimen Type ID", "Name"] @@ -52,6 +53,7 @@ def view_service(request): 'institutionheaders': institutionheaders, 'programheaders': programheaders, 'organheaders': organheaders, + 'assaytypeheaders': assaytypeheaders, 'disciplineheaders': disciplineheaders, 'speciesheaders': speciesheaders, 'specimentypeheaders': specimentypeheaders, @@ -61,4 +63,3 @@ def view_service(request): 'committeeheaders': committeeheaders, } ) - diff --git a/requirements-docker.txt b/requirements-docker.txt new file mode 100644 index 0000000..49bffb4 --- /dev/null +++ b/requirements-docker.txt @@ -0,0 +1,8 @@ +Django==1.9.13 +gunicorn==20.1.0 +psycopg2-binary==2.9.9 +requests==2.31.0 +simplejson==3.19.3 +biopython==1.79 +rdflib==6.3.2 +openpyxl==3.1.2 diff --git a/sitemain/settings_docker.py b/sitemain/settings_docker.py new file mode 100644 index 0000000..ac45362 --- /dev/null +++ b/sitemain/settings_docker.py @@ -0,0 +1,128 @@ +import datetime +import os + +from django.db.backends.postgresql import utils as postgresql_utils +from django.utils.timezone import utc + + +def _patched_utc_tzinfo_factory(offset): + # Django 1.9 expects an integer offset, but newer psycopg2 releases hand + # back a zero timedelta for UTC connections. + if isinstance(offset, datetime.timedelta): + offset = int(offset.total_seconds() // 60) + if offset != 0: + raise AssertionError("database connection isn't set to UTC") + return utc + + +postgresql_utils.utc_tzinfo_factory = _patched_utc_tzinfo_factory + +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +RDF_URL = os.environ.get("RDF_URL", "http://localhost:8000/") +MCL_URL = os.environ.get("MCL_URL", "http://localhost:8000/") + +SECRET_KEY = os.environ.get("DJANGO_SECRET_KEY", "ksdb-local-dev-secret-key") + +DEBUG = os.environ.get("DJANGO_DEBUG", "1").lower() in ("1", "true", "yes", "on") + +ALLOWED_HOSTS = [ + host.strip() + for host in os.environ.get("DJANGO_ALLOWED_HOSTS", "*").split(",") + if host.strip() +] + +INSTALLED_APPS = ( + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "ksdb", +) + +MIDDLEWARE_CLASSES = ( + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.auth.middleware.SessionAuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django.middleware.security.SecurityMiddleware", +) + +ROOT_URLCONF = "sitemain.urls" + +TEMPLATES = [ + { + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [os.path.join(BASE_DIR, "templates")], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", + ], + }, + }, +] + +WSGI_APPLICATION = "sitemain.wsgi_docker.application" + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.postgresql_psycopg2", + "NAME": os.environ.get("DATABASE_NAME", "ksdb"), + "USER": os.environ.get("DATABASE_USER", "ksdb"), + "PASSWORD": os.environ.get("DATABASE_PASSWORD", "ksdb"), + "HOST": os.environ.get("DATABASE_HOST", "db"), + "PORT": os.environ.get("DATABASE_PORT", "5432"), + } +} + +LANGUAGE_CODE = "en-us" +TIME_ZONE = "UTC" +USE_I18N = True +USE_L10N = True +USE_TZ = True + +STATIC_URL = "/static/" +STATIC_ROOT = os.path.join(BASE_DIR, "staticfiles") + +LOGIN_REDIRECT_URL = "/ksdb/view/" + +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "simple": { + "format": "%(levelname)s %(name)s %(message)s", + }, + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "simple", + }, + }, + "root": { + "handlers": ["console"], + "level": "INFO", + }, + "loggers": { + "django": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + "ksdb": { + "handlers": ["console"], + "level": "INFO", + "propagate": False, + }, + }, +} diff --git a/sitemain/wsgi_docker.py b/sitemain/wsgi_docker.py new file mode 100644 index 0000000..0c836e0 --- /dev/null +++ b/sitemain/wsgi_docker.py @@ -0,0 +1,8 @@ +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "sitemain.settings_docker") + +application = get_wsgi_application() + diff --git a/templates/assaytypeinput.html b/templates/assaytypeinput.html new file mode 100755 index 0000000..52b93f2 --- /dev/null +++ b/templates/assaytypeinput.html @@ -0,0 +1,89 @@ +{% extends request.GET.modal|yesno:'lib.html,base.html' %} +{% block content %} + +
+
+

{{ action }} Assay Type: {{ name }}

+

+ Please update the KSDB database with accurate Assay Type information. +

+
+
+
+
+ {% csrf_token %} +
+ + +
+
+ + +
+
+
+ +
+
+ {% ifequal action "Edit" %} + + + {% endifequal %} + {% ifequal action "New" %} + + {% endifequal %} +
+ +
+
+
+
+
+ + +
+
+
+ + + + +{% endblock %} diff --git a/templates/ksdb.html b/templates/ksdb.html index c273640..0bb02ae 100755 --- a/templates/ksdb.html +++ b/templates/ksdb.html @@ -89,6 +89,8 @@

Welcome to KSDB!

  • Degrees
    • +
  • Assay Type +
  • KSDB RDFs
    • @@ -126,6 +128,9 @@

    Welcome to KSDB!

    {% include "ksdbobjview.html" with ksdbobj="organ" ksdbobjtitle="Organ" ksdbobjheaders=organheaders %}
    +
    + {% include "ksdbobjview.html" with ksdbobj="assaytype" ksdbobjtitle="Assay Type" ksdbobjheaders=assaytypeheaders %} +
    {% include "ksdbobjview.html" with ksdbobj="discipline" ksdbobjtitle="Discipline" ksdbobjheaders=disciplineheaders %}
    diff --git a/tinymce/__init__.py b/tinymce/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tinymce/__init__.py @@ -0,0 +1 @@ + diff --git a/tinymce/models.py b/tinymce/models.py new file mode 100644 index 0000000..ebbbf1a --- /dev/null +++ b/tinymce/models.py @@ -0,0 +1,6 @@ +from django.db import models + + +class HTMLField(models.TextField): + pass + From 4ddf8e4b664ccfa4ad6588facb54f4b51d864b24 Mon Sep 17 00:00:00 2001 From: David Liu Date: Tue, 5 May 2026 12:17:05 -0700 Subject: [PATCH 2/2] Address Docker PR review feedback --- Dockerfile | 11 +- README.md | 9 +- docker-compose.yml | 35 +++++- docker/entrypoint.sh | 69 ----------- docker/init.sh | 71 +++++++++++ ksdb/ksdbviews.py | 70 ++++++----- ksdb/management/commands/ingestorgans.py | 28 +++-- ksdb/management/commands/ingestpersons.py | 106 ++++++++-------- ksdb/management/commands/ingestprotocols.py | 114 +++++++++--------- .../management/commands/ingestpublications.py | 78 ++++++------ 10 files changed, 322 insertions(+), 269 deletions(-) create mode 100755 docker/init.sh diff --git a/Dockerfile b/Dockerfile index 3ef435b..830da6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,17 +5,18 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ WORKDIR /app +COPY requirements-docker.txt /app/requirements-docker.txt RUN apt-get update \ && apt-get install -y --no-install-recommends build-essential ca-certificates \ + && pip install --no-cache-dir --upgrade pip setuptools wheel \ + && pip install --no-cache-dir -r /app/requirements-docker.txt \ + && apt-get purge -y --auto-remove build-essential \ && rm -rf /var/lib/apt/lists/* -COPY requirements-docker.txt /app/requirements-docker.txt -RUN pip install --no-cache-dir --upgrade pip setuptools wheel \ - && pip install --no-cache-dir -r /app/requirements-docker.txt - COPY . /app COPY docker/entrypoint.sh /usr/local/bin/ksdb-entrypoint -RUN chmod +x /usr/local/bin/ksdb-entrypoint +COPY docker/init.sh /usr/local/bin/ksdb-init +RUN chmod +x /usr/local/bin/ksdb-entrypoint /usr/local/bin/ksdb-init EXPOSE 8000 diff --git a/README.md b/README.md index 8839812..e9ebc60 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Knowledge System DB: The goal of this system is to maintain generalized project information to be used for any bioinformatics knowledge system that the JPL Informatics Center supports. For example, EDRN or MCL. -#DEPENDENCIES +# DEPENDENCIES sudo yum install postgresql-server postgresql-devel gcc openldap-devel openssl openssl-devel python3: @@ -16,9 +16,9 @@ pip3: curl -O https://bootstrap.pypa.io/get-pip.py sudo python3 get-pip.py -#SETUP POSTGRES +# SETUP POSTGRES -#setup postgres +# setup postgres sudo service postgresql initdb sudo chkconfig postgresql on sudo service postgresql start @@ -29,7 +29,7 @@ pip3: alter user edrn password 'edrn'; GRANT ALL ON database ksdb TO edrn; -#insert tables +# insert tables ctrl^D once to exit out of database sudo service postgresql restart export PGPASSWORD=edrn @@ -94,4 +94,5 @@ Notes: - `./deploy.sh` builds the image, starts Postgres and Django, loads the schema, preloads supported KSDB data, and exposes the app on `http://localhost:8000/` - `./deploy.sh fresh` also deletes the Docker volume and rebuilds the database from scratch +- One-time database initialization runs in a separate Compose `init` service before the web service starts - The default login is `admin` / `admin` unless you override the compose environment variables diff --git a/docker-compose.yml b/docker-compose.yml index 4be72d2..715a6a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,11 +18,11 @@ services: retries: 20 start_period: 5s - web: + init: build: context: . dockerfile: Dockerfile - restart: unless-stopped + restart: "no" depends_on: db: condition: service_healthy @@ -42,8 +42,39 @@ services: DJANGO_SUPERUSER_EMAIL: ${DJANGO_SUPERUSER_EMAIL:-admin@example.com} DJANGO_SUPERUSER_PASSWORD: ${DJANGO_SUPERUSER_PASSWORD:-admin} KSDB_PRELOAD_DATA: ${KSDB_PRELOAD_DATA:-1} + volumes: + - staticfiles:/app/staticfiles + command: ["/usr/local/bin/ksdb-init"] + + web: + build: + context: . + dockerfile: Dockerfile + restart: unless-stopped + depends_on: + init: + condition: service_completed_successfully + environment: + DJANGO_SETTINGS_MODULE: sitemain.settings_docker + DATABASE_NAME: ${POSTGRES_DB:-ksdb} + DATABASE_USER: ${POSTGRES_USER:-ksdb} + DATABASE_PASSWORD: ${POSTGRES_PASSWORD:-ksdb} + DATABASE_HOST: db + DATABASE_PORT: 5432 + DJANGO_DEBUG: ${DJANGO_DEBUG:-1} + DJANGO_SECRET_KEY: ${DJANGO_SECRET_KEY:-ksdb-local-dev-secret-key} + DJANGO_ALLOWED_HOSTS: ${DJANGO_ALLOWED_HOSTS:-*} + RDF_URL: ${RDF_URL:-http://localhost:8000/} + MCL_URL: ${MCL_URL:-http://localhost:8000/} + DJANGO_SUPERUSER_USERNAME: ${DJANGO_SUPERUSER_USERNAME:-admin} + DJANGO_SUPERUSER_EMAIL: ${DJANGO_SUPERUSER_EMAIL:-admin@example.com} + DJANGO_SUPERUSER_PASSWORD: ${DJANGO_SUPERUSER_PASSWORD:-admin} + KSDB_PRELOAD_DATA: ${KSDB_PRELOAD_DATA:-1} + volumes: + - staticfiles:/app/staticfiles ports: - "${KSDB_PORT:-8000}:8000" volumes: postgres_data: + staticfiles: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 4876d9b..ab68823 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -4,73 +4,4 @@ set -eu cp /app/sitemain/settings_docker.py /app/sitemain/settings.py cp /app/sitemain/wsgi_docker.py /app/sitemain/wsgi.py -python - <<'PY' -import os -import time - -import psycopg2 - -settings = { - "dbname": os.environ.get("DATABASE_NAME", "ksdb"), - "user": os.environ.get("DATABASE_USER", "ksdb"), - "password": os.environ.get("DATABASE_PASSWORD", "ksdb"), - "host": os.environ.get("DATABASE_HOST", "db"), - "port": os.environ.get("DATABASE_PORT", "5432"), -} - -for attempt in range(60): - try: - connection = psycopg2.connect(**settings) - except Exception as exc: - print("Waiting for database ({}/60): {}".format(attempt + 1, exc)) - time.sleep(2) - else: - connection.close() - print("Database is ready.") - break -else: - raise SystemExit("Database never became ready.") -PY - -python manage.py migrate --noinput --run-syncdb - -case "${KSDB_PRELOAD_DATA:-1}" in - 1|true|TRUE|yes|YES|on|ON) - python manage.py preloadksdbdata - ;; -esac - -python manage.py collectstatic --noinput - -if [ -n "${DJANGO_SUPERUSER_USERNAME:-}" ] && [ -n "${DJANGO_SUPERUSER_PASSWORD:-}" ]; then -python - <<'PY' -import os - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", os.environ.get("DJANGO_SETTINGS_MODULE", "sitemain.settings_docker")) - -import django - -django.setup() - -from django.contrib.auth import get_user_model - -User = get_user_model() -username = os.environ["DJANGO_SUPERUSER_USERNAME"] -email = os.environ.get("DJANGO_SUPERUSER_EMAIL", "") -password = os.environ["DJANGO_SUPERUSER_PASSWORD"] - -user, created = User.objects.get_or_create( - username=username, - defaults={"email": email, "is_staff": True, "is_superuser": True}, -) -user.email = email -user.is_staff = True -user.is_superuser = True -user.set_password(password) -user.save() - -print("{} Django superuser '{}'.".format("Created" if created else "Updated", username)) -PY -fi - exec "$@" diff --git a/docker/init.sh b/docker/init.sh new file mode 100755 index 0000000..97fcecb --- /dev/null +++ b/docker/init.sh @@ -0,0 +1,71 @@ +#!/bin/sh +set -eu + +python - <<'PY' +import os +import time + +import psycopg2 + +settings = { + "dbname": os.environ.get("DATABASE_NAME", "ksdb"), + "user": os.environ.get("DATABASE_USER", "ksdb"), + "password": os.environ.get("DATABASE_PASSWORD", "ksdb"), + "host": os.environ.get("DATABASE_HOST", "db"), + "port": os.environ.get("DATABASE_PORT", "5432"), +} + +for attempt in range(60): + try: + connection = psycopg2.connect(**settings) + except Exception as exc: + print("Waiting for database ({}/60): {}".format(attempt + 1, exc)) + time.sleep(2) + else: + connection.close() + print("Database is ready.") + break +else: + raise SystemExit("Database never became ready.") +PY + +python manage.py migrate --noinput --run-syncdb + +case "${KSDB_PRELOAD_DATA:-1}" in + 1|true|TRUE|yes|YES|on|ON) + python manage.py preloadksdbdata + ;; +esac + +python manage.py collectstatic --noinput + +if [ -n "${DJANGO_SUPERUSER_USERNAME:-}" ] && [ -n "${DJANGO_SUPERUSER_PASSWORD:-}" ]; then +python - <<'PY' +import os + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", os.environ.get("DJANGO_SETTINGS_MODULE", "sitemain.settings_docker")) + +import django + +django.setup() + +from django.contrib.auth import get_user_model + +User = get_user_model() +username = os.environ["DJANGO_SUPERUSER_USERNAME"] +email = os.environ.get("DJANGO_SUPERUSER_EMAIL", "") +password = os.environ["DJANGO_SUPERUSER_PASSWORD"] + +user, created = User.objects.get_or_create( + username=username, + defaults={"email": email, "is_staff": True, "is_superuser": True}, +) +user.email = email +user.is_staff = True +user.is_superuser = True +user.set_password(password) +user.save() + +print("{} Django superuser '{}'.".format("Created" if created else "Updated", username)) +PY +fi diff --git a/ksdb/ksdbviews.py b/ksdb/ksdbviews.py index d8c2f43..c90d8cc 100644 --- a/ksdb/ksdbviews.py +++ b/ksdb/ksdbviews.py @@ -1,8 +1,18 @@ from django.db.models import Q +from django.utils.html import format_html from ksdb.models import person, publication, program, institution, fundedsite, protocol, organ, labcas_assaytype, degree, group, disease, committee, discipline, species, specimentype from ksdb.ekeutils import _KSDBhref, getPersonNameByID, getProgramTitleByID from django_datatables_view.base_datatable_view import BaseDatatableView + +def _select_input(objtype, row_id): + return format_html('', objtype, row_id) + + +def _edit_link(objtype, row_id, label): + return format_html('{}', _KSDBhref, objtype, row_id, label) + + class PersonView(BaseDatatableView): model = person objtype = "person" @@ -16,10 +26,10 @@ class PersonView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(PersonView, self).render_column(row, column) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) if search: @@ -43,10 +53,10 @@ class PublicationView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(PublicationView, self).render_column(row, column) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) if search: @@ -70,10 +80,10 @@ class ProgramView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(ProgramView, self).render_column(row, column) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) @@ -96,10 +106,10 @@ class InstitutionView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(InstitutionView, self).render_column(row, column) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) if search: @@ -121,7 +131,7 @@ class CommitteeView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(CommitteeView, self).render_column(row, column) if column == 'members': @@ -138,7 +148,7 @@ def render_column(self, row, column): if proid: programs.append(proid) obj = ",".join(programs) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) @@ -162,7 +172,7 @@ class GroupView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(GroupView, self).render_column(row, column) if column == 'members': @@ -179,7 +189,7 @@ def render_column(self, row, column): if proid: programs.append(proid) obj = ",".join(programs) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) @@ -203,7 +213,7 @@ class FundedSiteView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(FundedSiteView, self).render_column(row, column) if column == 'pis': @@ -220,7 +230,7 @@ def render_column(self, row, column): if proid: programs.append(proid) obj = ",".join(programs) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) if search: @@ -244,10 +254,10 @@ class ProtocolView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(ProtocolView, self).render_column(row, column) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) if search: @@ -268,10 +278,10 @@ class DiseaseView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(DiseaseView, self).render_column(row, column) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) if search: @@ -291,10 +301,10 @@ class AssayTypeView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(AssayTypeView, self).render_column(row, column) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) if search: @@ -314,10 +324,10 @@ class OrganView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(OrganView, self).render_column(row, column) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) if search: @@ -337,10 +347,10 @@ class DisciplineView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(DisciplineView, self).render_column(row, column) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) if search: @@ -360,10 +370,10 @@ class SpecimenTypeView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(SpecimenTypeView, self).render_column(row, column) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) if search: @@ -383,10 +393,10 @@ class SpeciesView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(SpeciesView, self).render_column(row, column) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) if search: @@ -406,10 +416,10 @@ class DegreeView(BaseDatatableView): def render_column(self, row, column): # We want to render user as a custom column if column == 'Select': - return ''.format(self.objtype, row.id) + return _select_input(self.objtype, row.id) else: obj = super(DegreeView, self).render_column(row, column) - return '{3}'.format(_KSDBhref, self.objtype, row.id, obj) + return _edit_link(self.objtype, row.id, obj) def filter_queryset(self, qs): search = self.request.GET.get(u'search[value]', None) diff --git a/ksdb/management/commands/ingestorgans.py b/ksdb/management/commands/ingestorgans.py index 0498977..beb1e46 100755 --- a/ksdb/management/commands/ingestorgans.py +++ b/ksdb/management/commands/ingestorgans.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand +from django.db import transaction from rdflib.term import URIRef from ksdb.models import organ @@ -20,18 +21,19 @@ def handle(self, *args, **options): statements = parse_statements(graph) imported_count = 0 - for subject in statements: - title = text_value(statements[subject].get(self._body_system_title, [""])[0]) - if not title: - continue - if organ.objects.filter(name=title).exists(): - continue - - organ.objects.create( - id=next_sequence_value("organ_seq"), - name=title, - description="", - ) - imported_count += 1 + with transaction.atomic(): + for subject in statements: + title = text_value(statements[subject].get(self._body_system_title, [""])[0]) + if not title: + continue + if organ.objects.filter(name=title).exists(): + continue + + organ.objects.create( + id=next_sequence_value("organ_seq"), + name=title, + description="", + ) + imported_count += 1 logger.info("Successfully imported %s organs from cancerdataexpo rdf.", imported_count) diff --git a/ksdb/management/commands/ingestpersons.py b/ksdb/management/commands/ingestpersons.py index 5ccf06f..bf5bfbe 100755 --- a/ksdb/management/commands/ingestpersons.py +++ b/ksdb/management/commands/ingestpersons.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand +from django.db import transaction from rdflib.term import URIRef from ksdb.models import degree, person, person_degree_link @@ -29,63 +30,64 @@ def handle(self, *args, **options): imported_count = 0 updated_count = 0 - for subject in statements: - firstname = text_value(statements[subject].get(self._firstname_uri, [""])[0]) - lastname = text_value(statements[subject].get(self._lastname_uri, [""])[0]) - email = text_value(statements[subject].get(self._mbox_uri, [""])[0]).replace("mailto:", "") - telephone = text_value(statements[subject].get(self._phone_uri, [""])[0]).replace("tel:", "") + with transaction.atomic(): + for subject in statements: + firstname = text_value(statements[subject].get(self._firstname_uri, [""])[0]) + lastname = text_value(statements[subject].get(self._lastname_uri, [""])[0]) + email = text_value(statements[subject].get(self._mbox_uri, [""])[0]).replace("mailto:", "") + telephone = text_value(statements[subject].get(self._phone_uri, [""])[0]).replace("tel:", "") - if not (firstname and lastname and email): - continue - - person_obj = person.objects.filter(firstname=firstname, lastname=lastname).first() - if person_obj is None and email: - person_obj = person.objects.filter(email=email).first() + if not (firstname and lastname and email): + continue - if person_obj is None: - person_obj = person.objects.create( - id=next_sequence_value("person_seq"), - firstname=firstname, - lastname=lastname, - email=email, - telephone=telephone or None, - extension=None, - description="", - degrees=None, - dcp=False, - dcb=False, - ) - imported_count += 1 - else: - person_obj.firstname = firstname - person_obj.lastname = lastname - person_obj.email = email - if telephone: - person_obj.telephone = telephone - if person_obj.description is None: - person_obj.description = "" - if person_obj.dcp is None: - person_obj.dcp = False - if person_obj.dcb is None: - person_obj.dcb = False - person_obj.save() - updated_count += 1 + person_obj = person.objects.filter(firstname=firstname, lastname=lastname).first() + if person_obj is None and email: + person_obj = person.objects.filter(email=email).first() - for degree_uri in self._degree_uris: - degree_title = text_value(statements[subject].get(degree_uri, [""])[0]) - if not degree_title: - continue - degree_obj = degree.objects.filter(title=degree_title).first() - if degree_obj is None: - degree_obj = degree.objects.create( - id=next_sequence_value("degree_seq"), - title=degree_title, + if person_obj is None: + person_obj = person.objects.create( + id=next_sequence_value("person_seq"), + firstname=firstname, + lastname=lastname, + email=email, + telephone=telephone or None, + extension=None, description="", + degrees=None, + dcp=False, + dcb=False, + ) + imported_count += 1 + else: + person_obj.firstname = firstname + person_obj.lastname = lastname + person_obj.email = email + if telephone: + person_obj.telephone = telephone + if person_obj.description is None: + person_obj.description = "" + if person_obj.dcp is None: + person_obj.dcp = False + if person_obj.dcb is None: + person_obj.dcb = False + person_obj.save() + updated_count += 1 + + for degree_uri in self._degree_uris: + degree_title = text_value(statements[subject].get(degree_uri, [""])[0]) + if not degree_title: + continue + degree_obj = degree.objects.filter(title=degree_title).first() + if degree_obj is None: + degree_obj = degree.objects.create( + id=next_sequence_value("degree_seq"), + title=degree_title, + description="", + ) + person_degree_link.objects.get_or_create( + personid=person_obj.id, + degreeid=degree_obj.id, ) - person_degree_link.objects.get_or_create( - personid=person_obj.id, - degreeid=degree_obj.id, - ) logger.info( "Successfully imported %s persons and updated %s persons from cancerdataexpo rdf into KSDB.", diff --git a/ksdb/management/commands/ingestprotocols.py b/ksdb/management/commands/ingestprotocols.py index 081c729..75d5ba1 100755 --- a/ksdb/management/commands/ingestprotocols.py +++ b/ksdb/management/commands/ingestprotocols.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand +from django.db import transaction from rdflib.term import URIRef from ksdb.models import protocol, protocol_program_link @@ -31,73 +32,74 @@ class Command(BaseCommand): _title_uri = URIRef("http://purl.org/dc/terms/title") def handle(self, *args, **options): - ensure_default_programs() graph = fetch_rdf_graph(self.protocolurl) statements = parse_statements(graph) imported_count = 0 updated_count = 0 - for subject in statements: - if self._protocol_type_uri not in statements[subject].get(self._rdf_type, []): - continue + with transaction.atomic(): + ensure_default_programs() + for subject in statements: + if self._protocol_type_uri not in statements[subject].get(self._rdf_type, []): + continue - title = text_value(statements[subject].get(self._title_uri, [""])[0]) - if not title: - continue + title = text_value(statements[subject].get(self._title_uri, [""])[0]) + if not title: + continue - edrn_id = text_value(subject) - aims = text_value(statements[subject].get(self._aims_uri, [""])[0]) - abstract = text_value(statements[subject].get(self._abstract_uri, [""])[0]) - shortname = text_value(statements[subject].get(self._ab_name_uri, [""])[0]) - author = text_value(statements[subject].get(self._author_uri, [""])[0]) - end_date = parse_protocol_datetime(statements[subject].get(self._finish_date_uri, [""])[0]) - start_date = parse_protocol_datetime(statements[subject].get(self._start_date_uri, [""])[0]) + edrn_id = text_value(subject) + aims = text_value(statements[subject].get(self._aims_uri, [""])[0]) + abstract = text_value(statements[subject].get(self._abstract_uri, [""])[0]) + shortname = text_value(statements[subject].get(self._ab_name_uri, [""])[0]) + author = text_value(statements[subject].get(self._author_uri, [""])[0]) + end_date = parse_protocol_datetime(statements[subject].get(self._finish_date_uri, [""])[0]) + start_date = parse_protocol_datetime(statements[subject].get(self._start_date_uri, [""])[0]) - defaults = { - "title": title, - "edrn_id": edrn_id, - "shortname": shortname or None, - "organs": "", - "fundedsites": "", - "institutions": "", - "programs": "2", - "pis": author or "", - "cis": "", - "start_date": start_date, - "end_date": end_date, - "site_contact": None, - "site_contact_email": None, - "irb_approval": None, - "irb_approval_num": None, - "irb_contact": None, - "irb_contact_email": None, - "hum_sub_train": None, - "abstract": abstract or "", - "aims": aims or "", - "publications": "", - "data_custodians": "", - } + defaults = { + "title": title, + "edrn_id": edrn_id, + "shortname": shortname or None, + "organs": "", + "fundedsites": "", + "institutions": "", + "programs": "2", + "pis": author or "", + "cis": "", + "start_date": start_date, + "end_date": end_date, + "site_contact": None, + "site_contact_email": None, + "irb_approval": None, + "irb_approval_num": None, + "irb_contact": None, + "irb_contact_email": None, + "hum_sub_train": None, + "abstract": abstract or "", + "aims": aims or "", + "publications": "", + "data_custodians": "", + } - protocol_obj = protocol.objects.filter(edrn_id=edrn_id).first() - if protocol_obj is None: - protocol_obj = protocol.objects.filter(title=title).first() + protocol_obj = protocol.objects.filter(edrn_id=edrn_id).first() + if protocol_obj is None: + protocol_obj = protocol.objects.filter(title=title).first() - if protocol_obj is None: - protocol_obj = protocol.objects.create( - id=next_sequence_value("protocol_seq"), - **defaults - ) - imported_count += 1 - else: - for field, value in defaults.items(): - setattr(protocol_obj, field, value) - protocol_obj.save() - updated_count += 1 + if protocol_obj is None: + protocol_obj = protocol.objects.create( + id=next_sequence_value("protocol_seq"), + **defaults + ) + imported_count += 1 + else: + for field, value in defaults.items(): + setattr(protocol_obj, field, value) + protocol_obj.save() + updated_count += 1 - protocol_program_link.objects.get_or_create( - protocolid=protocol_obj.id, - programid=2, - ) + protocol_program_link.objects.get_or_create( + protocolid=protocol_obj.id, + programid=2, + ) logger.info( "Successfully imported %s protocols and updated %s protocols from cancerdataexpo rdf.", diff --git a/ksdb/management/commands/ingestpublications.py b/ksdb/management/commands/ingestpublications.py index 729f692..ca895f7 100755 --- a/ksdb/management/commands/ingestpublications.py +++ b/ksdb/management/commands/ingestpublications.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand +from django.db import transaction from rdflib.term import URIRef from ksdb.models import publication, publication_program_link @@ -26,54 +27,55 @@ class Command(BaseCommand): _title_uri = URIRef("http://purl.org/dc/terms/title") def handle(self, *args, **options): - ensure_default_programs() graph = fetch_rdf_graph(self.publicationurl) statements = parse_statements(graph) imported_count = 0 updated_count = 0 - for subject in statements: - title = text_value(statements[subject].get(self._title_uri, [""])[0]) - pmid = text_value(statements[subject].get(self._pmid_uri, [""])[0]) - year = text_value(statements[subject].get(self._year_uri, [""])[0]) - journal = text_value(statements[subject].get(self._journal_uri, [""])[0]) - authors = [text_value(author) for author in statements[subject].get(self._author_uri, [])] + with transaction.atomic(): + ensure_default_programs() + for subject in statements: + title = text_value(statements[subject].get(self._title_uri, [""])[0]) + pmid = text_value(statements[subject].get(self._pmid_uri, [""])[0]) + year = text_value(statements[subject].get(self._year_uri, [""])[0]) + journal = text_value(statements[subject].get(self._journal_uri, [""])[0]) + authors = [text_value(author) for author in statements[subject].get(self._author_uri, [])] - if not (title and pmid and year and journal and authors): - continue + if not (title and pmid and year and journal and authors): + continue - try: - pubyear = int(year) - except (TypeError, ValueError): - continue + try: + pubyear = int(year) + except (TypeError, ValueError): + continue - defaults = { - "title": title, - "authors": ", ".join([author for author in authors if author]), - "journal": journal, - "pubyear": pubyear, - "programs": "2", - } - existing = publication.objects.filter(pubmedid=pmid).first() + defaults = { + "title": title, + "authors": ", ".join([author for author in authors if author]), + "journal": journal, + "pubyear": pubyear, + "programs": "2", + } + existing = publication.objects.filter(pubmedid=pmid).first() - if existing is None: - publication_obj = publication.objects.create( - id=next_sequence_value("publication_seq"), - pubmedid=pmid, - **defaults - ) - imported_count += 1 - else: - for field, value in defaults.items(): - setattr(existing, field, value) - existing.save(update_fields=list(defaults.keys())) - publication_obj = existing - updated_count += 1 + if existing is None: + publication_obj = publication.objects.create( + id=next_sequence_value("publication_seq"), + pubmedid=pmid, + **defaults + ) + imported_count += 1 + else: + for field, value in defaults.items(): + setattr(existing, field, value) + existing.save(update_fields=list(defaults.keys())) + publication_obj = existing + updated_count += 1 - publication_program_link.objects.get_or_create( - publicationid=publication_obj.id, - programid=2, - ) + publication_program_link.objects.get_or_create( + publicationid=publication_obj.id, + programid=2, + ) logger.info( "Successfully imported %s publications and updated %s publications from cancerdataexpo rdf.",