From 89a3974438cf0065be602a8d4ccf60059fab2e02 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 28 Nov 2019 20:07:05 +0100 Subject: [PATCH 01/33] Initial sampling implementation --- pom.xml | 2 +- .../webapi/cohortsample/CohortSample.java | 121 +++++++++ .../cohortsample/CohortSamplingService.java | 241 ++++++++++++++++++ .../webapi/cohortsample/SampleElement.java | 67 +++++ .../cohortsample/dto/CohortSampleDTO.java | 35 +++ .../cohortsample/dto/SampleParametersDTO.java | 40 +++ .../webapi/service/CohortSampleService.java | 102 ++++++++ .../util/PreparedStatementRenderer.java | 10 +- .../sql/deleteElementBySampleId.sql | 3 + .../cohortsample/sql/deleteSampleById.sql | 3 + .../sql/findElementsByCohortSampleId.sql | 4 + .../sql/findSampleByCohortDefinitionId.sql | 4 + .../cohortsample/sql/findSampleById.sql | 4 + .../cohortsample/sql/generateSample.sql | 11 + .../cohortsample/sql/insertSample.sql | 3 + .../cohortsample/sql/insertSampleElement.sql | 3 + 16 files changed, 647 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java create mode 100644 src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java create mode 100644 src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java create mode 100644 src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java create mode 100644 src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java create mode 100644 src/main/java/org/ohdsi/webapi/service/CohortSampleService.java create mode 100644 src/main/resources/resources/cohortsample/sql/deleteElementBySampleId.sql create mode 100644 src/main/resources/resources/cohortsample/sql/deleteSampleById.sql create mode 100644 src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql create mode 100644 src/main/resources/resources/cohortsample/sql/findSampleByCohortDefinitionId.sql create mode 100644 src/main/resources/resources/cohortsample/sql/findSampleById.sql create mode 100644 src/main/resources/resources/cohortsample/sql/generateSample.sql create mode 100644 src/main/resources/resources/cohortsample/sql/insertSample.sql create mode 100644 src/main/resources/resources/cohortsample/sql/insertSampleElement.sql diff --git a/pom.xml b/pom.xml index b1df20557c..79b061d678 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ authDataSource - 8080 + 9000 diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java new file mode 100644 index 0000000000..2c9a648834 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java @@ -0,0 +1,121 @@ +package org.ohdsi.webapi.cohortsample; + +import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; +import org.ohdsi.webapi.model.CommonEntity; + +import javax.persistence.CascadeType; +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.FetchType; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.ManyToOne; +import javax.persistence.OneToMany; +import javax.persistence.OrderColumn; +import javax.persistence.Table; +import java.util.List; + +@Entity(name = "CohortSample") +@Table(name = "cohort_sample") +public class CohortSample extends CommonEntity { + @Id + @GenericGenerator( + name = "cohort_sample_generator", + strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", + parameters = { + @Parameter(name = "sequence_name", value = "cohort_sample_sequence"), + @Parameter(name = "increment_size", value = "1") + } + ) + @GeneratedValue(generator = "cohort_sample_generator") + private Integer id; + + @JoinColumn(name = "cohort_definition_id") + private int cohortDefinitionId; + + @Column(name = "age_min") + private Integer ageMin; + + @Column(name = "age_max") + private Integer ageMax; + + @Column(name = "gender_concept_id") + private Integer genderConceptId; + + @OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL, orphanRemoval = true) + @OrderColumn(name = "rank") + private List elements; + + private int size; + + @Override + public Integer getId() { + return this.id; + } + + public void setId(Integer id) { + this.id = id; + } + + public int getCohortDefinitionId() { + return cohortDefinitionId; + } + + public void setCohortDefinitionId(int cohortDefinitionId) { + this.cohortDefinitionId = cohortDefinitionId; + } + + public List getElements() { + return elements; + } + + public void setElements(List elements) { + this.elements = elements; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public Integer getAgeMin() { + return ageMin; + } + + public void setAgeMin(Integer ageMin) { + if (ageMin == 0) { + this.ageMin = null; + } else { + this.ageMin = ageMin; + } + } + + public Integer getAgeMax() { + return ageMax; + } + + public void setAgeMax(Integer ageMax) { + if (ageMax == 0) { + this.ageMax = null; + } else { + this.ageMax = ageMax; + } + } + + public Integer getGenderConceptId() { + return genderConceptId; + } + + public void setGenderConceptId(Integer genderConceptId) { + if (genderConceptId == 0) { + this.genderConceptId = null; + } else { + this.genderConceptId = genderConceptId; + } + } +} diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java new file mode 100644 index 0000000000..fa69f93b82 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -0,0 +1,241 @@ +package org.ohdsi.webapi.cohortsample; + +import org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO; +import org.ohdsi.webapi.service.AbstractDaoService; +import org.ohdsi.webapi.source.Source; +import org.ohdsi.webapi.source.SourceDaimon; +import org.ohdsi.webapi.util.CancelableJdbcTemplate; +import org.ohdsi.webapi.util.PreparedStatementRenderer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.StatementCallback; +import org.springframework.stereotype.Component; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; + +import javax.ws.rs.NotFoundException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Component +public class CohortSamplingService extends AbstractDaoService { + private final static CohortSampleRowMapper sampleRowMapper = new CohortSampleRowMapper(); + private final static CohortSampleElementRowMapper elementRowMapper = new CohortSampleElementRowMapper(); + private final TransactionTemplate transactionTemplate; + + @Autowired + public CohortSamplingService( + TransactionTemplate transactionTemplate) { + this.transactionTemplate = transactionTemplate; + } + + public List findSamples(Source source, int cohortDefinitionId) { + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + return jdbcTemplate.query( + new PreparedStatementRenderer(source, "/resources/cohortsample/findSampleByCohortDefinitionId.sql", + "results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results), + "cohortDefinitionId", cohortDefinitionId + ).getSql(), sampleRowMapper); + } + + public List findSampleElements(Source source, int cohortSampleId) { + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + return jdbcTemplate.query( + new PreparedStatementRenderer(source, "/resources/cohortsample/findElementsByCohortSampleId.sql", + "results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results), + "cohortSampleId", cohortSampleId + ).getSql(), elementRowMapper); + } + + public CohortSample findSample(Source source, int cohortSampleId) { + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + List samples = jdbcTemplate.query( + new PreparedStatementRenderer(source, "/resources/cohortsample/findSampleById.sql", + "results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results), + "cohortSampleId", cohortSampleId + ).getSql(), sampleRowMapper); + if (samples == null || samples.isEmpty()) { + throw new NotFoundException("Cohort sample with ID " + cohortSampleId + " does not exist."); + } + return samples.get(0); + } + + public CohortSample createSample(Source source, int cohortDefinitionId, SampleParametersDTO sampleParameters) { + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + + CohortSample sample = new CohortSample(); + sample.setCohortDefinitionId(cohortDefinitionId); + sample.setAgeMin(sampleParameters.getAgeMin()); + sample.setAgeMax(sampleParameters.getAgeMax()); + sample.setGenderConceptId(sampleParameters.getGenderConceptId()); + + final List elements = sampleElements(sample, jdbcTemplate, source); + + transactionTemplate.execute((TransactionCallback) transactionStatus -> { + int sampleId = insertSample(jdbcTemplate, source, sampleParameters, cohortDefinitionId); + sample.setId(sampleId); + + insertSampledElements(source, jdbcTemplate, sampleId, elements); + + sample.setElements(elements); + + return null; + }); + + return sample; + } + + private int insertSample(JdbcTemplate jdbcTemplate, Source source, SampleParametersDTO sampleParameters, int cohortDefinitionId) { + String[] parameters = new String[] { "results_schema" }; + String[] parameterValues = new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results) }; + String[] sqlParameters = new String[] { "cohortDefinitionId", "size", "ageMin", "ageMax", "genderConceptId", "createdById", "createdDate" }; + Object[] sqlParameterValues = new Object[] { cohortDefinitionId, sampleParameters.getSize(), sampleParameters.getAgeMin(), sampleParameters.getAgeMax(), sampleParameters.getGenderConceptId(), getCurrentUser(), new Date()}; + + return jdbcTemplate + .execute(new InsertStatementCallback( + new PreparedStatementRenderer(source, "/resources/cohortsample/sql/insertSample.sql", parameters, parameterValues, sqlParameters, sqlParameterValues).getSql(), + "id")); + } + + private void insertSampledElements(Source source, JdbcTemplate jdbcTemplate, int sampleId, List elements) { + String[] parameters = new String[] { "results_schema" }; + String[] parameterValues = new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results) }; + String[] sqlParameters = new String[] { "cohortSampleId", "rank", "personId", "age", "genderConceptId" }; + Object[] sqlValues = new Object[5]; + sqlValues[0] = sampleId; + + String[] statements = new String[elements.size()]; + int i = 0; + for (SampleElement element : elements) { + sqlValues[1] = element.getRank(); + sqlValues[2] = element.getPersonId(); + sqlValues[3] = element.getAge(); + sqlValues[4] = element.getGenderConceptId(); + statements[i] = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/insertSample.sql", parameters, parameterValues, sqlParameters, sqlValues).getSql(); + i++; + } + + jdbcTemplate.batchUpdate(statements); + } + + private List sampleElements(CohortSample sample, JdbcTemplate jdbcTemplate, Source source) { + StringBuilder expressionBuilder = new StringBuilder(); + Map parameters = new LinkedHashMap<>(); + Map sqlVariables = new LinkedHashMap<>(); + + parameters.put("results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results)); + parameters.put("CDM_schema", source.getTableQualifier(SourceDaimon.DaimonType.CDM)); + sqlVariables.put("cohort_definition_id", sample.getCohortDefinitionId()); + if (sample.getAgeMin() != null) { + expressionBuilder.append("AND cast(year(c.cohort_start_date) - p.year_of_birth as int) >= @age_min"); + sqlVariables.put("age_min", sample.getAgeMin()); + } + if (sample.getAgeMax() != null) { + expressionBuilder.append("AND cast(year(c.cohort_start_date) - p.year_of_birth as int) < @gender_concept_id").append(sample.getAgeMax()); + sqlVariables.put("age_max", sample.getAgeMax()); + } + if (sample.getGenderConceptId() != null) { + expressionBuilder.append("AND p.gender_concept_id = ").append(sample.getGenderConceptId()); + sqlVariables.put("gender_concept_id", sample.getGenderConceptId()); + } + + parameters.put("expression", expressionBuilder.toString()); + + PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/generateSample.sql", + parameters.keySet().toArray(new String[0]), + parameters.values().toArray(new String[0]), + sqlVariables.keySet().toArray(new String[0]), + sqlVariables.values().toArray(new Object[0])); + + CancelableJdbcTemplate template = new CancelableJdbcTemplate(); + template.setMaxRows(sample.getSize()); + + return jdbcTemplate.query(renderer.getSql(), (rs, rowNum) -> { + SampleElement element = new SampleElement(); + element.setRank(rowNum); + element.setAge(rs.getInt("age")); + element.setGenderConceptId(rs.getInt("gender_concept_id")); + element.setPersonId(rs.getLong("person_id")); + return element; + }); + } + + public void deleteSample(Source source, int sampleId) { + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + String resultsSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); + String[] statements = new String[] { + new PreparedStatementRenderer( + source, + "/resources/cohortsample/sql/deleteSampleById.sql", + "results_schema", + resultsSchema, + "cohortSampleId", + sampleId) + .getSql(), + new PreparedStatementRenderer( + source, + "/resources/cohortsample/sql/deleteElementsBySampleId.sql", + "results_schema", + resultsSchema, + "cohortSampleId", + sampleId) + .getSql() + }; + + transactionTemplate.execute((TransactionCallback) transactionStatus -> { + jdbcTemplate.batchUpdate(statements); + return null; + }); + } + + private static class CohortSampleRowMapper implements RowMapper { + @Override + public CohortSample mapRow(ResultSet rs, int rowNum) throws SQLException { + CohortSample sample = new CohortSample(); + sample.setId(rs.getInt("id")); + sample.setCohortDefinitionId(rs.getInt("cohort_definition_id")); + sample.setSize(rs.getInt("size")); + sample.setGenderConceptId(rs.getInt("gender_concept_id")); + sample.setAgeMax(rs.getInt("age_max")); + sample.setAgeMin(rs.getInt("age_min")); + return sample; + } + } + + + private static class CohortSampleElementRowMapper implements RowMapper { + @Override + public SampleElement mapRow(ResultSet rs, int rowNum) throws SQLException { + SampleElement sample = new SampleElement(); + sample.setRank(rs.getInt("rank")); + sample.setSampleId(rs.getInt("cohort_sample_id")); + sample.setPersonId(rs.getInt("person_id")); + sample.setGenderConceptId(rs.getInt("gender_concept_id")); + sample.setAge(rs.getInt("age")); + return sample; + } + } + + private static class InsertStatementCallback implements StatementCallback { + private final String sql; + private final String idColumn; + + InsertStatementCallback(String sql, String idColumn) { + this.sql = sql; + this.idColumn = idColumn; + } + + @Override + public Integer doInStatement(Statement stmt) throws SQLException { + stmt.executeUpdate(sql, new String[] { idColumn }); + + return stmt.getGeneratedKeys().getInt(idColumn); + } + } +} diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java b/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java new file mode 100644 index 0000000000..2ef302f684 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java @@ -0,0 +1,67 @@ +package org.ohdsi.webapi.cohortsample; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.ws.rs.Consumes; + +@Entity(name = "CohortSampleElement") +public class SampleElement { + @Id + @Column(name = "id") + private Integer id; + + @Column(name = "cohort_sample_id") + private int sampleId; + + private int rank; + + @Column(name = "person_id") + private long personId; + + @Column(name = "gender_concept_id") + private long genderConceptId; + + @Column(name = "age") + private int age; + + public int getSampleId() { + return sampleId; + } + + public void setSampleId(int sampleId) { + this.sampleId = sampleId; + } + + public int getRank() { + return rank; + } + + public void setRank(int rank) { + this.rank = rank; + } + + public long getPersonId() { + return personId; + } + + public void setPersonId(long personId) { + this.personId = personId; + } + + public long getGenderConceptId() { + return genderConceptId; + } + + public void setGenderConceptId(long genderConceptId) { + this.genderConceptId = genderConceptId; + } + + public int getAge() { + return age; + } + + public void setAge(int age) { + this.age = age; + } +} diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java new file mode 100644 index 0000000000..22921b4d90 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java @@ -0,0 +1,35 @@ +package org.ohdsi.webapi.cohortsample.dto; + +import org.ohdsi.webapi.cohortsample.SampleElement; + +import java.util.List; + +public class CohortSampleDTO { + private int id; + private int size; + private List elements; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public List getElements() { + return elements; + } + + public void setElements(List elements) { + this.elements = elements; + } +} diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java new file mode 100644 index 0000000000..e6eda1e608 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java @@ -0,0 +1,40 @@ +package org.ohdsi.webapi.cohortsample.dto; + +public class SampleParametersDTO { + private int size; + private Integer ageMin; + private Integer ageMax; + private Integer genderConceptId; + + public Integer getAgeMin() { + return ageMin; + } + + public void setAgeMin(Integer ageMin) { + this.ageMin = ageMin; + } + + public Integer getAgeMax() { + return ageMax; + } + + public void setAgeMax(Integer ageMax) { + this.ageMax = ageMax; + } + + public Integer getGenderConceptId() { + return genderConceptId; + } + + public void setGenderConceptId(Integer genderConceptId) { + this.genderConceptId = genderConceptId; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } +} diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java new file mode 100644 index 0000000000..ee773eb554 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -0,0 +1,102 @@ +package org.ohdsi.webapi.service; + +import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; +import org.ohdsi.webapi.cohortsample.CohortSample; +import org.ohdsi.webapi.cohortsample.CohortSamplingService; +import org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO; +import org.ohdsi.webapi.source.Source; +import org.ohdsi.webapi.source.SourceRepository; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.NotFoundException; +import javax.ws.rs.POST; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.util.List; + +@Path("/cohortsample/{cohortDefinitionId}/{sourceKey}") +@Component +@Produces(MediaType.APPLICATION_JSON) +public class CohortSampleService { + private final CohortDefinitionRepository cohortDefinitionRepository; + private final CohortSamplingService samplingService; + private final SourceRepository sourceRepository; + + @Autowired + public CohortSampleService( + CohortSamplingService samplingService, + SourceRepository sourceRepository, + CohortDefinitionRepository cohortDefinitionRepository + ) { + this.samplingService = samplingService; + this.sourceRepository = sourceRepository; + this.cohortDefinitionRepository = cohortDefinitionRepository; + } + + @Path("/") + @GET + public List listCohortSamples( + @QueryParam("cohortDefinitionId") int cohortDefinitionId, + @QueryParam("sourceKey") String sourceKey + ) { + Source source = getSource(sourceKey); + return this.samplingService.findSamples(source, cohortDefinitionId); + } + + @Path("/{sampleId}") + @GET + public CohortSample getCohortSample( + @QueryParam("sourceKey") String sourceKey, + @QueryParam("sampleId") int sampleId + ) { + Source source = getSource(sourceKey); + CohortSample sample = samplingService.findSample(source, sampleId); + if (sample == null) { + throw new NotFoundException("Cohort sample with ID " + sampleId + " not found"); + } + sample.setElements(this.samplingService.findSampleElements(source, sampleId)); + return sample; + } + + + @Path("/") + @POST + @Consumes(MediaType.APPLICATION_JSON) + public CohortSample createCohortSample( + @QueryParam("sourceKey") String sourceKey, + @QueryParam("cohortDefinitionId") int cohortDefinitionId, + SampleParametersDTO sampleParameters + ) { + Source source = getSource(sourceKey); + if (cohortDefinitionRepository.findOne(cohortDefinitionId) == null) { + throw new NotFoundException("Cohort definition " + cohortDefinitionId + " does not exist."); + } + return samplingService.createSample(source, cohortDefinitionId, sampleParameters); + } + + @Path("/{sampleId}") + @DELETE + public Response deleteCohortSample( + @QueryParam("sourceKey") String sourceKey, + @QueryParam("sampleId") int sampleId + ) { + Source source = getSource(sourceKey); + samplingService.deleteSample(source, sampleId); + return Response.status(Response.Status.NO_CONTENT).build(); + } + + private Source getSource(String sourceKey) { + Source source = sourceRepository.findBySourceKey(sourceKey); + if (source == null) { + throw new NotFoundException("Source " + sourceKey + " does not exist"); + } + return source; + } +} diff --git a/src/main/java/org/ohdsi/webapi/util/PreparedStatementRenderer.java b/src/main/java/org/ohdsi/webapi/util/PreparedStatementRenderer.java index b69d021bbd..2bc30fdcaf 100644 --- a/src/main/java/org/ohdsi/webapi/util/PreparedStatementRenderer.java +++ b/src/main/java/org/ohdsi/webapi/util/PreparedStatementRenderer.java @@ -293,18 +293,18 @@ public PreparedStatementSetter getSetter() { private Object convertPrimitiveArraysToWrapperArrays(Object value) { - if (value == null) return value; + if (value == null) return null; if (!value.getClass().isArray()) return value; if (value instanceof boolean[]) return ArrayUtils.toObject((boolean[]) value); if (value instanceof byte[]) return ArrayUtils.toObject((byte[]) value); if (value instanceof char[]) return ArrayUtils.toObject((char[]) value); - if (value instanceof char[]) return ArrayUtils.toObject((double[]) value); - if (value instanceof char[]) return ArrayUtils.toObject((float[]) value); - if (value instanceof char[]) return ArrayUtils.toObject((int[]) value); + if (value instanceof double[]) return ArrayUtils.toObject((double[]) value); + if (value instanceof float[]) return ArrayUtils.toObject((float[]) value); + if (value instanceof int[]) return ArrayUtils.toObject((int[]) value); if (value instanceof long[]) return ArrayUtils.toObject((long[]) value); - if (value instanceof char[]) return ArrayUtils.toObject((short[]) value); + if (value instanceof short[]) return ArrayUtils.toObject((short[]) value); return value; } diff --git a/src/main/resources/resources/cohortsample/sql/deleteElementBySampleId.sql b/src/main/resources/resources/cohortsample/sql/deleteElementBySampleId.sql new file mode 100644 index 0000000000..88d8d69574 --- /dev/null +++ b/src/main/resources/resources/cohortsample/sql/deleteElementBySampleId.sql @@ -0,0 +1,3 @@ +DELETE FROM @results_schema.cohort_sample_element se +WHERE se.cohort_sample_id = @cohortSampleId +; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/deleteSampleById.sql b/src/main/resources/resources/cohortsample/sql/deleteSampleById.sql new file mode 100644 index 0000000000..de463499eb --- /dev/null +++ b/src/main/resources/resources/cohortsample/sql/deleteSampleById.sql @@ -0,0 +1,3 @@ +DELETE FROM @results_schema.cohort_sample s +WHERE s.id = @cohortSampleId +; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql new file mode 100644 index 0000000000..14153cd806 --- /dev/null +++ b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql @@ -0,0 +1,4 @@ +SELECT * +FROM @results_schema.cohort_sample_element s +WHERE s.cohort_sample_id = @cohortSampleId +; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/findSampleByCohortDefinitionId.sql b/src/main/resources/resources/cohortsample/sql/findSampleByCohortDefinitionId.sql new file mode 100644 index 0000000000..4bfdffce6d --- /dev/null +++ b/src/main/resources/resources/cohortsample/sql/findSampleByCohortDefinitionId.sql @@ -0,0 +1,4 @@ +SELECT * +FROM @results_schema.cohort_sample s +WHERE s.cohort_definition_id = @cohortDefinitionId +; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/findSampleById.sql b/src/main/resources/resources/cohortsample/sql/findSampleById.sql new file mode 100644 index 0000000000..a93433e91b --- /dev/null +++ b/src/main/resources/resources/cohortsample/sql/findSampleById.sql @@ -0,0 +1,4 @@ +SELECT * +FROM @results_schema.cohort_sample s +WHERE s.id = @cohortSampleId +; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/generateSample.sql b/src/main/resources/resources/cohortsample/sql/generateSample.sql new file mode 100644 index 0000000000..472aeb08e8 --- /dev/null +++ b/src/main/resources/resources/cohortsample/sql/generateSample.sql @@ -0,0 +1,11 @@ +select p.person_id, + c.cohort_definition_id as cohort_definition_id, + p.gender_concept_id as gender_concept_id, + cast(year(c.cohort_start_date) - p.year_of_birth as int) as age +from @results_schema.cohort c +join @CDM_schema.person p +on p.person_id = c.subject_id +where c.cohort_definition_id = @cohort_definition_id +@expression +order by RAND() +; diff --git a/src/main/resources/resources/cohortsample/sql/insertSample.sql b/src/main/resources/resources/cohortsample/sql/insertSample.sql new file mode 100644 index 0000000000..7f3c689a1c --- /dev/null +++ b/src/main/resources/resources/cohortsample/sql/insertSample.sql @@ -0,0 +1,3 @@ +INSERT INTO @results_schema.cohort_sample (cohort_definition_id, size, age_min, age_max, gender_concept_id, created_by_id, created_date) +VALUES (@cohortDefinitionId, @size, @ageMin, @ageMax, @genderConceptId, @createdById, @createdDate) +; diff --git a/src/main/resources/resources/cohortsample/sql/insertSampleElement.sql b/src/main/resources/resources/cohortsample/sql/insertSampleElement.sql new file mode 100644 index 0000000000..d3f2bf80ad --- /dev/null +++ b/src/main/resources/resources/cohortsample/sql/insertSampleElement.sql @@ -0,0 +1,3 @@ +INSERT INTO @results_schema.cohort_sample_element (cohort_sample_id, rank, person_id, age, gender_concept_id) +VALUES (@cohortSampleId, @rank, @personId, @age, @genderConceptId) +; From 3eaf3d627d7b51c419e5e0fb26bbd84b9beac35f Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 5 Dec 2019 11:12:05 +0100 Subject: [PATCH 02/33] Add name to cohort sample --- src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java index 2c9a648834..fb4517744e 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java @@ -32,6 +32,8 @@ public class CohortSample extends CommonEntity { @GeneratedValue(generator = "cohort_sample_generator") private Integer id; + private String name; + @JoinColumn(name = "cohort_definition_id") private int cohortDefinitionId; From dbe6328042724c1cc2bc34232bc9adb3a476df81 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 19 Dec 2019 16:40:10 +0100 Subject: [PATCH 03/33] Revert server port to 8080 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 79b061d678..b1df20557c 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ authDataSource - 9000 + 8080 From c0cfc658b0ffeb210313357574b02f6e8103fc2e Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 8 Jan 2020 12:38:46 +0100 Subject: [PATCH 04/33] Fix queryParam -> pathParam --- .../webapi/service/CohortSampleService.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index ee773eb554..56e5914418 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -15,6 +15,7 @@ import javax.ws.rs.NotFoundException; import javax.ws.rs.POST; import javax.ws.rs.Path; +import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; @@ -43,8 +44,8 @@ public CohortSampleService( @Path("/") @GET public List listCohortSamples( - @QueryParam("cohortDefinitionId") int cohortDefinitionId, - @QueryParam("sourceKey") String sourceKey + @PathParam("cohortDefinitionId") int cohortDefinitionId, + @PathParam("sourceKey") String sourceKey ) { Source source = getSource(sourceKey); return this.samplingService.findSamples(source, cohortDefinitionId); @@ -53,8 +54,8 @@ public List listCohortSamples( @Path("/{sampleId}") @GET public CohortSample getCohortSample( - @QueryParam("sourceKey") String sourceKey, - @QueryParam("sampleId") int sampleId + @PathParam("sourceKey") String sourceKey, + @PathParam("sampleId") int sampleId ) { Source source = getSource(sourceKey); CohortSample sample = samplingService.findSample(source, sampleId); @@ -70,8 +71,8 @@ public CohortSample getCohortSample( @POST @Consumes(MediaType.APPLICATION_JSON) public CohortSample createCohortSample( - @QueryParam("sourceKey") String sourceKey, - @QueryParam("cohortDefinitionId") int cohortDefinitionId, + @PathParam("sourceKey") String sourceKey, + @PathParam("cohortDefinitionId") int cohortDefinitionId, SampleParametersDTO sampleParameters ) { Source source = getSource(sourceKey); @@ -84,8 +85,8 @@ public CohortSample createCohortSample( @Path("/{sampleId}") @DELETE public Response deleteCohortSample( - @QueryParam("sourceKey") String sourceKey, - @QueryParam("sampleId") int sampleId + @PathParam("sourceKey") String sourceKey, + @PathParam("sampleId") int sampleId ) { Source source = getSource(sourceKey); samplingService.deleteSample(source, sampleId); From ea742fff068183e11290ae405ba56623c2cf75f7 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 8 Jan 2020 15:26:54 +0100 Subject: [PATCH 05/33] Initial update to fix SQL commands for sample generation --- .../cohortanalysis/CohortAnalysisTasklet.java | 2 +- .../webapi/cohortsample/CohortSample.java | 36 +------------------ .../cohortsample/CohortSamplingService.java | 34 +++++++----------- .../webapi/cohortsample/SampleElement.java | 14 -------- .../resources/ddl/results/cohort_sample.sql | 11 ++++++ .../ddl/results/cohort_sample_element.sql | 16 +++++++++ .../sql/deleteElementBySampleId.sql | 3 -- .../cohortsample/sql/deleteSampleById.sql | 1 + 8 files changed, 42 insertions(+), 75 deletions(-) create mode 100644 src/main/resources/ddl/results/cohort_sample.sql create mode 100644 src/main/resources/ddl/results/cohort_sample_element.sql delete mode 100644 src/main/resources/resources/cohortsample/sql/deleteElementBySampleId.sql diff --git a/src/main/java/org/ohdsi/webapi/cohortanalysis/CohortAnalysisTasklet.java b/src/main/java/org/ohdsi/webapi/cohortanalysis/CohortAnalysisTasklet.java index 86e739df49..6839425afe 100644 --- a/src/main/java/org/ohdsi/webapi/cohortanalysis/CohortAnalysisTasklet.java +++ b/src/main/java/org/ohdsi/webapi/cohortanalysis/CohortAnalysisTasklet.java @@ -6,7 +6,7 @@ import java.util.stream.Stream; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.text.FormattableUtils; import static org.ohdsi.webapi.util.SecurityUtils.whitelist; import org.ohdsi.sql.SqlSplit; diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java index fb4517744e..be1d1a2a64 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java @@ -1,58 +1,24 @@ package org.ohdsi.webapi.cohortsample; -import org.hibernate.annotations.GenericGenerator; -import org.hibernate.annotations.Parameter; -import org.ohdsi.webapi.model.CommonEntity; - -import javax.persistence.CascadeType; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.FetchType; -import javax.persistence.GeneratedValue; -import javax.persistence.Id; -import javax.persistence.JoinColumn; -import javax.persistence.ManyToOne; -import javax.persistence.OneToMany; -import javax.persistence.OrderColumn; -import javax.persistence.Table; import java.util.List; -@Entity(name = "CohortSample") -@Table(name = "cohort_sample") -public class CohortSample extends CommonEntity { - @Id - @GenericGenerator( - name = "cohort_sample_generator", - strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", - parameters = { - @Parameter(name = "sequence_name", value = "cohort_sample_sequence"), - @Parameter(name = "increment_size", value = "1") - } - ) - @GeneratedValue(generator = "cohort_sample_generator") +public class CohortSample { private Integer id; private String name; - @JoinColumn(name = "cohort_definition_id") private int cohortDefinitionId; - @Column(name = "age_min") private Integer ageMin; - @Column(name = "age_max") private Integer ageMax; - @Column(name = "gender_concept_id") private Integer genderConceptId; - @OneToMany(fetch = FetchType.LAZY, cascade = CascadeType.ALL, orphanRemoval = true) - @OrderColumn(name = "rank") private List elements; private int size; - @Override public Integer getId() { return this.id; } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index fa69f93b82..31b77797c5 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -38,7 +38,7 @@ public CohortSamplingService( public List findSamples(Source source, int cohortDefinitionId) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); return jdbcTemplate.query( - new PreparedStatementRenderer(source, "/resources/cohortsample/findSampleByCohortDefinitionId.sql", + new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findSampleByCohortDefinitionId.sql", "results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results), "cohortDefinitionId", cohortDefinitionId ).getSql(), sampleRowMapper); @@ -47,7 +47,7 @@ public List findSamples(Source source, int cohortDefinitionId) { public List findSampleElements(Source source, int cohortSampleId) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); return jdbcTemplate.query( - new PreparedStatementRenderer(source, "/resources/cohortsample/findElementsByCohortSampleId.sql", + new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleId.sql", "results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results), "cohortSampleId", cohortSampleId ).getSql(), elementRowMapper); @@ -56,7 +56,7 @@ public List findSampleElements(Source source, int cohortSampleId) public CohortSample findSample(Source source, int cohortSampleId) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); List samples = jdbcTemplate.query( - new PreparedStatementRenderer(source, "/resources/cohortsample/findSampleById.sql", + new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findSampleById.sql", "results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results), "cohortSampleId", cohortSampleId ).getSql(), sampleRowMapper); @@ -169,27 +169,17 @@ private List sampleElements(CohortSample sample, JdbcTemplate jdb public void deleteSample(Source source, int sampleId) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); String resultsSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); - String[] statements = new String[] { - new PreparedStatementRenderer( - source, - "/resources/cohortsample/sql/deleteSampleById.sql", - "results_schema", - resultsSchema, - "cohortSampleId", - sampleId) - .getSql(), - new PreparedStatementRenderer( - source, - "/resources/cohortsample/sql/deleteElementsBySampleId.sql", - "results_schema", - resultsSchema, - "cohortSampleId", - sampleId) - .getSql() - }; + String sql = new PreparedStatementRenderer( + source, + "/resources/cohortsample/sql/deleteSampleById.sql", + "results_schema", + resultsSchema, + "cohortSampleId", + sampleId) + .getSql(); transactionTemplate.execute((TransactionCallback) transactionStatus -> { - jdbcTemplate.batchUpdate(statements); + jdbcTemplate.update(sql); return null; }); } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java b/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java index 2ef302f684..f38e5dba0f 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java @@ -1,28 +1,14 @@ package org.ohdsi.webapi.cohortsample; -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.ws.rs.Consumes; - -@Entity(name = "CohortSampleElement") public class SampleElement { - @Id - @Column(name = "id") - private Integer id; - - @Column(name = "cohort_sample_id") private int sampleId; private int rank; - @Column(name = "person_id") private long personId; - @Column(name = "gender_concept_id") private long genderConceptId; - @Column(name = "age") private int age; public int getSampleId() { diff --git a/src/main/resources/ddl/results/cohort_sample.sql b/src/main/resources/ddl/results/cohort_sample.sql new file mode 100644 index 0000000000..232f4a8a16 --- /dev/null +++ b/src/main/resources/ddl/results/cohort_sample.sql @@ -0,0 +1,11 @@ +IF OBJECT_ID('@results_schema.cohort_sample', 'U') IS NULL +CREATE TABLE @results_schema.cohort_sample( + id int NOT NULL, + name varchar(255) NOT NULL, + cohort_definition_id int NOT NULL, + age_min int NOT NULL, + age_max int NOT NULL, + gender_concept_id int NOT NULL, + size int NOT NULL, + CONSTRAINT pk_cohort_sample_id PRIMARY KEY (id) +); diff --git a/src/main/resources/ddl/results/cohort_sample_element.sql b/src/main/resources/ddl/results/cohort_sample_element.sql new file mode 100644 index 0000000000..de5997d931 --- /dev/null +++ b/src/main/resources/ddl/results/cohort_sample_element.sql @@ -0,0 +1,16 @@ +IF OBJECT_ID('@results_schema.cohort_sample', 'U') IS NULL +CREATE TABLE @results_schema.cohort_sample_element( + id int NOT NULL, + cohort_sample_id int NOT NULL, + rank int NOT NULL, + person_id bigint NOT NULL, + age int NOT NULL, + gender_concept_id int NOT NULL, + CONSTRAINT fk_cohort_sample_element_id + FOREIGN KEY (cohort_sample_id) + REFERENCES @results_schema.cohort_sample (id) + ON DELETE CASCADE, +); + +CREATE CLUSTERED INDEX idx_cohort_sample_element_rank ON @results_schema.cohort_sample_element (cohort_sample_id, rank) + diff --git a/src/main/resources/resources/cohortsample/sql/deleteElementBySampleId.sql b/src/main/resources/resources/cohortsample/sql/deleteElementBySampleId.sql deleted file mode 100644 index 88d8d69574..0000000000 --- a/src/main/resources/resources/cohortsample/sql/deleteElementBySampleId.sql +++ /dev/null @@ -1,3 +0,0 @@ -DELETE FROM @results_schema.cohort_sample_element se -WHERE se.cohort_sample_id = @cohortSampleId -; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/deleteSampleById.sql b/src/main/resources/resources/cohortsample/sql/deleteSampleById.sql index de463499eb..f87f90a220 100644 --- a/src/main/resources/resources/cohortsample/sql/deleteSampleById.sql +++ b/src/main/resources/resources/cohortsample/sql/deleteSampleById.sql @@ -1,3 +1,4 @@ DELETE FROM @results_schema.cohort_sample s WHERE s.id = @cohortSampleId +CASCADE ; \ No newline at end of file From ca86aade74238ee9db15ba0431b425a6603feb9c Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Jan 2020 14:13:56 +0100 Subject: [PATCH 06/33] Attempt to move cohort_sample table to ohdsi --- .../cohortanalysis/CohortAnalysisTasklet.java | 2 +- .../webapi/cohortsample/CohortSample.java | 32 ++- ...e.java => CohortSampleElementService.java} | 33 ++- .../cohortsample/CohortSampleRepository.java | 204 ++++++++++++++++++ .../webapi/service/CohortSampleService.java | 7 +- .../org/ohdsi/webapi/service/DDLService.java | 3 + ...0.20200109100200__cohort_sample_tables.sql | 21 ++ ...0.20200109100200__cohort_sample_tables.sql | 21 ++ ...0.20200109100200__cohort_sample_tables.sql | 21 ++ .../resources/ddl/results/cohort_sample.sql | 6 +- .../ddl/results/cohort_sample_element.sql | 12 +- 11 files changed, 333 insertions(+), 29 deletions(-) rename src/main/java/org/ohdsi/webapi/cohortsample/{CohortSamplingService.java => CohortSampleElementService.java} (92%) create mode 100644 src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java create mode 100644 src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql create mode 100644 src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql create mode 100644 src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql diff --git a/src/main/java/org/ohdsi/webapi/cohortanalysis/CohortAnalysisTasklet.java b/src/main/java/org/ohdsi/webapi/cohortanalysis/CohortAnalysisTasklet.java index 6839425afe..86e739df49 100644 --- a/src/main/java/org/ohdsi/webapi/cohortanalysis/CohortAnalysisTasklet.java +++ b/src/main/java/org/ohdsi/webapi/cohortanalysis/CohortAnalysisTasklet.java @@ -6,7 +6,7 @@ import java.util.stream.Stream; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.text.FormattableUtils; +import org.apache.commons.lang3.StringUtils; import static org.ohdsi.webapi.util.SecurityUtils.whitelist; import org.ohdsi.sql.SqlSplit; diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java index be1d1a2a64..7d546e956d 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java @@ -1,22 +1,48 @@ package org.ohdsi.webapi.cohortsample; +import org.hibernate.annotations.GenericGenerator; +import org.hibernate.annotations.Parameter; +import org.ohdsi.webapi.model.CommonEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.JoinColumn; +import javax.persistence.Table; import java.util.List; -public class CohortSample { +@Entity(name = "CohortSample") +@Table(name = "cohort_sample") +public class CohortSample extends CommonEntity { + @Id + @GenericGenerator( + name = "cohort_sample_generator", + strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", + parameters = { + @Parameter(name = "sequence_name", value = "cohort_sample_sequence"), + @Parameter(name = "increment_size", value = "1") + } + ) + @GeneratedValue(generator = "cohort_sample_generator") private Integer id; + @Column private String name; + @JoinColumn(name = "cohort_definition_id") private int cohortDefinitionId; + @Column(name = "age_min") private Integer ageMin; + @Column(name = "age_max") private Integer ageMax; + @Column(name = "gender_concept_id") private Integer genderConceptId; - private List elements; - + @Column private int size; public Integer getId() { diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleElementService.java similarity index 92% rename from src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java rename to src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleElementService.java index 31b77797c5..f3b6fa18a4 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleElementService.java @@ -24,13 +24,13 @@ import java.util.Map; @Component -public class CohortSamplingService extends AbstractDaoService { +public class CohortSampleElementService extends AbstractDaoService { private final static CohortSampleRowMapper sampleRowMapper = new CohortSampleRowMapper(); private final static CohortSampleElementRowMapper elementRowMapper = new CohortSampleElementRowMapper(); private final TransactionTemplate transactionTemplate; @Autowired - public CohortSamplingService( + public CohortSampleElementService( TransactionTemplate transactionTemplate) { this.transactionTemplate = transactionTemplate; } @@ -169,19 +169,30 @@ private List sampleElements(CohortSample sample, JdbcTemplate jdb public void deleteSample(Source source, int sampleId) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); String resultsSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); - String sql = new PreparedStatementRenderer( - source, - "/resources/cohortsample/sql/deleteSampleById.sql", - "results_schema", - resultsSchema, - "cohortSampleId", - sampleId) - .getSql(); - + String[] statements = new String[] { + new PreparedStatementRenderer( + source, + "/resources/cohortsample/sql/deleteSampleById.sql", + "results_schema", + resultsSchema, + "cohortSampleId", + sampleId) + .getSql(), + new PreparedStatementRenderer( + source, + "/resources/cohortsample/sql/deleteElementsBySampleId.sql", + "results_schema", + resultsSchema, + "cohortSampleId", + sampleId) + .getSql() + }; transactionTemplate.execute((TransactionCallback) transactionStatus -> { jdbcTemplate.update(sql); + jdbcTemplate.batchUpdate(statements); return null; }); + } private static class CohortSampleRowMapper implements RowMapper { diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java new file mode 100644 index 0000000000..367743e0bd --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java @@ -0,0 +1,204 @@ +package org.ohdsi.webapi.cohortsample; + +import org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO; +import org.ohdsi.webapi.service.AbstractDaoService; +import org.ohdsi.webapi.source.Source; +import org.ohdsi.webapi.source.SourceDaimon; +import org.ohdsi.webapi.util.CancelableJdbcTemplate; +import org.ohdsi.webapi.util.PreparedStatementRenderer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.repository.CrudRepository; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.jdbc.core.StatementCallback; +import org.springframework.stereotype.Component; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.transaction.support.TransactionTemplate; + +import javax.ws.rs.NotFoundException; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +@Component +public interface CohortSampleRepository extends CrudRepository { + List findByCohortDefinitionIdAndSourceId(int cohortDefinitionId, int sourceId); + + public CohortSample createSample(Source source, int cohortDefinitionId, SampleParametersDTO sampleParameters) { + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + + CohortSample sample = new CohortSample(); + sample.setCohortDefinitionId(cohortDefinitionId); + sample.setAgeMin(sampleParameters.getAgeMin()); + sample.setAgeMax(sampleParameters.getAgeMax()); + sample.setGenderConceptId(sampleParameters.getGenderConceptId()); + + final List elements = sampleElements(sample, jdbcTemplate, source); + + transactionTemplate.execute((TransactionCallback) transactionStatus -> { + int sampleId = insertSample(jdbcTemplate, source, sampleParameters, cohortDefinitionId); + sample.setId(sampleId); + + insertSampledElements(source, jdbcTemplate, sampleId, elements); + + sample.setElements(elements); + + return null; + }); + + return sample; + } + + private int insertSample(JdbcTemplate jdbcTemplate, Source source, SampleParametersDTO sampleParameters, int cohortDefinitionId) { + String[] parameters = new String[] { "results_schema" }; + String[] parameterValues = new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results) }; + String[] sqlParameters = new String[] { "cohortDefinitionId", "size", "ageMin", "ageMax", "genderConceptId", "createdById", "createdDate" }; + Object[] sqlParameterValues = new Object[] { cohortDefinitionId, sampleParameters.getSize(), sampleParameters.getAgeMin(), sampleParameters.getAgeMax(), sampleParameters.getGenderConceptId(), getCurrentUser(), new Date()}; + + return jdbcTemplate + .execute(new InsertStatementCallback( + new PreparedStatementRenderer(source, "/resources/cohortsample/sql/insertSample.sql", parameters, parameterValues, sqlParameters, sqlParameterValues).getSql(), + "id")); + } + + private void insertSampledElements(Source source, JdbcTemplate jdbcTemplate, int sampleId, List elements) { + String[] parameters = new String[] { "results_schema" }; + String[] parameterValues = new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results) }; + String[] sqlParameters = new String[] { "cohortSampleId", "rank", "personId", "age", "genderConceptId" }; + Object[] sqlValues = new Object[5]; + sqlValues[0] = sampleId; + + String[] statements = new String[elements.size()]; + int i = 0; + for (SampleElement element : elements) { + sqlValues[1] = element.getRank(); + sqlValues[2] = element.getPersonId(); + sqlValues[3] = element.getAge(); + sqlValues[4] = element.getGenderConceptId(); + statements[i] = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/insertSample.sql", parameters, parameterValues, sqlParameters, sqlValues).getSql(); + i++; + } + + jdbcTemplate.batchUpdate(statements); + } + + private List sampleElements(CohortSample sample, JdbcTemplate jdbcTemplate, Source source) { + StringBuilder expressionBuilder = new StringBuilder(); + Map parameters = new LinkedHashMap<>(); + Map sqlVariables = new LinkedHashMap<>(); + + parameters.put("results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results)); + parameters.put("CDM_schema", source.getTableQualifier(SourceDaimon.DaimonType.CDM)); + sqlVariables.put("cohort_definition_id", sample.getCohortDefinitionId()); + if (sample.getAgeMin() != null) { + expressionBuilder.append("AND cast(year(c.cohort_start_date) - p.year_of_birth as int) >= @age_min"); + sqlVariables.put("age_min", sample.getAgeMin()); + } + if (sample.getAgeMax() != null) { + expressionBuilder.append("AND cast(year(c.cohort_start_date) - p.year_of_birth as int) < @gender_concept_id").append(sample.getAgeMax()); + sqlVariables.put("age_max", sample.getAgeMax()); + } + if (sample.getGenderConceptId() != null) { + expressionBuilder.append("AND p.gender_concept_id = ").append(sample.getGenderConceptId()); + sqlVariables.put("gender_concept_id", sample.getGenderConceptId()); + } + + parameters.put("expression", expressionBuilder.toString()); + + PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/generateSample.sql", + parameters.keySet().toArray(new String[0]), + parameters.values().toArray(new String[0]), + sqlVariables.keySet().toArray(new String[0]), + sqlVariables.values().toArray(new Object[0])); + + CancelableJdbcTemplate template = new CancelableJdbcTemplate(); + template.setMaxRows(sample.getSize()); + + return jdbcTemplate.query(renderer.getSql(), (rs, rowNum) -> { + SampleElement element = new SampleElement(); + element.setRank(rowNum); + element.setAge(rs.getInt("age")); + element.setGenderConceptId(rs.getInt("gender_concept_id")); + element.setPersonId(rs.getLong("person_id")); + return element; + }); + } + + public void deleteSample(Source source, int sampleId) { + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + String resultsSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); + String[] statements = new String[] { + new PreparedStatementRenderer( + source, + "/resources/cohortsample/sql/deleteSampleById.sql", + "results_schema", + resultsSchema, + "cohortSampleId", + sampleId) + .getSql(), + new PreparedStatementRenderer( + source, + "/resources/cohortsample/sql/deleteElementsBySampleId.sql", + "results_schema", + resultsSchema, + "cohortSampleId", + sampleId) + .getSql() + }; + transactionTemplate.execute((TransactionCallback) transactionStatus -> { + jdbcTemplate.update(sql); + jdbcTemplate.batchUpdate(statements); + return null; + }); + + } + + private static class CohortSampleRowMapper implements RowMapper { + @Override + public CohortSample mapRow(ResultSet rs, int rowNum) throws SQLException { + CohortSample sample = new CohortSample(); + sample.setId(rs.getInt("id")); + sample.setCohortDefinitionId(rs.getInt("cohort_definition_id")); + sample.setSize(rs.getInt("size")); + sample.setGenderConceptId(rs.getInt("gender_concept_id")); + sample.setAgeMax(rs.getInt("age_max")); + sample.setAgeMin(rs.getInt("age_min")); + return sample; + } + } + + + private static class CohortSampleElementRowMapper implements RowMapper { + @Override + public SampleElement mapRow(ResultSet rs, int rowNum) throws SQLException { + SampleElement sample = new SampleElement(); + sample.setRank(rs.getInt("rank")); + sample.setSampleId(rs.getInt("cohort_sample_id")); + sample.setPersonId(rs.getInt("person_id")); + sample.setGenderConceptId(rs.getInt("gender_concept_id")); + sample.setAge(rs.getInt("age")); + return sample; + } + } + + private static class InsertStatementCallback implements StatementCallback { + private final String sql; + private final String idColumn; + + InsertStatementCallback(String sql, String idColumn) { + this.sql = sql; + this.idColumn = idColumn; + } + + @Override + public Integer doInStatement(Statement stmt) throws SQLException { + stmt.executeUpdate(sql, new String[] { idColumn }); + + return stmt.getGeneratedKeys().getInt(idColumn); + } + } +} diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index 56e5914418..680eee5293 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -2,7 +2,7 @@ import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; import org.ohdsi.webapi.cohortsample.CohortSample; -import org.ohdsi.webapi.cohortsample.CohortSamplingService; +import org.ohdsi.webapi.cohortsample.CohortSampleElementService; import org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO; import org.ohdsi.webapi.source.Source; import org.ohdsi.webapi.source.SourceRepository; @@ -17,7 +17,6 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.util.List; @@ -27,12 +26,12 @@ @Produces(MediaType.APPLICATION_JSON) public class CohortSampleService { private final CohortDefinitionRepository cohortDefinitionRepository; - private final CohortSamplingService samplingService; + private final CohortSampleElementService samplingService; private final SourceRepository sourceRepository; @Autowired public CohortSampleService( - CohortSamplingService samplingService, + CohortSampleElementService samplingService, SourceRepository sourceRepository, CohortDefinitionRepository cohortDefinitionRepository ) { diff --git a/src/main/java/org/ohdsi/webapi/service/DDLService.java b/src/main/java/org/ohdsi/webapi/service/DDLService.java index e0a5df29be..da58755aa3 100644 --- a/src/main/java/org/ohdsi/webapi/service/DDLService.java +++ b/src/main/java/org/ohdsi/webapi/service/DDLService.java @@ -75,6 +75,9 @@ public class DDLService { "/ddl/results/heracles_results.sql", "/ddl/results/heracles_results_dist.sql", "/ddl/results/heracles_periods.sql", + // cohort sampling + "/ddl/results/cohort_sample.sql", + "/ddl/results/cohort_sample_element.sql", // incidence rates "/ddl/results/ir_analysis_dist.sql", "/ddl/results/ir_analysis_result.sql", diff --git a/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql new file mode 100644 index 0000000000..d22156c904 --- /dev/null +++ b/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -0,0 +1,21 @@ +CREATE SEQUENCE ${ohdsiSchema}.cohort_sample_sequence; + +CREATE TABLE ${ohdsiSchema}.cohort_sample( + id NUMBER(10) PRIMARY KEY, + name VARCHAR(255) NOT NULL, + cohort_definition_id NUMBER(10) NOT NULL, + source_id NUMBER(10) NOT NULL, + size NUMBER(9) NOT NULL, + age_min NUMBER(4), + age_max NUMBER(4), + gender_concept_id NUMBER(19), + created_by_id INTEGER, + created_date TIMESTAMP WITH TIME ZONE DEFAULT sysdate NOT NULL, + modified_by_id INTEGER, + modified_date TIMESTAMP WITH TIME ZONE, + CONSTRAINT fk_cohort_sample_definition_id FOREIGN KEY (cohort_definition_id) + REFERENCES ${ohdsiSchema}.cohort_definition (cohort_definition_id) ON DELETE CASCADE, + CONSTRAINT fk_cohort_sample_source_id FOREIGN KEY (source_id) + REFERENCES ${ohdsiSchema}.source (source_id) ON DELETE CASCADE, + INDEX idx_cohort_sample_source (cohort_definition_id, source_id) +); diff --git a/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql new file mode 100644 index 0000000000..dd193939ff --- /dev/null +++ b/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -0,0 +1,21 @@ +CREATE SEQUENCE ${ohdsiSchema}.cohort_sample_sequence; + +CREATE TABLE ${ohdsiSchema}.cohort_sample( + id INTEGER PRIMARY KEY NOT NULL, + name VARCHAR(255) NOT NULL, + cohort_definition_id INTEGER NOT NULL, + source_id INTEGER NOT NULL, + size INTEGER NOT NULL, + age_min SMALLINT NULL, + age_max SMALLINT NULL, + gender_concept_id INTEGER NULL, + created_by_id INTEGER, + created_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT (now()), + modified_by_id INTEGER, + modified_date TIMESTAMP WITH TIME ZONE, + CONSTRAINT fk_cohort_sample_definition_id FOREIGN KEY (cohort_definition_id) + REFERENCES ${ohdsiSchema}.cohort_definition (cohort_definition_id) ON DELETE CASCADE, + CONSTRAINT fk_cohort_sample_source_id FOREIGN KEY (source_id) + REFERENCES ${ohdsiSchema}.source (source_id) ON DELETE CASCADE, + INDEX idx_cohort_sample_source (cohort_definition_id, source_id) +); diff --git a/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql new file mode 100644 index 0000000000..399513d37f --- /dev/null +++ b/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -0,0 +1,21 @@ +CREATE SEQUENCE ${ohdsiSchema}.cohort_sample_sequence; + +CREATE TABLE ${ohdsiSchema}.cohort_sample( + id NUMBER(19) PRIMARY KEY CLUSTERED NOT NULL, + name VARCHAR(255) NOT NULL, + cohort_definition_id INTEGER NOT NULL, + source_id INTEGER NOT NULL, + size NUMBER(9) NOT NULL, + age_min NUMBER(4) NULL, + age_max NUMBER(4) NULL, + gender_concept_id NUMBER(19) NULL, + created_by_id INTEGER, + created_date DATETIMEOFFSET, + modified_by_id INTEGER, + modified_date DATETIMEOFFSET, + CONSTRAINT fk_cohort_sample_definition_id FOREIGN KEY (cohort_definition_id) + REFERENCES ${ohdsiSchema}.cohort_definition (cohort_definition_id) ON DELETE CASCADE, + CONSTRAINT fk_cohort_sample_source_id FOREIGN KEY (source_id) + REFERENCES ${ohdsiSchema}.source (source_id) ON DELETE CASCADE, + INDEX idx_cohort_sample_source (cohort_definition_id, source_id) +); diff --git a/src/main/resources/ddl/results/cohort_sample.sql b/src/main/resources/ddl/results/cohort_sample.sql index 232f4a8a16..2be7cb2650 100644 --- a/src/main/resources/ddl/results/cohort_sample.sql +++ b/src/main/resources/ddl/results/cohort_sample.sql @@ -3,9 +3,9 @@ CREATE TABLE @results_schema.cohort_sample( id int NOT NULL, name varchar(255) NOT NULL, cohort_definition_id int NOT NULL, - age_min int NOT NULL, - age_max int NOT NULL, - gender_concept_id int NOT NULL, + age_min int, + age_max int, + gender_concept_id int, size int NOT NULL, CONSTRAINT pk_cohort_sample_id PRIMARY KEY (id) ); diff --git a/src/main/resources/ddl/results/cohort_sample_element.sql b/src/main/resources/ddl/results/cohort_sample_element.sql index de5997d931..4b2e639923 100644 --- a/src/main/resources/ddl/results/cohort_sample_element.sql +++ b/src/main/resources/ddl/results/cohort_sample_element.sql @@ -1,16 +1,14 @@ IF OBJECT_ID('@results_schema.cohort_sample', 'U') IS NULL CREATE TABLE @results_schema.cohort_sample_element( - id int NOT NULL, cohort_sample_id int NOT NULL, rank int NOT NULL, person_id bigint NOT NULL, - age int NOT NULL, - gender_concept_id int NOT NULL, + age int, + gender_concept_id int, CONSTRAINT fk_cohort_sample_element_id - FOREIGN KEY (cohort_sample_id) + FOREIGN KEY (cohort_sample_i d) REFERENCES @results_schema.cohort_sample (id) - ON DELETE CASCADE, + ON DELETE CASCADE ); -CREATE CLUSTERED INDEX idx_cohort_sample_element_rank ON @results_schema.cohort_sample_element (cohort_sample_id, rank) - +CREATE UNIQUE INDEX idx_cohort_sample_element_rank ON @results_schema.cohort_sample_element (cohort_sample_id, rank); From d70cc25f3a7fc0ab5eb43c55826b5a05a1a3ce50 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Jan 2020 16:36:52 +0100 Subject: [PATCH 07/33] Separate cohort_sample from cohort_sample_element --- .../webapi/cohortsample/CohortSample.java | 18 +- .../cohortsample/CohortSampleRepository.java | 193 ------------------ ...ervice.java => CohortSamplingService.java} | 109 ++-------- .../cohortsample/dto/SampleParametersDTO.java | 33 +++ .../webapi/service/CohortSampleService.java | 20 +- .../cohortsample/sql/deleteSampleById.sql | 4 - .../sql/deleteSampleElementsById.sql | 3 + .../sql/findElementsByCohortSampleId.sql | 1 + .../sql/findSampleByCohortDefinitionId.sql | 4 - .../cohortsample/sql/findSampleById.sql | 4 - .../cohortsample/sql/insertSample.sql | 3 - 11 files changed, 82 insertions(+), 310 deletions(-) rename src/main/java/org/ohdsi/webapi/cohortsample/{CohortSampleElementService.java => CohortSamplingService.java} (59%) delete mode 100644 src/main/resources/resources/cohortsample/sql/deleteSampleById.sql create mode 100644 src/main/resources/resources/cohortsample/sql/deleteSampleElementsById.sql delete mode 100644 src/main/resources/resources/cohortsample/sql/findSampleByCohortDefinitionId.sql delete mode 100644 src/main/resources/resources/cohortsample/sql/findSampleById.sql delete mode 100644 src/main/resources/resources/cohortsample/sql/insertSample.sql diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java index 7d546e956d..234c5be154 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java @@ -8,8 +8,8 @@ import javax.persistence.Entity; import javax.persistence.GeneratedValue; import javax.persistence.Id; -import javax.persistence.JoinColumn; import javax.persistence.Table; +import javax.persistence.Transient; import java.util.List; @Entity(name = "CohortSample") @@ -30,9 +30,12 @@ public class CohortSample extends CommonEntity { @Column private String name; - @JoinColumn(name = "cohort_definition_id") + @Column(name = "cohort_definition_id") private int cohortDefinitionId; + @Column(name = "source_id") + private int sourceId; + @Column(name = "age_min") private Integer ageMin; @@ -45,6 +48,9 @@ public class CohortSample extends CommonEntity { @Column private int size; + @Transient + private List elements; + public Integer getId() { return this.id; } @@ -112,4 +118,12 @@ public void setGenderConceptId(Integer genderConceptId) { this.genderConceptId = genderConceptId; } } + + public int getSourceId() { + return sourceId; + } + + public void setSourceId(Integer sourceId) { + this.sourceId = sourceId; + } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java index 367743e0bd..39e57e0aa4 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java @@ -1,204 +1,11 @@ package org.ohdsi.webapi.cohortsample; -import org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO; -import org.ohdsi.webapi.service.AbstractDaoService; -import org.ohdsi.webapi.source.Source; -import org.ohdsi.webapi.source.SourceDaimon; -import org.ohdsi.webapi.util.CancelableJdbcTemplate; -import org.ohdsi.webapi.util.PreparedStatementRenderer; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.repository.CrudRepository; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.StatementCallback; import org.springframework.stereotype.Component; -import org.springframework.transaction.support.TransactionCallback; -import org.springframework.transaction.support.TransactionTemplate; -import javax.ws.rs.NotFoundException; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Date; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; @Component public interface CohortSampleRepository extends CrudRepository { List findByCohortDefinitionIdAndSourceId(int cohortDefinitionId, int sourceId); - - public CohortSample createSample(Source source, int cohortDefinitionId, SampleParametersDTO sampleParameters) { - JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); - - CohortSample sample = new CohortSample(); - sample.setCohortDefinitionId(cohortDefinitionId); - sample.setAgeMin(sampleParameters.getAgeMin()); - sample.setAgeMax(sampleParameters.getAgeMax()); - sample.setGenderConceptId(sampleParameters.getGenderConceptId()); - - final List elements = sampleElements(sample, jdbcTemplate, source); - - transactionTemplate.execute((TransactionCallback) transactionStatus -> { - int sampleId = insertSample(jdbcTemplate, source, sampleParameters, cohortDefinitionId); - sample.setId(sampleId); - - insertSampledElements(source, jdbcTemplate, sampleId, elements); - - sample.setElements(elements); - - return null; - }); - - return sample; - } - - private int insertSample(JdbcTemplate jdbcTemplate, Source source, SampleParametersDTO sampleParameters, int cohortDefinitionId) { - String[] parameters = new String[] { "results_schema" }; - String[] parameterValues = new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results) }; - String[] sqlParameters = new String[] { "cohortDefinitionId", "size", "ageMin", "ageMax", "genderConceptId", "createdById", "createdDate" }; - Object[] sqlParameterValues = new Object[] { cohortDefinitionId, sampleParameters.getSize(), sampleParameters.getAgeMin(), sampleParameters.getAgeMax(), sampleParameters.getGenderConceptId(), getCurrentUser(), new Date()}; - - return jdbcTemplate - .execute(new InsertStatementCallback( - new PreparedStatementRenderer(source, "/resources/cohortsample/sql/insertSample.sql", parameters, parameterValues, sqlParameters, sqlParameterValues).getSql(), - "id")); - } - - private void insertSampledElements(Source source, JdbcTemplate jdbcTemplate, int sampleId, List elements) { - String[] parameters = new String[] { "results_schema" }; - String[] parameterValues = new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results) }; - String[] sqlParameters = new String[] { "cohortSampleId", "rank", "personId", "age", "genderConceptId" }; - Object[] sqlValues = new Object[5]; - sqlValues[0] = sampleId; - - String[] statements = new String[elements.size()]; - int i = 0; - for (SampleElement element : elements) { - sqlValues[1] = element.getRank(); - sqlValues[2] = element.getPersonId(); - sqlValues[3] = element.getAge(); - sqlValues[4] = element.getGenderConceptId(); - statements[i] = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/insertSample.sql", parameters, parameterValues, sqlParameters, sqlValues).getSql(); - i++; - } - - jdbcTemplate.batchUpdate(statements); - } - - private List sampleElements(CohortSample sample, JdbcTemplate jdbcTemplate, Source source) { - StringBuilder expressionBuilder = new StringBuilder(); - Map parameters = new LinkedHashMap<>(); - Map sqlVariables = new LinkedHashMap<>(); - - parameters.put("results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results)); - parameters.put("CDM_schema", source.getTableQualifier(SourceDaimon.DaimonType.CDM)); - sqlVariables.put("cohort_definition_id", sample.getCohortDefinitionId()); - if (sample.getAgeMin() != null) { - expressionBuilder.append("AND cast(year(c.cohort_start_date) - p.year_of_birth as int) >= @age_min"); - sqlVariables.put("age_min", sample.getAgeMin()); - } - if (sample.getAgeMax() != null) { - expressionBuilder.append("AND cast(year(c.cohort_start_date) - p.year_of_birth as int) < @gender_concept_id").append(sample.getAgeMax()); - sqlVariables.put("age_max", sample.getAgeMax()); - } - if (sample.getGenderConceptId() != null) { - expressionBuilder.append("AND p.gender_concept_id = ").append(sample.getGenderConceptId()); - sqlVariables.put("gender_concept_id", sample.getGenderConceptId()); - } - - parameters.put("expression", expressionBuilder.toString()); - - PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/generateSample.sql", - parameters.keySet().toArray(new String[0]), - parameters.values().toArray(new String[0]), - sqlVariables.keySet().toArray(new String[0]), - sqlVariables.values().toArray(new Object[0])); - - CancelableJdbcTemplate template = new CancelableJdbcTemplate(); - template.setMaxRows(sample.getSize()); - - return jdbcTemplate.query(renderer.getSql(), (rs, rowNum) -> { - SampleElement element = new SampleElement(); - element.setRank(rowNum); - element.setAge(rs.getInt("age")); - element.setGenderConceptId(rs.getInt("gender_concept_id")); - element.setPersonId(rs.getLong("person_id")); - return element; - }); - } - - public void deleteSample(Source source, int sampleId) { - JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); - String resultsSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); - String[] statements = new String[] { - new PreparedStatementRenderer( - source, - "/resources/cohortsample/sql/deleteSampleById.sql", - "results_schema", - resultsSchema, - "cohortSampleId", - sampleId) - .getSql(), - new PreparedStatementRenderer( - source, - "/resources/cohortsample/sql/deleteElementsBySampleId.sql", - "results_schema", - resultsSchema, - "cohortSampleId", - sampleId) - .getSql() - }; - transactionTemplate.execute((TransactionCallback) transactionStatus -> { - jdbcTemplate.update(sql); - jdbcTemplate.batchUpdate(statements); - return null; - }); - - } - - private static class CohortSampleRowMapper implements RowMapper { - @Override - public CohortSample mapRow(ResultSet rs, int rowNum) throws SQLException { - CohortSample sample = new CohortSample(); - sample.setId(rs.getInt("id")); - sample.setCohortDefinitionId(rs.getInt("cohort_definition_id")); - sample.setSize(rs.getInt("size")); - sample.setGenderConceptId(rs.getInt("gender_concept_id")); - sample.setAgeMax(rs.getInt("age_max")); - sample.setAgeMin(rs.getInt("age_min")); - return sample; - } - } - - - private static class CohortSampleElementRowMapper implements RowMapper { - @Override - public SampleElement mapRow(ResultSet rs, int rowNum) throws SQLException { - SampleElement sample = new SampleElement(); - sample.setRank(rs.getInt("rank")); - sample.setSampleId(rs.getInt("cohort_sample_id")); - sample.setPersonId(rs.getInt("person_id")); - sample.setGenderConceptId(rs.getInt("gender_concept_id")); - sample.setAge(rs.getInt("age")); - return sample; - } - } - - private static class InsertStatementCallback implements StatementCallback { - private final String sql; - private final String idColumn; - - InsertStatementCallback(String sql, String idColumn) { - this.sql = sql; - this.idColumn = idColumn; - } - - @Override - public Integer doInStatement(Statement stmt) throws SQLException { - stmt.executeUpdate(sql, new String[] { idColumn }); - - return stmt.getGeneratedKeys().getInt(idColumn); - } - } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleElementService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java similarity index 59% rename from src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleElementService.java rename to src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index f3b6fa18a4..544df7e05c 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleElementService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -9,39 +9,29 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; -import org.springframework.jdbc.core.StatementCallback; import org.springframework.stereotype.Component; import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; -import javax.ws.rs.NotFoundException; import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @Component -public class CohortSampleElementService extends AbstractDaoService { - private final static CohortSampleRowMapper sampleRowMapper = new CohortSampleRowMapper(); - private final static CohortSampleElementRowMapper elementRowMapper = new CohortSampleElementRowMapper(); +public class CohortSamplingService extends AbstractDaoService { private final TransactionTemplate transactionTemplate; + private final CohortSampleRepository sampleRepository; + private final CohortSampleElementRowMapper elementRowMapper = new CohortSampleElementRowMapper(); @Autowired - public CohortSampleElementService( - TransactionTemplate transactionTemplate) { + public CohortSamplingService( + TransactionTemplate transactionTemplate, + CohortSampleRepository sampleRepository) { this.transactionTemplate = transactionTemplate; - } - - public List findSamples(Source source, int cohortDefinitionId) { - JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); - return jdbcTemplate.query( - new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findSampleByCohortDefinitionId.sql", - "results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results), - "cohortDefinitionId", cohortDefinitionId - ).getSql(), sampleRowMapper); + this.sampleRepository = sampleRepository; } public List findSampleElements(Source source, int cohortSampleId) { @@ -53,35 +43,25 @@ public List findSampleElements(Source source, int cohortSampleId) ).getSql(), elementRowMapper); } - public CohortSample findSample(Source source, int cohortSampleId) { - JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); - List samples = jdbcTemplate.query( - new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findSampleById.sql", - "results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results), - "cohortSampleId", cohortSampleId - ).getSql(), sampleRowMapper); - if (samples == null || samples.isEmpty()) { - throw new NotFoundException("Cohort sample with ID " + cohortSampleId + " does not exist."); - } - return samples.get(0); - } - public CohortSample createSample(Source source, int cohortDefinitionId, SampleParametersDTO sampleParameters) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); CohortSample sample = new CohortSample(); sample.setCohortDefinitionId(cohortDefinitionId); + sample.setSourceId(source.getId()); + sample.setSize(sampleParameters.getSize()); sample.setAgeMin(sampleParameters.getAgeMin()); sample.setAgeMax(sampleParameters.getAgeMax()); sample.setGenderConceptId(sampleParameters.getGenderConceptId()); + sample.setCreatedBy(getCurrentUser()); + sample.setCreatedDate(new Date()); final List elements = sampleElements(sample, jdbcTemplate, source); transactionTemplate.execute((TransactionCallback) transactionStatus -> { - int sampleId = insertSample(jdbcTemplate, source, sampleParameters, cohortDefinitionId); - sample.setId(sampleId); + CohortSample updatedSample = sampleRepository.save(sample); - insertSampledElements(source, jdbcTemplate, sampleId, elements); + insertSampledElements(source, jdbcTemplate, updatedSample.getId(), elements); sample.setElements(elements); @@ -91,18 +71,6 @@ public CohortSample createSample(Source source, int cohortDefinitionId, SamplePa return sample; } - private int insertSample(JdbcTemplate jdbcTemplate, Source source, SampleParametersDTO sampleParameters, int cohortDefinitionId) { - String[] parameters = new String[] { "results_schema" }; - String[] parameterValues = new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results) }; - String[] sqlParameters = new String[] { "cohortDefinitionId", "size", "ageMin", "ageMax", "genderConceptId", "createdById", "createdDate" }; - Object[] sqlParameterValues = new Object[] { cohortDefinitionId, sampleParameters.getSize(), sampleParameters.getAgeMin(), sampleParameters.getAgeMax(), sampleParameters.getGenderConceptId(), getCurrentUser(), new Date()}; - - return jdbcTemplate - .execute(new InsertStatementCallback( - new PreparedStatementRenderer(source, "/resources/cohortsample/sql/insertSample.sql", parameters, parameterValues, sqlParameters, sqlParameterValues).getSql(), - "id")); - } - private void insertSampledElements(Source source, JdbcTemplate jdbcTemplate, int sampleId, List elements) { String[] parameters = new String[] { "results_schema" }; String[] parameterValues = new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results) }; @@ -169,47 +137,21 @@ private List sampleElements(CohortSample sample, JdbcTemplate jdb public void deleteSample(Source source, int sampleId) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); String resultsSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); - String[] statements = new String[] { - new PreparedStatementRenderer( - source, - "/resources/cohortsample/sql/deleteSampleById.sql", - "results_schema", - resultsSchema, - "cohortSampleId", - sampleId) - .getSql(), - new PreparedStatementRenderer( + String sql = new PreparedStatementRenderer( source, "/resources/cohortsample/sql/deleteElementsBySampleId.sql", "results_schema", resultsSchema, "cohortSampleId", - sampleId) - .getSql() - }; + sampleId).getSql(); + transactionTemplate.execute((TransactionCallback) transactionStatus -> { + sampleRepository.delete(sampleId); jdbcTemplate.update(sql); - jdbcTemplate.batchUpdate(statements); return null; }); - - } - - private static class CohortSampleRowMapper implements RowMapper { - @Override - public CohortSample mapRow(ResultSet rs, int rowNum) throws SQLException { - CohortSample sample = new CohortSample(); - sample.setId(rs.getInt("id")); - sample.setCohortDefinitionId(rs.getInt("cohort_definition_id")); - sample.setSize(rs.getInt("size")); - sample.setGenderConceptId(rs.getInt("gender_concept_id")); - sample.setAgeMax(rs.getInt("age_max")); - sample.setAgeMin(rs.getInt("age_min")); - return sample; - } } - private static class CohortSampleElementRowMapper implements RowMapper { @Override public SampleElement mapRow(ResultSet rs, int rowNum) throws SQLException { @@ -222,21 +164,4 @@ public SampleElement mapRow(ResultSet rs, int rowNum) throws SQLException { return sample; } } - - private static class InsertStatementCallback implements StatementCallback { - private final String sql; - private final String idColumn; - - InsertStatementCallback(String sql, String idColumn) { - this.sql = sql; - this.idColumn = idColumn; - } - - @Override - public Integer doInStatement(Statement stmt) throws SQLException { - stmt.executeUpdate(sql, new String[] { idColumn }); - - return stmt.getGeneratedKeys().getInt(idColumn); - } - } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java index e6eda1e608..9c1d9ee116 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java @@ -1,11 +1,44 @@ package org.ohdsi.webapi.cohortsample.dto; +import javax.ws.rs.BadRequestException; + public class SampleParametersDTO { + private static final int SIZE_MAX = 500; + private static final int AGE_MAX = 500; + private int size; private Integer ageMin; private Integer ageMax; private Integer genderConceptId; + public void validate() { + if (size <= 0) { + throw new BadRequestException("sample paramater size must fall in the range (1, " + SIZE_MAX + ")"); + } + if (size > SIZE_MAX) { + throw new BadRequestException("sample paramater size must fall in the range (1, " + SIZE_MAX + ")"); + } + if (ageMin != null) { + if (ageMin < 0) { + throw new BadRequestException("Minimum age may not be less than 0"); + } + if (ageMin >= AGE_MAX) { + throw new BadRequestException("Minimum age must be smaller than " + AGE_MAX); + } + } + if (ageMax != null) { + if (ageMax < 0) { + throw new BadRequestException("Maximum age may not be less than 0"); + } + if (ageMax >= 500) { + throw new BadRequestException("Minimum age must be smaller than " + AGE_MAX); + } + if (ageMin != null && ageMax < ageMin) { + throw new BadRequestException("Maximum age " + ageMax + " may not be less than minimum age " + ageMin); + } + } + } + public Integer getAgeMin() { return ageMin; } diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index 680eee5293..507081d616 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -2,7 +2,8 @@ import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; import org.ohdsi.webapi.cohortsample.CohortSample; -import org.ohdsi.webapi.cohortsample.CohortSampleElementService; +import org.ohdsi.webapi.cohortsample.CohortSampleRepository; +import org.ohdsi.webapi.cohortsample.CohortSamplingService; import org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO; import org.ohdsi.webapi.source.Source; import org.ohdsi.webapi.source.SourceRepository; @@ -26,15 +27,18 @@ @Produces(MediaType.APPLICATION_JSON) public class CohortSampleService { private final CohortDefinitionRepository cohortDefinitionRepository; - private final CohortSampleElementService samplingService; + private final CohortSampleRepository sampleRepository; + private final CohortSamplingService samplingService; private final SourceRepository sourceRepository; @Autowired public CohortSampleService( - CohortSampleElementService samplingService, + CohortSampleRepository sampleRepository, + CohortSamplingService samplingService, SourceRepository sourceRepository, CohortDefinitionRepository cohortDefinitionRepository ) { + this.sampleRepository = sampleRepository; this.samplingService = samplingService; this.sourceRepository = sourceRepository; this.cohortDefinitionRepository = cohortDefinitionRepository; @@ -47,20 +51,19 @@ public List listCohortSamples( @PathParam("sourceKey") String sourceKey ) { Source source = getSource(sourceKey); - return this.samplingService.findSamples(source, cohortDefinitionId); + return this.sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, source.getId()); } @Path("/{sampleId}") @GET public CohortSample getCohortSample( - @PathParam("sourceKey") String sourceKey, - @PathParam("sampleId") int sampleId + @PathParam("sampleId") Integer sampleId ) { - Source source = getSource(sourceKey); - CohortSample sample = samplingService.findSample(source, sampleId); + CohortSample sample = sampleRepository.findOne(sampleId); if (sample == null) { throw new NotFoundException("Cohort sample with ID " + sampleId + " not found"); } + Source source = sourceRepository.findBySourceId(sample.getSourceId()); sample.setElements(this.samplingService.findSampleElements(source, sampleId)); return sample; } @@ -74,6 +77,7 @@ public CohortSample createCohortSample( @PathParam("cohortDefinitionId") int cohortDefinitionId, SampleParametersDTO sampleParameters ) { + sampleParameters.validate(); Source source = getSource(sourceKey); if (cohortDefinitionRepository.findOne(cohortDefinitionId) == null) { throw new NotFoundException("Cohort definition " + cohortDefinitionId + " does not exist."); diff --git a/src/main/resources/resources/cohortsample/sql/deleteSampleById.sql b/src/main/resources/resources/cohortsample/sql/deleteSampleById.sql deleted file mode 100644 index f87f90a220..0000000000 --- a/src/main/resources/resources/cohortsample/sql/deleteSampleById.sql +++ /dev/null @@ -1,4 +0,0 @@ -DELETE FROM @results_schema.cohort_sample s -WHERE s.id = @cohortSampleId -CASCADE -; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/deleteSampleElementsById.sql b/src/main/resources/resources/cohortsample/sql/deleteSampleElementsById.sql new file mode 100644 index 0000000000..1331e60131 --- /dev/null +++ b/src/main/resources/resources/cohortsample/sql/deleteSampleElementsById.sql @@ -0,0 +1,3 @@ +DELETE FROM @results_schema.cohort_sample_element s +WHERE s.cohort_sample_id = @cohortSampleId +; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql index 14153cd806..4a9d0005f6 100644 --- a/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql +++ b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql @@ -1,4 +1,5 @@ SELECT * FROM @results_schema.cohort_sample_element s WHERE s.cohort_sample_id = @cohortSampleId +ORDER BY s.rank ; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/findSampleByCohortDefinitionId.sql b/src/main/resources/resources/cohortsample/sql/findSampleByCohortDefinitionId.sql deleted file mode 100644 index 4bfdffce6d..0000000000 --- a/src/main/resources/resources/cohortsample/sql/findSampleByCohortDefinitionId.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT * -FROM @results_schema.cohort_sample s -WHERE s.cohort_definition_id = @cohortDefinitionId -; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/findSampleById.sql b/src/main/resources/resources/cohortsample/sql/findSampleById.sql deleted file mode 100644 index a93433e91b..0000000000 --- a/src/main/resources/resources/cohortsample/sql/findSampleById.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT * -FROM @results_schema.cohort_sample s -WHERE s.id = @cohortSampleId -; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/insertSample.sql b/src/main/resources/resources/cohortsample/sql/insertSample.sql deleted file mode 100644 index 7f3c689a1c..0000000000 --- a/src/main/resources/resources/cohortsample/sql/insertSample.sql +++ /dev/null @@ -1,3 +0,0 @@ -INSERT INTO @results_schema.cohort_sample (cohort_definition_id, size, age_min, age_max, gender_concept_id, created_by_id, created_date) -VALUES (@cohortDefinitionId, @size, @ageMin, @ageMax, @genderConceptId, @createdById, @createdDate) -; From c3adeba8159fad14130b90d0aeaded25c2b1797b Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 9 Jan 2020 16:45:48 +0100 Subject: [PATCH 08/33] Create index in separate query --- .../oracle/V2.8.0.20200109100200__cohort_sample_tables.sql | 7 ++++--- .../V2.8.0.20200109100200__cohort_sample_tables.sql | 7 ++++--- .../V2.8.0.20200109100200__cohort_sample_tables.sql | 7 ++++--- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql index d22156c904..2350b86cdb 100644 --- a/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql +++ b/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -14,8 +14,9 @@ CREATE TABLE ${ohdsiSchema}.cohort_sample( modified_by_id INTEGER, modified_date TIMESTAMP WITH TIME ZONE, CONSTRAINT fk_cohort_sample_definition_id FOREIGN KEY (cohort_definition_id) - REFERENCES ${ohdsiSchema}.cohort_definition (cohort_definition_id) ON DELETE CASCADE, + REFERENCES ${ohdsiSchema}.cohort_definition (id) ON DELETE CASCADE, CONSTRAINT fk_cohort_sample_source_id FOREIGN KEY (source_id) - REFERENCES ${ohdsiSchema}.source (source_id) ON DELETE CASCADE, - INDEX idx_cohort_sample_source (cohort_definition_id, source_id) + REFERENCES ${ohdsiSchema}.source (source_id) ON DELETE CASCADE ); + +CREATE INDEX idx_cohort_sample_source ON ${ohdsiSchema}.cohort_sample (cohort_definition_id, source_id); diff --git a/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql index dd193939ff..d8b1ffc727 100644 --- a/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql +++ b/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -14,8 +14,9 @@ CREATE TABLE ${ohdsiSchema}.cohort_sample( modified_by_id INTEGER, modified_date TIMESTAMP WITH TIME ZONE, CONSTRAINT fk_cohort_sample_definition_id FOREIGN KEY (cohort_definition_id) - REFERENCES ${ohdsiSchema}.cohort_definition (cohort_definition_id) ON DELETE CASCADE, + REFERENCES ${ohdsiSchema}.cohort_definition (id) ON DELETE CASCADE, CONSTRAINT fk_cohort_sample_source_id FOREIGN KEY (source_id) - REFERENCES ${ohdsiSchema}.source (source_id) ON DELETE CASCADE, - INDEX idx_cohort_sample_source (cohort_definition_id, source_id) + REFERENCES ${ohdsiSchema}.source (source_id) ON DELETE CASCADE ); + +CREATE INDEX idx_cohort_sample_source ON ${ohdsiSchema}.cohort_sample (cohort_definition_id, source_id); diff --git a/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql index 399513d37f..3c9fd0e20a 100644 --- a/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql +++ b/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -14,8 +14,9 @@ CREATE TABLE ${ohdsiSchema}.cohort_sample( modified_by_id INTEGER, modified_date DATETIMEOFFSET, CONSTRAINT fk_cohort_sample_definition_id FOREIGN KEY (cohort_definition_id) - REFERENCES ${ohdsiSchema}.cohort_definition (cohort_definition_id) ON DELETE CASCADE, + REFERENCES ${ohdsiSchema}.cohort_definition (id) ON DELETE CASCADE, CONSTRAINT fk_cohort_sample_source_id FOREIGN KEY (source_id) - REFERENCES ${ohdsiSchema}.source (source_id) ON DELETE CASCADE, - INDEX idx_cohort_sample_source (cohort_definition_id, source_id) + REFERENCES ${ohdsiSchema}.source (source_id) ON DELETE CASCADE ); + +CREATE INDEX idx_cohort_sample_source ON ${ohdsiSchema}.cohort_sample (cohort_definition_id, source_id); From 7933edf4d87241b3115be1208a73d0ea426ae298 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Wed, 15 Jan 2020 15:38:32 +0100 Subject: [PATCH 09/33] Misc fixes --- .../webapi/cohortsample/CohortSample.java | 35 ++- .../cohortsample/CohortSamplingService.java | 241 ++++++++++++++---- .../cohortsample/dto/CohortSampleDTO.java | 67 ++++- .../cohortsample/dto/SampleElementDTO.java | 56 ++++ .../cohortsample/dto/SampleParametersDTO.java | 194 +++++++++++--- .../webapi/service/CohortSampleService.java | 24 +- .../org/ohdsi/webapi/service/DDLService.java | 1 - ...0.20200109100200__cohort_sample_tables.sql | 1 + ...0.20200109100200__cohort_sample_tables.sql | 1 + ...0.20200109100200__cohort_sample_tables.sql | 1 + .../resources/ddl/results/cohort_sample.sql | 11 - .../ddl/results/cohort_sample_element.sql | 8 +- 12 files changed, 514 insertions(+), 126 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java delete mode 100644 src/main/resources/ddl/results/cohort_sample.sql diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java index 234c5be154..5fa045bfe0 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java @@ -36,6 +36,9 @@ public class CohortSample extends CommonEntity { @Column(name = "source_id") private int sourceId; + @Column(name = "age_mode") + private String ageMode; + @Column(name = "age_min") private Integer ageMin; @@ -88,11 +91,7 @@ public Integer getAgeMin() { } public void setAgeMin(Integer ageMin) { - if (ageMin == 0) { - this.ageMin = null; - } else { - this.ageMin = ageMin; - } + this.ageMin = ageMin; } public Integer getAgeMax() { @@ -100,11 +99,7 @@ public Integer getAgeMax() { } public void setAgeMax(Integer ageMax) { - if (ageMax == 0) { - this.ageMax = null; - } else { - this.ageMax = ageMax; - } + this.ageMax = ageMax; } public Integer getGenderConceptId() { @@ -112,7 +107,7 @@ public Integer getGenderConceptId() { } public void setGenderConceptId(Integer genderConceptId) { - if (genderConceptId == 0) { + if (genderConceptId == null || genderConceptId == 0) { this.genderConceptId = null; } else { this.genderConceptId = genderConceptId; @@ -123,7 +118,23 @@ public int getSourceId() { return sourceId; } - public void setSourceId(Integer sourceId) { + public void setSourceId(int sourceId) { this.sourceId = sourceId; } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } + + public String getAgeMode() { + return ageMode; + } + + public void setAgeMode(String ageMode) { + this.ageMode = ageMode; + } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index 544df7e05c..3f196620f1 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -1,10 +1,11 @@ package org.ohdsi.webapi.cohortsample; +import org.ohdsi.webapi.cohortsample.dto.CohortSampleDTO; +import org.ohdsi.webapi.cohortsample.dto.SampleElementDTO; import org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO; import org.ohdsi.webapi.service.AbstractDaoService; import org.ohdsi.webapi.source.Source; import org.ohdsi.webapi.source.SourceDaimon; -import org.ohdsi.webapi.util.CancelableJdbcTemplate; import org.ohdsi.webapi.util.PreparedStatementRenderer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; @@ -13,12 +14,17 @@ import org.springframework.transaction.support.TransactionCallback; import org.springframework.transaction.support.TransactionTemplate; +import javax.ws.rs.BadRequestException; +import javax.ws.rs.NotFoundException; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; @Component public class CohortSamplingService extends AbstractDaoService { @@ -36,95 +42,226 @@ public CohortSamplingService( public List findSampleElements(Source source, int cohortSampleId) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); - return jdbcTemplate.query( - new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleId.sql", - "results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results), - "cohortSampleId", cohortSampleId - ).getSql(), elementRowMapper); + PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleId.sql", + "results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results), + "cohortSampleId", cohortSampleId); + return jdbcTemplate.query(renderer.getSql(), renderer.getOrderedParams(), elementRowMapper); } - public CohortSample createSample(Source source, int cohortDefinitionId, SampleParametersDTO sampleParameters) { + public CohortSampleDTO createSample(Source source, int cohortDefinitionId, SampleParametersDTO sampleParameters) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); CohortSample sample = new CohortSample(); + sample.setName(sampleParameters.getName()); sample.setCohortDefinitionId(cohortDefinitionId); sample.setSourceId(source.getId()); sample.setSize(sampleParameters.getSize()); - sample.setAgeMin(sampleParameters.getAgeMin()); - sample.setAgeMax(sampleParameters.getAgeMax()); - sample.setGenderConceptId(sampleParameters.getGenderConceptId()); + + SampleParametersDTO.AgeDTO age = sampleParameters.getAge(); + if (age != null) { + switch (age.getMode()) { + case LESS_THAN: + case LESS_THAN_OR_EQUAL: + sample.setAgeMax(age.getValue()); + break; + case GREATER_THAN: + case GREATER_THAN_OR_EQUAL: + sample.setAgeMin(age.getValue()); + break; + case EQUAL_TO: + sample.setAgeMin(age.getValue()); + sample.setAgeMax(age.getValue()); + break; + case BETWEEN: + case NOT_BETWEEN: + sample.setAgeMin(age.getMin()); + sample.setAgeMax(age.getMax()); + break; + } + sample.setAgeMode(age.getMode().getSerialName()); + } + + SampleParametersDTO.GenderDTO gender = sampleParameters.getGender(); + if (gender != null) { + sample.setGenderConceptId(gender.getConceptId()); + } sample.setCreatedBy(getCurrentUser()); sample.setCreatedDate(new Date()); + log.info("Sampling elements"); final List elements = sampleElements(sample, jdbcTemplate, source); + if (elements.size() < sample.getSize()) { + sample.setSize(elements.size()); + } + transactionTemplate.execute((TransactionCallback) transactionStatus -> { + log.info("Saving sample"); CohortSample updatedSample = sampleRepository.save(sample); - insertSampledElements(source, jdbcTemplate, updatedSample.getId(), elements); - sample.setElements(elements); - return null; }); - return sample; + return sampleToSampleDTO(sample, elements); + } + + public CohortSampleDTO sampleToSampleDTO(CohortSample sample, List elements) { + CohortSampleDTO sampleDTO = new CohortSampleDTO(); + sampleDTO.setId(sample.getId()); + sampleDTO.setSize(sample.getSize()); + sampleDTO.setCohortDefinitionId(sample.getCohortDefinitionId()); + sampleDTO.setSourceId(sample.getSourceId()); + sampleDTO.setCreatedDate(sample.getCreatedDate()); + sampleDTO.setCreatedBy(sample.getCreatedBy()); + + SampleParametersDTO.AgeMode ageMode = SampleParametersDTO.AgeMode.fromSerialName(sample.getAgeMode()); + if (ageMode != null) { + SampleParametersDTO.AgeDTO age = new SampleParametersDTO.AgeDTO(); + age.setMode(ageMode); + switch (ageMode) { + case LESS_THAN: + case LESS_THAN_OR_EQUAL: + case EQUAL_TO: + age.setValue(sample.getAgeMax()); + break; + case GREATER_THAN: + case GREATER_THAN_OR_EQUAL: + age.setValue(sample.getAgeMin()); + break; + case BETWEEN: + case NOT_BETWEEN: + age.setMin(sample.getAgeMin()); + age.setMax(sample.getAgeMax()); + break; + } + sampleDTO.setAge(age); + } + if (sample.getGenderConceptId() != null) { + SampleParametersDTO.GenderDTO gender = new SampleParametersDTO.GenderDTO(); + gender.setConceptId(sample.getGenderConceptId()); + sampleDTO.setGender(gender); + } + + sampleDTO.setElements(sampleElementToDTO(elements)); + return sampleDTO; + } + + private List sampleElementToDTO(List elements) { + if (elements == null) { + return null; + } + + return elements.stream() + .map(el -> { + SampleElementDTO elementDTO = new SampleElementDTO(); + elementDTO.setRank(el.getRank()); + elementDTO.setPersonId(el.getPersonId()); + elementDTO.setAge(el.getAge()); + elementDTO.setGenderConceptId(el.getGenderConceptId()); + return elementDTO; + }) + .collect(Collectors.toList()); } private void insertSampledElements(Source source, JdbcTemplate jdbcTemplate, int sampleId, List elements) { + if (elements.isEmpty()) { + return; + } + String[] parameters = new String[] { "results_schema" }; String[] parameterValues = new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results) }; String[] sqlParameters = new String[] { "cohortSampleId", "rank", "personId", "age", "genderConceptId" }; - Object[] sqlValues = new Object[5]; - sqlValues[0] = sampleId; - String[] statements = new String[elements.size()]; - int i = 0; + String statement = null; + List variables = new ArrayList<>(elements.size()); for (SampleElement element : elements) { - sqlValues[1] = element.getRank(); - sqlValues[2] = element.getPersonId(); - sqlValues[3] = element.getAge(); - sqlValues[4] = element.getGenderConceptId(); - statements[i] = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/insertSample.sql", parameters, parameterValues, sqlParameters, sqlValues).getSql(); - i++; + Object[] sqlValues = new Object[] { + sampleId, + element.getRank(), + element.getPersonId(), + element.getAge(), + element.getGenderConceptId() }; + + PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/insertSampleElement.sql", parameters, parameterValues, sqlParameters, sqlValues); + + if (statement == null) { + statement = renderer.getSql(); + } + + variables.add(renderer.getOrderedParams()); } - jdbcTemplate.batchUpdate(statements); + jdbcTemplate.batchUpdate(statement, variables); } private List sampleElements(CohortSample sample, JdbcTemplate jdbcTemplate, Source source) { StringBuilder expressionBuilder = new StringBuilder(); - Map parameters = new LinkedHashMap<>(); Map sqlVariables = new LinkedHashMap<>(); - parameters.put("results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results)); - parameters.put("CDM_schema", source.getTableQualifier(SourceDaimon.DaimonType.CDM)); sqlVariables.put("cohort_definition_id", sample.getCohortDefinitionId()); - if (sample.getAgeMin() != null) { - expressionBuilder.append("AND cast(year(c.cohort_start_date) - p.year_of_birth as int) >= @age_min"); - sqlVariables.put("age_min", sample.getAgeMin()); - } - if (sample.getAgeMax() != null) { - expressionBuilder.append("AND cast(year(c.cohort_start_date) - p.year_of_birth as int) < @gender_concept_id").append(sample.getAgeMax()); - sqlVariables.put("age_max", sample.getAgeMax()); + + SampleParametersDTO.AgeMode ageMode = SampleParametersDTO.AgeMode.fromSerialName(sample.getAgeMode()); + + if (ageMode != null) { + switch (ageMode) { + case LESS_THAN: + expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) < @age"); + sqlVariables.put("age", sample.getAgeMax()); + break; + case LESS_THAN_OR_EQUAL: + expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) <= @age"); + sqlVariables.put("age", sample.getAgeMax()); + break; + case GREATER_THAN: + expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) > @age"); + sqlVariables.put("age", sample.getAgeMin()); + break; + case GREATER_THAN_OR_EQUAL: + expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) >= @age"); + sqlVariables.put("age", sample.getAgeMin()); + break; + case EQUAL_TO: + expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) = @age"); + sqlVariables.put("age", sample.getAgeMin()); + break; + case BETWEEN: + expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) <= @age_max AND cast(year(c.cohort_start_date) - p.year_of_birth as int) >= @age_min"); + sqlVariables.put("age_min", sample.getAgeMin()); + sqlVariables.put("age_max", sample.getAgeMax()); + break; + case NOT_BETWEEN: + expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) > @age_max AND cast(year(c.cohort_start_date) - p.year_of_birth as int) < @age_min"); + sqlVariables.put("age_min", sample.getAgeMin()); + sqlVariables.put("age_max", sample.getAgeMax()); + break; + } } + if (sample.getGenderConceptId() != null) { - expressionBuilder.append("AND p.gender_concept_id = ").append(sample.getGenderConceptId()); + expressionBuilder.append(" AND p.gender_concept_id = @gender_concept_id"); sqlVariables.put("gender_concept_id", sample.getGenderConceptId()); } - parameters.put("expression", expressionBuilder.toString()); + String[] parameterKeys = new String[] { "results_schema", "CDM_schema", "expression"}; + String[] parameterValues = new String[] { + source.getTableQualifier(SourceDaimon.DaimonType.Results), + source.getTableQualifier(SourceDaimon.DaimonType.CDM), + expressionBuilder.toString() }; + String[] sqlVariableKeys = sqlVariables.keySet().toArray(new String[0]); + Object[] sqlVariableValues = Stream.of(sqlVariableKeys) + .map(sqlVariables::get) + .toArray(Object[]::new); PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/generateSample.sql", - parameters.keySet().toArray(new String[0]), - parameters.values().toArray(new String[0]), - sqlVariables.keySet().toArray(new String[0]), - sqlVariables.values().toArray(new Object[0])); + parameterKeys, + parameterValues, + sqlVariableKeys, + sqlVariableValues); - CancelableJdbcTemplate template = new CancelableJdbcTemplate(); - template.setMaxRows(sample.getSize()); + jdbcTemplate.setMaxRows(sample.getSize()); - return jdbcTemplate.query(renderer.getSql(), (rs, rowNum) -> { + return jdbcTemplate.query(renderer.getSql(), renderer.getOrderedParams(), (rs, rowNum) -> { SampleElement element = new SampleElement(); element.setRank(rowNum); element.setAge(rs.getInt("age")); @@ -134,20 +271,30 @@ private List sampleElements(CohortSample sample, JdbcTemplate jdb }); } - public void deleteSample(Source source, int sampleId) { + public void deleteSample(int cohortDefinitionId, Source source, int sampleId) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); String resultsSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); String sql = new PreparedStatementRenderer( source, - "/resources/cohortsample/sql/deleteElementsBySampleId.sql", + "/resources/cohortsample/sql/deleteSampleElementsById.sql", "results_schema", resultsSchema, "cohortSampleId", sampleId).getSql(); + CohortSample sample = sampleRepository.findOne(sampleId); + if (sample == null) { + throw new NotFoundException("Sample with ID " + sampleId + " does not exist"); + } + if (sample.getCohortDefinitionId() != cohortDefinitionId) { + throw new BadRequestException("Cohort definition ID " + sample.getCohortDefinitionId() + " does not match provided cohort definition id " + cohortDefinitionId); + } + if (sample.getSourceId() != source.getId()) { + throw new BadRequestException("Source " + sample.getSourceId() + " does not match provided source " + source.getId()); + } transactionTemplate.execute((TransactionCallback) transactionStatus -> { sampleRepository.delete(sampleId); - jdbcTemplate.update(sql); + jdbcTemplate.update(sql, sampleId); return null; }); } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java index 22921b4d90..891ea6f491 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java @@ -1,13 +1,26 @@ package org.ohdsi.webapi.cohortsample.dto; +import com.fasterxml.jackson.annotation.JsonInclude; import org.ohdsi.webapi.cohortsample.SampleElement; +import org.ohdsi.webapi.shiro.Entities.UserEntity; +import java.util.Date; import java.util.List; +@JsonInclude(JsonInclude.Include.NON_NULL) public class CohortSampleDTO { private int id; private int size; - private List elements; + + + private Date createdDate; + private UserEntity createdBy; + private int cohortDefinitionId; + private int sourceId; + private SampleParametersDTO.AgeDTO age; + private SampleParametersDTO.GenderDTO gender; + + private List elements; public int getId() { return id; @@ -17,6 +30,38 @@ public void setId(int id) { this.id = id; } + public Date setCreatedDate() { + return createdDate; + } + + public void setCreatedDate(Date createdDate) { + this.createdDate = createdDate; + } + + public UserEntity getCreatedBy() { + return createdBy; + } + + public void setCreatedBy(UserEntity createdBy) { + this.createdBy = createdBy; + } + + public int getCohortDefinitionId() { + return cohortDefinitionId; + } + + public void setCohortDefinitionId(int cohortDefinitionId) { + this.cohortDefinitionId = cohortDefinitionId; + } + + public int getSourceId() { + return sourceId; + } + + public void setSourceId(int sourceId) { + this.sourceId = sourceId; + } + public int getSize() { return size; } @@ -25,11 +70,27 @@ public void setSize(int size) { this.size = size; } - public List getElements() { + public List getElements() { return elements; } - public void setElements(List elements) { + public void setElements(List elements) { this.elements = elements; } + + public SampleParametersDTO.AgeDTO getAge() { + return age; + } + + public void setAge(SampleParametersDTO.AgeDTO age) { + this.age = age; + } + + public SampleParametersDTO.GenderDTO getGender() { + return gender; + } + + public void setGender(SampleParametersDTO.GenderDTO gender) { + this.gender = gender; + } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java new file mode 100644 index 0000000000..d04336229c --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java @@ -0,0 +1,56 @@ +package org.ohdsi.webapi.cohortsample.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public class SampleElementDTO { + private Integer sampleId; + + private int rank; + + private long personId; + + private long genderConceptId; + + private int age; + + public Integer getSampleId() { + return sampleId; + } + + public void setSampleId(Integer sampleId) { + this.sampleId = sampleId; + } + + public int getRank() { + return rank; + } + + public void setRank(int rank) { + this.rank = rank; + } + + public long getPersonId() { + return personId; + } + + public void setPersonId(long personId) { + this.personId = personId; + } + + public long getGenderConceptId() { + return genderConceptId; + } + + public void setGenderConceptId(long genderConceptId) { + this.genderConceptId = genderConceptId; + } + + public int getAge() { + return age; + } + + public void setAge(int age) { + this.age = age; + } +} diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java index 9c1d9ee116..bd92a67791 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java @@ -1,66 +1,52 @@ package org.ohdsi.webapi.cohortsample.dto; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonValue; + import javax.ws.rs.BadRequestException; +import java.util.stream.Stream; public class SampleParametersDTO { private static final int SIZE_MAX = 500; private static final int AGE_MAX = 500; private int size; - private Integer ageMin; - private Integer ageMax; - private Integer genderConceptId; + private String name; + private GenderDTO gender; + + private AgeDTO age; public void validate() { + if (name == null) { + throw new BadRequestException("Sample must have a name"); + } if (size <= 0) { - throw new BadRequestException("sample paramater size must fall in the range (1, " + SIZE_MAX + ")"); + throw new BadRequestException("sample parameter size must fall in the range (1, " + SIZE_MAX + ")"); } if (size > SIZE_MAX) { - throw new BadRequestException("sample paramater size must fall in the range (1, " + SIZE_MAX + ")"); + throw new BadRequestException("sample parameter size must fall in the range (1, " + SIZE_MAX + ")"); } - if (ageMin != null) { - if (ageMin < 0) { - throw new BadRequestException("Minimum age may not be less than 0"); - } - if (ageMin >= AGE_MAX) { - throw new BadRequestException("Minimum age must be smaller than " + AGE_MAX); - } - } - if (ageMax != null) { - if (ageMax < 0) { - throw new BadRequestException("Maximum age may not be less than 0"); - } - if (ageMax >= 500) { - throw new BadRequestException("Minimum age must be smaller than " + AGE_MAX); - } - if (ageMin != null && ageMax < ageMin) { - throw new BadRequestException("Maximum age " + ageMax + " may not be less than minimum age " + ageMin); + if (age != null) { + if (!age.validate()) { + age = null; } } } - public Integer getAgeMin() { - return ageMin; - } - - public void setAgeMin(Integer ageMin) { - this.ageMin = ageMin; + public AgeDTO getAge() { + return age; } - public Integer getAgeMax() { - return ageMax; + public void setAge(AgeDTO age) { + this.age = age; } - public void setAgeMax(Integer ageMax) { - this.ageMax = ageMax; + public GenderDTO getGender() { + return gender; } - public Integer getGenderConceptId() { - return genderConceptId; - } - - public void setGenderConceptId(Integer genderConceptId) { - this.genderConceptId = genderConceptId; + public void setGender(GenderDTO gender) { + this.gender = gender; } public int getSize() { @@ -70,4 +56,136 @@ public int getSize() { public void setSize(int size) { this.size = size; } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public enum AgeMode { + LESS_THAN("lessThan"), + LESS_THAN_OR_EQUAL("lessThanOrEqual"), + GREATER_THAN("greaterThan"), + GREATER_THAN_OR_EQUAL("greaterThanOrEqual"), + EQUAL_TO("equalTo"), + BETWEEN("between"), + NOT_BETWEEN("notBetween"); + + private final String serialName; + + AgeMode(String serialName) { + this.serialName = serialName; + } + + @JsonValue + public String getSerialName() { + return serialName; + } + + public static AgeMode fromSerialName(String name) { + return Stream.of(SampleParametersDTO.AgeMode.values()) + .filter(mode -> mode.getSerialName().equals(name)) + .findFirst() + .orElse(null); + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class GenderDTO { + private Integer conceptId; + + public Integer getConceptId() { + return conceptId; + } + + public void setConceptId(Integer conceptId) { + this.conceptId = conceptId; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class AgeDTO { + private Integer min; + private Integer max; + private Integer value; + private AgeMode mode; + + public boolean validate() { + if (mode == null) { + if (min != null || max != null || value != null) { + throw new BadRequestException("Cannot specify age without a mode to use age with."); + } else { + return false; + } + } + switch (mode) { + case LESS_THAN: + case LESS_THAN_OR_EQUAL: + case GREATER_THAN: + case GREATER_THAN_OR_EQUAL: + case EQUAL_TO: + if (value == null) { + throw new BadRequestException("Cannot use single age comparison mode " + mode + " without age property."); + } + if (min != null || max != null) { + throw new BadRequestException("Cannot use age range property with comparison mode " + mode + "."); + } + break; + case BETWEEN: + case NOT_BETWEEN: + if (min == null || max == null) { + throw new BadRequestException("Cannot use age range comparison mode " + mode + " without ageMin and ageMax properties."); + } + if (value != null) { + throw new BadRequestException("Cannot use single age property with comparison mode " + mode + "."); + } + if (min < 0) { + throw new BadRequestException("Minimum age may not be less than 0"); + } + if (max >= AGE_MAX) { + throw new BadRequestException("Minimum age must be smaller than " + AGE_MAX); + } + if (min > max) { + throw new BadRequestException("Maximum age " + max + " may not be less than minimum age " + min); + } + break; + } + return true; + } + + public Integer getMin() { + return min; + } + + public void setMin(Integer min) { + this.min = min; + } + + public Integer getMax() { + return max; + } + + public void setMax(Integer max) { + this.max = max; + } + + public Integer getValue() { + return value; + } + + public void setValue(Integer value) { + this.value = value; + } + + public AgeMode getMode() { + return mode; + } + + public void setMode(AgeMode mode) { + this.mode = mode; + } + } } diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index 507081d616..c72293689c 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -4,6 +4,8 @@ import org.ohdsi.webapi.cohortsample.CohortSample; import org.ohdsi.webapi.cohortsample.CohortSampleRepository; import org.ohdsi.webapi.cohortsample.CohortSamplingService; +import org.ohdsi.webapi.cohortsample.SampleElement; +import org.ohdsi.webapi.cohortsample.dto.CohortSampleDTO; import org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO; import org.ohdsi.webapi.source.Source; import org.ohdsi.webapi.source.SourceRepository; @@ -21,6 +23,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.util.List; +import java.util.stream.Collectors; @Path("/cohortsample/{cohortDefinitionId}/{sourceKey}") @Component @@ -46,17 +49,19 @@ public CohortSampleService( @Path("/") @GET - public List listCohortSamples( + public List listCohortSamples( @PathParam("cohortDefinitionId") int cohortDefinitionId, @PathParam("sourceKey") String sourceKey ) { Source source = getSource(sourceKey); - return this.sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, source.getId()); + return this.sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, source.getId()).stream() + .map(s -> samplingService.sampleToSampleDTO(s, null)) + .collect(Collectors.toList()); } @Path("/{sampleId}") @GET - public CohortSample getCohortSample( + public CohortSampleDTO getCohortSample( @PathParam("sampleId") Integer sampleId ) { CohortSample sample = sampleRepository.findOne(sampleId); @@ -64,15 +69,14 @@ public CohortSample getCohortSample( throw new NotFoundException("Cohort sample with ID " + sampleId + " not found"); } Source source = sourceRepository.findBySourceId(sample.getSourceId()); - sample.setElements(this.samplingService.findSampleElements(source, sampleId)); - return sample; + List elements = this.samplingService.findSampleElements(source, sampleId); + return samplingService.sampleToSampleDTO(sample, elements); } - @Path("/") @POST @Consumes(MediaType.APPLICATION_JSON) - public CohortSample createCohortSample( + public CohortSampleDTO createCohortSample( @PathParam("sourceKey") String sourceKey, @PathParam("cohortDefinitionId") int cohortDefinitionId, SampleParametersDTO sampleParameters @@ -89,10 +93,14 @@ public CohortSample createCohortSample( @DELETE public Response deleteCohortSample( @PathParam("sourceKey") String sourceKey, + @PathParam("cohortDefinitionId") int cohortDefinitionId, @PathParam("sampleId") int sampleId ) { Source source = getSource(sourceKey); - samplingService.deleteSample(source, sampleId); + if (cohortDefinitionRepository.findOne(cohortDefinitionId) == null) { + throw new NotFoundException("Cohort definition " + cohortDefinitionId + " does not exist."); + } + samplingService.deleteSample(cohortDefinitionId, source, sampleId); return Response.status(Response.Status.NO_CONTENT).build(); } diff --git a/src/main/java/org/ohdsi/webapi/service/DDLService.java b/src/main/java/org/ohdsi/webapi/service/DDLService.java index da58755aa3..9a80d94803 100644 --- a/src/main/java/org/ohdsi/webapi/service/DDLService.java +++ b/src/main/java/org/ohdsi/webapi/service/DDLService.java @@ -76,7 +76,6 @@ public class DDLService { "/ddl/results/heracles_results_dist.sql", "/ddl/results/heracles_periods.sql", // cohort sampling - "/ddl/results/cohort_sample.sql", "/ddl/results/cohort_sample_element.sql", // incidence rates "/ddl/results/ir_analysis_dist.sql", diff --git a/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql index 2350b86cdb..0ac122df2b 100644 --- a/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql +++ b/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -8,6 +8,7 @@ CREATE TABLE ${ohdsiSchema}.cohort_sample( size NUMBER(9) NOT NULL, age_min NUMBER(4), age_max NUMBER(4), + age_mode VARCHAR(24), gender_concept_id NUMBER(19), created_by_id INTEGER, created_date TIMESTAMP WITH TIME ZONE DEFAULT sysdate NOT NULL, diff --git a/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql index d8b1ffc727..3702433bdf 100644 --- a/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql +++ b/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -8,6 +8,7 @@ CREATE TABLE ${ohdsiSchema}.cohort_sample( size INTEGER NOT NULL, age_min SMALLINT NULL, age_max SMALLINT NULL, + age_mode VARCHAR(24), gender_concept_id INTEGER NULL, created_by_id INTEGER, created_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT (now()), diff --git a/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql index 3c9fd0e20a..ab6b2f946e 100644 --- a/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql +++ b/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -8,6 +8,7 @@ CREATE TABLE ${ohdsiSchema}.cohort_sample( size NUMBER(9) NOT NULL, age_min NUMBER(4) NULL, age_max NUMBER(4) NULL, + age_mode VARCHAR(24), gender_concept_id NUMBER(19) NULL, created_by_id INTEGER, created_date DATETIMEOFFSET, diff --git a/src/main/resources/ddl/results/cohort_sample.sql b/src/main/resources/ddl/results/cohort_sample.sql deleted file mode 100644 index 2be7cb2650..0000000000 --- a/src/main/resources/ddl/results/cohort_sample.sql +++ /dev/null @@ -1,11 +0,0 @@ -IF OBJECT_ID('@results_schema.cohort_sample', 'U') IS NULL -CREATE TABLE @results_schema.cohort_sample( - id int NOT NULL, - name varchar(255) NOT NULL, - cohort_definition_id int NOT NULL, - age_min int, - age_max int, - gender_concept_id int, - size int NOT NULL, - CONSTRAINT pk_cohort_sample_id PRIMARY KEY (id) -); diff --git a/src/main/resources/ddl/results/cohort_sample_element.sql b/src/main/resources/ddl/results/cohort_sample_element.sql index 4b2e639923..902f80e02e 100644 --- a/src/main/resources/ddl/results/cohort_sample_element.sql +++ b/src/main/resources/ddl/results/cohort_sample_element.sql @@ -4,11 +4,7 @@ CREATE TABLE @results_schema.cohort_sample_element( rank int NOT NULL, person_id bigint NOT NULL, age int, - gender_concept_id int, - CONSTRAINT fk_cohort_sample_element_id - FOREIGN KEY (cohort_sample_i d) - REFERENCES @results_schema.cohort_sample (id) - ON DELETE CASCADE + gender_concept_id int ); -CREATE UNIQUE INDEX idx_cohort_sample_element_rank ON @results_schema.cohort_sample_element (cohort_sample_id, rank); +CREATE INDEX idx_cohort_sample_element_rank ON @results_schema.cohort_sample_element (cohort_sample_id, rank); From d82594e068215f6f704b3a8602921e440d5a3801 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 16 Jan 2020 12:34:26 +0100 Subject: [PATCH 10/33] Fix gender sampling and DTOs --- .../webapi/cohortsample/CohortSample.java | 16 ++-- .../cohortsample/CohortSamplingService.java | 91 ++++++++++++++++--- .../cohortsample/dto/CohortSampleDTO.java | 21 +++-- .../cohortsample/dto/SampleParametersDTO.java | 52 +++++++++++ ...0.20200109100200__cohort_sample_tables.sql | 2 +- ...0.20200109100200__cohort_sample_tables.sql | 2 +- ...0.20200109100200__cohort_sample_tables.sql | 2 +- 7 files changed, 153 insertions(+), 33 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java index 5fa045bfe0..691c86cf7a 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java @@ -45,8 +45,8 @@ public class CohortSample extends CommonEntity { @Column(name = "age_max") private Integer ageMax; - @Column(name = "gender_concept_id") - private Integer genderConceptId; + @Column(name = "gender_concept_ids") + private String genderConceptIds; @Column private int size; @@ -102,16 +102,12 @@ public void setAgeMax(Integer ageMax) { this.ageMax = ageMax; } - public Integer getGenderConceptId() { - return genderConceptId; + public String getGenderConceptIds() { + return genderConceptIds; } - public void setGenderConceptId(Integer genderConceptId) { - if (genderConceptId == null || genderConceptId == 0) { - this.genderConceptId = null; - } else { - this.genderConceptId = genderConceptId; - } + public void setGenderConceptIds(String genderConceptIds) { + this.genderConceptIds = genderConceptIds; } public int getSourceId() { diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index 3f196620f1..045dfc2bf5 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -4,8 +4,10 @@ import org.ohdsi.webapi.cohortsample.dto.SampleElementDTO; import org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO; import org.ohdsi.webapi.service.AbstractDaoService; +import org.ohdsi.webapi.shiro.Entities.UserEntity; import org.ohdsi.webapi.source.Source; import org.ohdsi.webapi.source.SourceDaimon; +import org.ohdsi.webapi.user.dto.UserDTO; import org.ohdsi.webapi.util.PreparedStatementRenderer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; @@ -19,6 +21,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; @@ -26,6 +29,9 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO.GenderDTO.GENDER_FEMALE_CONCEPT_ID; +import static org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO.GenderDTO.GENDER_MALE_CONCEPT_ID; + @Component public class CohortSamplingService extends AbstractDaoService { private final TransactionTemplate transactionTemplate; @@ -83,20 +89,33 @@ public CohortSampleDTO createSample(Source source, int cohortDefinitionId, Sampl SampleParametersDTO.GenderDTO gender = sampleParameters.getGender(); if (gender != null) { - sample.setGenderConceptId(gender.getConceptId()); + StringBuilder sb = new StringBuilder(12); + for (Integer conceptId : gender.getConceptIds()) { + if (sb.length() > 0) { + sb.append(','); + } + sb.append(conceptId); + } + if (gender.isOtherNonBinary()) { + if (sb.length() > 0) { + sb.append(','); + } + sb.append(-1); + } + sample.setGenderConceptIds(sb.toString()); } sample.setCreatedBy(getCurrentUser()); sample.setCreatedDate(new Date()); - log.info("Sampling elements"); - final List elements = sampleElements(sample, jdbcTemplate, source); + log.info("Sampling {} elements for cohort {}", sampleParameters.getSize(), cohortDefinitionId); + final List elements = sampleElements(sampleParameters, sample, jdbcTemplate, source); if (elements.size() < sample.getSize()) { sample.setSize(elements.size()); } transactionTemplate.execute((TransactionCallback) transactionStatus -> { - log.info("Saving sample"); + log.debug("Saving {} sample elements for cohort {}", sample.getSize(), cohortDefinitionId); CohortSample updatedSample = sampleRepository.save(sample); insertSampledElements(source, jdbcTemplate, updatedSample.getId(), elements); @@ -109,11 +128,19 @@ public CohortSampleDTO createSample(Source source, int cohortDefinitionId, Sampl public CohortSampleDTO sampleToSampleDTO(CohortSample sample, List elements) { CohortSampleDTO sampleDTO = new CohortSampleDTO(); sampleDTO.setId(sample.getId()); + sampleDTO.setName(sample.getName()); sampleDTO.setSize(sample.getSize()); sampleDTO.setCohortDefinitionId(sample.getCohortDefinitionId()); sampleDTO.setSourceId(sample.getSourceId()); sampleDTO.setCreatedDate(sample.getCreatedDate()); - sampleDTO.setCreatedBy(sample.getCreatedBy()); + UserEntity createdBy = sample.getCreatedBy(); + if (createdBy != null) { + UserDTO userDto = new UserDTO(); + userDto.setId(createdBy.getId()); + userDto.setLogin(createdBy.getLogin()); + userDto.setName(createdBy.getName()); + sampleDTO.setCreatedBy(userDto); + } SampleParametersDTO.AgeMode ageMode = SampleParametersDTO.AgeMode.fromSerialName(sample.getAgeMode()); if (ageMode != null) { @@ -137,10 +164,19 @@ public CohortSampleDTO sampleToSampleDTO(CohortSample sample, List conceptIds = Arrays.stream(sample.getGenderConceptIds().split(",")) + .map(Integer::valueOf) + .collect(Collectors.toList()); + + SampleParametersDTO.GenderDTO genderDto = new SampleParametersDTO.GenderDTO(); + + if (conceptIds.remove(Integer.valueOf(-1))) { + genderDto.setOtherNonBinary(true); + } + + genderDto.setConceptIds(conceptIds); + sampleDTO.setGender(genderDto); } sampleDTO.setElements(sampleElementToDTO(elements)); @@ -195,7 +231,7 @@ private void insertSampledElements(Source source, JdbcTemplate jdbcTemplate, int jdbcTemplate.batchUpdate(statement, variables); } - private List sampleElements(CohortSample sample, JdbcTemplate jdbcTemplate, Source source) { + private List sampleElements(SampleParametersDTO sampleParametersDTO, CohortSample sample, JdbcTemplate jdbcTemplate, Source source) { StringBuilder expressionBuilder = new StringBuilder(); Map sqlVariables = new LinkedHashMap<>(); @@ -238,9 +274,38 @@ private List sampleElements(CohortSample sample, JdbcTemplate jdb } } - if (sample.getGenderConceptId() != null) { - expressionBuilder.append(" AND p.gender_concept_id = @gender_concept_id"); - sqlVariables.put("gender_concept_id", sample.getGenderConceptId()); + SampleParametersDTO.GenderDTO gender = sampleParametersDTO.getGender(); + if (gender != null) { + List conceptIds = gender.getConceptIds(); + if (gender.isOtherNonBinary()) { + if (conceptIds.size() == 0) { + expressionBuilder.append(" AND p.gender_concept_id NOT IN (") + .append(GENDER_MALE_CONCEPT_ID) + .append(',') + .append(GENDER_FEMALE_CONCEPT_ID) + .append(')'); + } else if (conceptIds.size() == 1) { + if (conceptIds.get(0) == GENDER_FEMALE_CONCEPT_ID) { + expressionBuilder.append(" AND p.gender_concept_id <> ") + .append(GENDER_MALE_CONCEPT_ID); + + } else { + expressionBuilder.append(" AND p.gender_concept_id <> ") + .append(GENDER_FEMALE_CONCEPT_ID); + } + } + // else: all genders are selected + } else if (!conceptIds.isEmpty()) { + expressionBuilder.append(" AND p.gender_concept_id IN ("); + for (int i = 0; i < conceptIds.size(); i++) { + if (i > 0) { + expressionBuilder.append(','); + } + expressionBuilder.append(conceptIds.get(i)); + } + expressionBuilder.append(')'); + } + // else: all genders are selected } String[] parameterKeys = new String[] { "results_schema", "CDM_schema", "expression"}; diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java index 891ea6f491..e2e0de0733 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java @@ -1,8 +1,7 @@ package org.ohdsi.webapi.cohortsample.dto; import com.fasterxml.jackson.annotation.JsonInclude; -import org.ohdsi.webapi.cohortsample.SampleElement; -import org.ohdsi.webapi.shiro.Entities.UserEntity; +import org.ohdsi.webapi.user.dto.UserDTO; import java.util.Date; import java.util.List; @@ -10,11 +9,11 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class CohortSampleDTO { private int id; + private String name; private int size; - private Date createdDate; - private UserEntity createdBy; + private UserDTO createdBy; private int cohortDefinitionId; private int sourceId; private SampleParametersDTO.AgeDTO age; @@ -30,7 +29,7 @@ public void setId(int id) { this.id = id; } - public Date setCreatedDate() { + public Date getCreatedDate() { return createdDate; } @@ -38,11 +37,11 @@ public void setCreatedDate(Date createdDate) { this.createdDate = createdDate; } - public UserEntity getCreatedBy() { + public UserDTO getCreatedBy() { return createdBy; } - public void setCreatedBy(UserEntity createdBy) { + public void setCreatedBy(UserDTO createdBy) { this.createdBy = createdBy; } @@ -93,4 +92,12 @@ public SampleParametersDTO.GenderDTO getGender() { public void setGender(SampleParametersDTO.GenderDTO gender) { this.gender = gender; } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java index bd92a67791..0c2f8f374c 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java @@ -4,6 +4,9 @@ import com.fasterxml.jackson.annotation.JsonValue; import javax.ws.rs.BadRequestException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; import java.util.stream.Stream; public class SampleParametersDTO { @@ -31,6 +34,11 @@ public void validate() { age = null; } } + if (gender != null) { + if (!gender.validate()) { + gender = null; + } + } } public AgeDTO getAge() { @@ -95,8 +103,36 @@ public static AgeMode fromSerialName(String name) { @JsonInclude(JsonInclude.Include.NON_NULL) public static class GenderDTO { + public final static int GENDER_MALE_CONCEPT_ID = 8507; + public final static int GENDER_FEMALE_CONCEPT_ID = 8532; + private Integer conceptId; + private List conceptIds; + + private boolean otherNonBinary = false; + + public boolean validate() { + if (conceptIds == null) { + conceptIds = new ArrayList<>(); + } else if (conceptIds.contains(null)) { + conceptIds.removeIf(Objects::isNull); + } + if (conceptId != null) { + conceptIds.add(conceptId); + conceptId = null; + } + + if (!isOtherNonBinary() && conceptIds.isEmpty()) { + return false; + } + + if (isOtherNonBinary()) { + conceptIds.removeIf(i -> i != GENDER_MALE_CONCEPT_ID && i != GENDER_FEMALE_CONCEPT_ID); + } + return true; + } + public Integer getConceptId() { return conceptId; } @@ -104,6 +140,22 @@ public Integer getConceptId() { public void setConceptId(Integer conceptId) { this.conceptId = conceptId; } + + public List getConceptIds() { + return conceptIds; + } + + public void setConceptIds(List conceptIds) { + this.conceptIds = conceptIds; + } + + public boolean isOtherNonBinary() { + return otherNonBinary; + } + + public void setOtherNonBinary(boolean otherNonBinary) { + this.otherNonBinary = otherNonBinary; + } } @JsonInclude(JsonInclude.Include.NON_NULL) diff --git a/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql index 0ac122df2b..dbc2915aff 100644 --- a/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql +++ b/src/main/resources/db/migration/oracle/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -9,7 +9,7 @@ CREATE TABLE ${ohdsiSchema}.cohort_sample( age_min NUMBER(4), age_max NUMBER(4), age_mode VARCHAR(24), - gender_concept_id NUMBER(19), + gender_concept_ids VARCHAR(255) NULL, created_by_id INTEGER, created_date TIMESTAMP WITH TIME ZONE DEFAULT sysdate NOT NULL, modified_by_id INTEGER, diff --git a/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql index 3702433bdf..ebedb5cb72 100644 --- a/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql +++ b/src/main/resources/db/migration/postgresql/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -9,7 +9,7 @@ CREATE TABLE ${ohdsiSchema}.cohort_sample( age_min SMALLINT NULL, age_max SMALLINT NULL, age_mode VARCHAR(24), - gender_concept_id INTEGER NULL, + gender_concept_ids VARCHAR(255) NULL, created_by_id INTEGER, created_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT (now()), modified_by_id INTEGER, diff --git a/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql index ab6b2f946e..69b5a729e6 100644 --- a/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql +++ b/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -9,7 +9,7 @@ CREATE TABLE ${ohdsiSchema}.cohort_sample( age_min NUMBER(4) NULL, age_max NUMBER(4) NULL, age_mode VARCHAR(24), - gender_concept_id NUMBER(19) NULL, + gender_concept_ids VARCHAR(255) NULL, created_by_id INTEGER, created_date DATETIMEOFFSET, modified_by_id INTEGER, From 1e7ef5a53abd6b86b8fda84bbf91e77d5f55f984 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 20 Jan 2020 12:23:31 +0100 Subject: [PATCH 11/33] Documentation and exception string updates --- .../webapi/cohortsample/CohortSample.java | 3 ++ .../cohortsample/CohortSampleRepository.java | 3 ++ .../cohortsample/CohortSamplingService.java | 22 ++++++++++ .../webapi/cohortsample/SampleElement.java | 1 + .../cohortsample/dto/CohortSampleDTO.java | 31 ++++++++++++++ .../cohortsample/dto/SampleElementDTO.java | 16 ++++++++ .../cohortsample/dto/SampleParametersDTO.java | 40 +++++++++++++------ 7 files changed, 103 insertions(+), 13 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java index 691c86cf7a..f55da0b494 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java @@ -12,6 +12,9 @@ import javax.persistence.Transient; import java.util.List; +/** + * Cohort sample details. + */ @Entity(name = "CohortSample") @Table(name = "cohort_sample") public class CohortSample extends CommonEntity { diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java index 39e57e0aa4..eca3f3de3d 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java @@ -5,6 +5,9 @@ import java.util.List; +/** + * Repository of samples. This does not fetch any sample elements. + */ @Component public interface CohortSampleRepository extends CrudRepository { List findByCohortDefinitionIdAndSourceId(int cohortDefinitionId, int sourceId); diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index 045dfc2bf5..878905c54f 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -32,6 +32,9 @@ import static org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO.GenderDTO.GENDER_FEMALE_CONCEPT_ID; import static org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO.GenderDTO.GENDER_MALE_CONCEPT_ID; +/** + * Service to do manage samples of a cohort definition. + */ @Component public class CohortSamplingService extends AbstractDaoService { private final TransactionTemplate transactionTemplate; @@ -46,6 +49,12 @@ public CohortSamplingService( this.sampleRepository = sampleRepository; } + /** + * Find all sample elements of a sample. + * @param source Source to use + * @param cohortSampleId sample ID of the elements. + * @return list of elements. + */ public List findSampleElements(Source source, int cohortSampleId) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleId.sql", @@ -54,6 +63,13 @@ public List findSampleElements(Source source, int cohortSampleId) return jdbcTemplate.query(renderer.getSql(), renderer.getOrderedParams(), elementRowMapper); } + /** + * Create a new sample in given source and cohort definition, using sample parameters. + * @param source Source to use + * @param cohortDefinitionId cohort definition ID to sample + * @param sampleParameters parameters to define the sample + * @return list of elements. + */ public CohortSampleDTO createSample(Source source, int cohortDefinitionId, SampleParametersDTO sampleParameters) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); @@ -125,6 +141,7 @@ public CohortSampleDTO createSample(Source source, int cohortDefinitionId, Sampl return sampleToSampleDTO(sample, elements); } + /** Convert a given sample with given elements to a DTO. */ public CohortSampleDTO sampleToSampleDTO(CohortSample sample, List elements) { CohortSampleDTO sampleDTO = new CohortSampleDTO(); sampleDTO.setId(sample.getId()); @@ -183,6 +200,7 @@ public CohortSampleDTO sampleToSampleDTO(CohortSample sample, List sampleElementToDTO(List elements) { if (elements == null) { return null; @@ -200,6 +218,7 @@ private List sampleElementToDTO(List elements) .collect(Collectors.toList()); } + /** Insert elements that have been sampled. */ private void insertSampledElements(Source source, JdbcTemplate jdbcTemplate, int sampleId, List elements) { if (elements.isEmpty()) { return; @@ -231,6 +250,7 @@ private void insertSampledElements(Source source, JdbcTemplate jdbcTemplate, int jdbcTemplate.batchUpdate(statement, variables); } + /** Sample elements based on parameters. */ private List sampleElements(SampleParametersDTO sampleParametersDTO, CohortSample sample, JdbcTemplate jdbcTemplate, Source source) { StringBuilder expressionBuilder = new StringBuilder(); Map sqlVariables = new LinkedHashMap<>(); @@ -336,6 +356,7 @@ private List sampleElements(SampleParametersDTO sampleParametersD }); } + /** Delete a sample and its elements. */ public void deleteSample(int cohortDefinitionId, Source source, int sampleId) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); String resultsSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); @@ -364,6 +385,7 @@ public void deleteSample(int cohortDefinitionId, Source source, int sampleId) { }); } + /** Maps a SQL result to a sample element. */ private static class CohortSampleElementRowMapper implements RowMapper { @Override public SampleElement mapRow(ResultSet rs, int rowNum) throws SQLException { diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java b/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java index f38e5dba0f..07196c8ee0 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java @@ -1,5 +1,6 @@ package org.ohdsi.webapi.cohortsample; +/** A single person that is part of a given sample. */ public class SampleElement { private int sampleId; diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java index e2e0de0733..33f5dce8ff 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java @@ -8,17 +8,48 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class CohortSampleDTO { + /** Cohort sample ID. */ private int id; + /** Cohort sample name. */ private String name; + /** + * Actual sample size. This may be different from the size specified by the user if not enough + * persons could be found matching the criteria. + */ private int size; + /** + * Date that the sample was created. + */ private Date createdDate; + + /** + * User that created the sample. If no login system is used, this is null. + */ private UserDTO createdBy; + + /** + * Cohort definition ID that was sampled. + */ private int cohortDefinitionId; + /** + * Source ID that was sampled. + */ private int sourceId; + + /** + * Age criteria used to create the sample. + */ private SampleParametersDTO.AgeDTO age; + + /** + * Gender criteria used to create the sample. + */ private SampleParametersDTO.GenderDTO gender; + /** + * Actually sampled elements. + */ private List elements; public int getId() { diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java index d04336229c..59f9ee2bb6 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java @@ -4,14 +4,30 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class SampleElementDTO { + /** + * Sample ID that this element belongs to. May be null if this object is part of a + * {@link CohortSampleDTO} object. + */ private Integer sampleId; + /** + * Rank of the object within the sample. This establishes order between elements. + */ private int rank; + /** + * Person ID of the element. + */ private long personId; + /** + * Gender ID of the person. + */ private long genderConceptId; + /** + * Age of the person. + */ private int age; public Integer getSampleId() { diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java index 0c2f8f374c..ab850c490e 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java @@ -13,12 +13,21 @@ public class SampleParametersDTO { private static final int SIZE_MAX = 500; private static final int AGE_MAX = 500; + /** Sample size. */ private int size; + /** Sample name. */ private String name; + + /** Gender criteria. */ private GenderDTO gender; + /** Age criteria. */ private AgeDTO age; + /** + * Validate this DTO. + * @throws BadRequestException if the DTO is not valid. + */ public void validate() { if (name == null) { throw new BadRequestException("Sample must have a name"); @@ -29,15 +38,11 @@ public void validate() { if (size > SIZE_MAX) { throw new BadRequestException("sample parameter size must fall in the range (1, " + SIZE_MAX + ")"); } - if (age != null) { - if (!age.validate()) { - age = null; - } + if (age != null && !age.validate()) { + age = null; } - if (gender != null) { - if (!gender.validate()) { - gender = null; - } + if (gender != null && !gender.validate()) { + gender = null; } } @@ -112,6 +117,10 @@ public static class GenderDTO { private boolean otherNonBinary = false; + /** + * Validate this DTO. + * @return true if this DTO contains any information, false otherwise. + */ public boolean validate() { if (conceptIds == null) { conceptIds = new ArrayList<>(); @@ -165,6 +174,11 @@ public static class AgeDTO { private Integer value; private AgeMode mode; + /** + * Validate this DTO. + * @return true if this DTO contains any information, false otherwise. + * @throws BadRequestException if the DTO is not valid. + */ public boolean validate() { if (mode == null) { if (min != null || max != null || value != null) { @@ -180,25 +194,25 @@ public boolean validate() { case GREATER_THAN_OR_EQUAL: case EQUAL_TO: if (value == null) { - throw new BadRequestException("Cannot use single age comparison mode " + mode + " without age property."); + throw new BadRequestException("Cannot use single age comparison mode " + mode.getSerialName() + " without age property."); } if (min != null || max != null) { - throw new BadRequestException("Cannot use age range property with comparison mode " + mode + "."); + throw new BadRequestException("Cannot use age range property with comparison mode " + mode.getSerialName() + "."); } break; case BETWEEN: case NOT_BETWEEN: if (min == null || max == null) { - throw new BadRequestException("Cannot use age range comparison mode " + mode + " without ageMin and ageMax properties."); + throw new BadRequestException("Cannot use age range comparison mode " + mode.getSerialName() + " without ageMin and ageMax properties."); } if (value != null) { - throw new BadRequestException("Cannot use single age property with comparison mode " + mode + "."); + throw new BadRequestException("Cannot use single age property with comparison mode " + mode.getSerialName() + "."); } if (min < 0) { throw new BadRequestException("Minimum age may not be less than 0"); } if (max >= AGE_MAX) { - throw new BadRequestException("Minimum age must be smaller than " + AGE_MAX); + throw new BadRequestException("Maximum age must be smaller than " + AGE_MAX); } if (min > max) { throw new BadRequestException("Maximum age " + max + " may not be less than minimum age " + min); From d65efea144124ad2ffb85782499200d7d4b3dc72 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 20 Jan 2020 14:48:18 +0100 Subject: [PATCH 12/33] Provide record counts with each sample element Note: this makes reading a sample much much slower. --- .../cohortsample/CohortSamplingService.java | 9 ++-- .../webapi/cohortsample/SampleElement.java | 10 +++++ .../cohortsample/dto/SampleElementDTO.java | 10 +++++ .../sql/findElementsByCohortSampleId.sql | 41 ++++++++++++++++++- 4 files changed, 66 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index 878905c54f..479ebe8155 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -58,7 +58,8 @@ public CohortSamplingService( public List findSampleElements(Source source, int cohortSampleId) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleId.sql", - "results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results), + new String[] {"results_schema", "CDM_schema"}, + new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results), source.getTableQualifier(SourceDaimon.DaimonType.CDM) }, "cohortSampleId", cohortSampleId); return jdbcTemplate.query(renderer.getSql(), renderer.getOrderedParams(), elementRowMapper); } @@ -213,6 +214,7 @@ private List sampleElementToDTO(List elements) elementDTO.setPersonId(el.getPersonId()); elementDTO.setAge(el.getAge()); elementDTO.setGenderConceptId(el.getGenderConceptId()); + elementDTO.setRecordCount(el.getRecordCount()); return elementDTO; }) .collect(Collectors.toList()); @@ -314,7 +316,7 @@ private List sampleElements(SampleParametersDTO sampleParametersD .append(GENDER_FEMALE_CONCEPT_ID); } } - // else: all genders are selected + // else: all genders are selected, no where statement needed } else if (!conceptIds.isEmpty()) { expressionBuilder.append(" AND p.gender_concept_id IN ("); for (int i = 0; i < conceptIds.size(); i++) { @@ -325,7 +327,7 @@ private List sampleElements(SampleParametersDTO sampleParametersD } expressionBuilder.append(')'); } - // else: all genders are selected + // else: all genders are selected, no where statement needed } String[] parameterKeys = new String[] { "results_schema", "CDM_schema", "expression"}; @@ -395,6 +397,7 @@ public SampleElement mapRow(ResultSet rs, int rowNum) throws SQLException { sample.setPersonId(rs.getInt("person_id")); sample.setGenderConceptId(rs.getInt("gender_concept_id")); sample.setAge(rs.getInt("age")); + sample.setRecordCount(rs.getInt("record_count")); return sample; } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java b/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java index 07196c8ee0..c022e1ae71 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java @@ -12,6 +12,8 @@ public class SampleElement { private int age; + private Integer recordCount; + public int getSampleId() { return sampleId; } @@ -51,4 +53,12 @@ public int getAge() { public void setAge(int age) { this.age = age; } + + public Integer getRecordCount() { + return recordCount; + } + + public void setRecordCount(Integer recordCount) { + this.recordCount = recordCount; + } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java index 59f9ee2bb6..aea5872196 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java @@ -30,6 +30,8 @@ public class SampleElementDTO { */ private int age; + private Integer recordCount; + public Integer getSampleId() { return sampleId; } @@ -69,4 +71,12 @@ public int getAge() { public void setAge(int age) { this.age = age; } + + public Integer getRecordCount() { + return recordCount; + } + + public void setRecordCount(Integer recordCount) { + this.recordCount = recordCount; + } } diff --git a/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql index 4a9d0005f6..fe8753aabb 100644 --- a/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql +++ b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql @@ -1,4 +1,43 @@ -SELECT * +SELECT *, + ((select COUNT(*) + from @CDM_schema.drug_exposure c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.condition_occurrence c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.condition_era c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.observation c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.visit_occurrence c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.death c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.measurement c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.device_exposure c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.procedure_occurrence c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.specimen c + where c.person_id = s.person_id)) as record_count FROM @results_schema.cohort_sample_element s WHERE s.cohort_sample_id = @cohortSampleId ORDER BY s.rank From dafbb219f4f4a214f3c1cfd125fc1b16c2af7dd4 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 20 Jan 2020 14:56:17 +0100 Subject: [PATCH 13/33] Make sample record counts easier to turn off --- .../cohortsample/CohortSamplingService.java | 20 ++++++--- .../webapi/service/CohortSampleService.java | 2 +- .../sql/findElementsByCohortSampleId.sql | 41 +---------------- ...findElementsByCohortSampleIdWithCounts.sql | 44 +++++++++++++++++++ 4 files changed, 61 insertions(+), 46 deletions(-) create mode 100644 src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index 479ebe8155..50aa80c3e7 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -53,14 +53,24 @@ public CohortSamplingService( * Find all sample elements of a sample. * @param source Source to use * @param cohortSampleId sample ID of the elements. + * @param withRecordCounts whether to return record counts. This makes the query much slower. * @return list of elements. */ - public List findSampleElements(Source source, int cohortSampleId) { + public List findSampleElements(Source source, int cohortSampleId, boolean withRecordCounts) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); - PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleId.sql", - new String[] {"results_schema", "CDM_schema"}, - new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results), source.getTableQualifier(SourceDaimon.DaimonType.CDM) }, - "cohortSampleId", cohortSampleId); + PreparedStatementRenderer renderer; + + if (withRecordCounts) { + renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql", + new String[]{"results_schema", "CDM_schema"}, + new String[]{source.getTableQualifier(SourceDaimon.DaimonType.Results), source.getTableQualifier(SourceDaimon.DaimonType.CDM)}, + "cohortSampleId", cohortSampleId); + } else { + renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleId.sql", + "results_schema", + source.getTableQualifier(SourceDaimon.DaimonType.Results), + "cohortSampleId", cohortSampleId); + } return jdbcTemplate.query(renderer.getSql(), renderer.getOrderedParams(), elementRowMapper); } diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index c72293689c..d2e8774761 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -69,7 +69,7 @@ public CohortSampleDTO getCohortSample( throw new NotFoundException("Cohort sample with ID " + sampleId + " not found"); } Source source = sourceRepository.findBySourceId(sample.getSourceId()); - List elements = this.samplingService.findSampleElements(source, sampleId); + List elements = this.samplingService.findSampleElements(source, sampleId, true); return samplingService.sampleToSampleDTO(sample, elements); } diff --git a/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql index fe8753aabb..4a9d0005f6 100644 --- a/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql +++ b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql @@ -1,43 +1,4 @@ -SELECT *, - ((select COUNT(*) - from @CDM_schema.drug_exposure c - where c.person_id = s.person_id) - + - (select COUNT(*) - from @CDM_schema.condition_occurrence c - where c.person_id = s.person_id) - + - (select COUNT(*) - from @CDM_schema.condition_era c - where c.person_id = s.person_id) - + - (select COUNT(*) - from @CDM_schema.observation c - where c.person_id = s.person_id) - + - (select COUNT(*) - from @CDM_schema.visit_occurrence c - where c.person_id = s.person_id) - + - (select COUNT(*) - from @CDM_schema.death c - where c.person_id = s.person_id) - + - (select COUNT(*) - from @CDM_schema.measurement c - where c.person_id = s.person_id) - + - (select COUNT(*) - from @CDM_schema.device_exposure c - where c.person_id = s.person_id) - + - (select COUNT(*) - from @CDM_schema.procedure_occurrence c - where c.person_id = s.person_id) - + - (select COUNT(*) - from @CDM_schema.specimen c - where c.person_id = s.person_id)) as record_count +SELECT * FROM @results_schema.cohort_sample_element s WHERE s.cohort_sample_id = @cohortSampleId ORDER BY s.rank diff --git a/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql new file mode 100644 index 0000000000..fe8753aabb --- /dev/null +++ b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql @@ -0,0 +1,44 @@ +SELECT *, + ((select COUNT(*) + from @CDM_schema.drug_exposure c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.condition_occurrence c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.condition_era c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.observation c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.visit_occurrence c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.death c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.measurement c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.device_exposure c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.procedure_occurrence c + where c.person_id = s.person_id) + + + (select COUNT(*) + from @CDM_schema.specimen c + where c.person_id = s.person_id)) as record_count +FROM @results_schema.cohort_sample_element s +WHERE s.cohort_sample_id = @cohortSampleId +ORDER BY s.rank +; \ No newline at end of file From 83fd26cbed496e0b708f8952275ab47f70b536c1 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 21 Jan 2020 09:09:30 +0100 Subject: [PATCH 14/33] Make record counts optional --- .../org/ohdsi/webapi/service/CohortSampleService.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index d2e8774761..1367c2c283 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -14,14 +14,17 @@ import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.NotFoundException; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -62,14 +65,17 @@ public List listCohortSamples( @Path("/{sampleId}") @GET public CohortSampleDTO getCohortSample( - @PathParam("sampleId") Integer sampleId + @PathParam("sampleId") Integer sampleId, + @DefaultValue("") @QueryParam("fields") String fields ) { CohortSample sample = sampleRepository.findOne(sampleId); if (sample == null) { throw new NotFoundException("Cohort sample with ID " + sampleId + " not found"); } + List returnFields = Arrays.asList(fields.split(",")); + boolean withRecordCounts = returnFields.contains("recordCount"); Source source = sourceRepository.findBySourceId(sample.getSourceId()); - List elements = this.samplingService.findSampleElements(source, sampleId, true); + List elements = this.samplingService.findSampleElements(source, sampleId, withRecordCounts); return samplingService.sampleToSampleDTO(sample, elements); } From 76ef810e53b5ed22536d4ce7e6a3e807f0edd26b Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 21 Jan 2020 11:48:14 +0100 Subject: [PATCH 15/33] Delete samples on cohort redefinition --- .../cohortsample/CohortSamplingService.java | 42 +++++++++++++++++-- .../service/CohortDefinitionService.java | 6 +++ .../webapi/service/CohortSampleService.java | 16 ++++++- .../sql/deleteSampleElementsById.sql | 2 +- 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index 50aa80c3e7..daacfa9c74 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -22,6 +22,8 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.Date; import java.util.LinkedHashMap; import java.util.List; @@ -39,7 +41,6 @@ public class CohortSamplingService extends AbstractDaoService { private final TransactionTemplate transactionTemplate; private final CohortSampleRepository sampleRepository; - private final CohortSampleElementRowMapper elementRowMapper = new CohortSampleElementRowMapper(); @Autowired public CohortSamplingService( @@ -59,19 +60,22 @@ public CohortSamplingService( public List findSampleElements(Source source, int cohortSampleId, boolean withRecordCounts) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); PreparedStatementRenderer renderer; + Collection optionalFields; if (withRecordCounts) { renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql", new String[]{"results_schema", "CDM_schema"}, new String[]{source.getTableQualifier(SourceDaimon.DaimonType.Results), source.getTableQualifier(SourceDaimon.DaimonType.CDM)}, "cohortSampleId", cohortSampleId); + optionalFields = Collections.singleton("record_count"); } else { renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleId.sql", "results_schema", source.getTableQualifier(SourceDaimon.DaimonType.Results), "cohortSampleId", cohortSampleId); + optionalFields = Collections.emptySet(); } - return jdbcTemplate.query(renderer.getSql(), renderer.getOrderedParams(), elementRowMapper); + return jdbcTemplate.query(renderer.getSql(), renderer.getOrderedParams(), new CohortSampleElementRowMapper(optionalFields)); } /** @@ -397,8 +401,38 @@ public void deleteSample(int cohortDefinitionId, Source source, int sampleId) { }); } + public void deleteSamples(int cohortDefinitionId, Source source) { + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + String resultsSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); + + transactionTemplate.execute(transactionStatus -> { + List samples = sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, source.getId()); + sampleRepository.delete(samples); + String cohortSampleId = samples.stream() + .map(s -> s.getId().toString()) + .collect(Collectors.joining(",")) + + String sql = new PreparedStatementRenderer( + source, + "/resources/cohortsample/sql/deleteSampleElementsById.sql", + "results_schema", + resultsSchema, + "cohortSampleId", + cohortSampleId).getSql(); + + jdbcTemplate.update(sql, cohortSampleId); + return null; + }); + } + /** Maps a SQL result to a sample element. */ private static class CohortSampleElementRowMapper implements RowMapper { + private final Collection optionalFields; + + CohortSampleElementRowMapper(Collection optionalFields) { + this.optionalFields = optionalFields; + } + @Override public SampleElement mapRow(ResultSet rs, int rowNum) throws SQLException { SampleElement sample = new SampleElement(); @@ -407,7 +441,9 @@ public SampleElement mapRow(ResultSet rs, int rowNum) throws SQLException { sample.setPersonId(rs.getInt("person_id")); sample.setGenderConceptId(rs.getInt("gender_concept_id")); sample.setAge(rs.getInt("age")); - sample.setRecordCount(rs.getInt("record_count")); + if (optionalFields.contains("record_count")) { + sample.setRecordCount(rs.getInt("record_count")); + } return sample; } } diff --git a/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java b/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java index 45e206a408..5637221bfd 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java @@ -20,6 +20,7 @@ import org.ohdsi.webapi.cohortdefinition.*; import org.ohdsi.webapi.cohortdefinition.dto.CohortDTO; import org.ohdsi.webapi.cohortdefinition.dto.CohortMetadataDTO; +import org.ohdsi.webapi.cohortsample.CohortSamplingService; import org.ohdsi.webapi.common.SourceMapKey; import org.ohdsi.webapi.common.generation.GenerateSqlResult; import org.ohdsi.webapi.common.sensitiveinfo.CohortGenerationSensitiveInfoService; @@ -136,6 +137,9 @@ public class CohortDefinitionService extends AbstractDaoService { @Autowired private ObjectMapper objectMapper; + @Autowired + private CohortSamplingService samplingService; + @PersistenceContext protected EntityManager entityManager; @@ -435,6 +439,8 @@ public CohortDTO saveCohortDefinition(@PathParam("id") final int id, CohortDTO d currentDefinition.setModifiedBy(modifier); currentDefinition.setModifiedDate(currentTime); + sourceService.getSources().forEach(source -> this.samplingService.deleteSamples(id, source)); + this.cohortDefinitionRepository.save(currentDefinition); return getCohortDefinition(id); } diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index 1367c2c283..fdef0085cb 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -66,7 +66,7 @@ public List listCohortSamples( @GET public CohortSampleDTO getCohortSample( @PathParam("sampleId") Integer sampleId, - @DefaultValue("") @QueryParam("fields") String fields + @DefaultValue("recordCount") @QueryParam("fields") String fields ) { CohortSample sample = sampleRepository.findOne(sampleId); if (sample == null) { @@ -110,6 +110,20 @@ public Response deleteCohortSample( return Response.status(Response.Status.NO_CONTENT).build(); } + @Path("/") + @DELETE + public Response deleteCohortSamples( + @PathParam("sourceKey") String sourceKey, + @PathParam("cohortDefinitionId") int cohortDefinitionId + ) { + Source source = getSource(sourceKey); + if (cohortDefinitionRepository.findOne(cohortDefinitionId) == null) { + throw new NotFoundException("Cohort definition " + cohortDefinitionId + " does not exist."); + } + samplingService.deleteSamples(cohortDefinitionId, source); + return Response.status(Response.Status.NO_CONTENT).build(); + } + private Source getSource(String sourceKey) { Source source = sourceRepository.findBySourceKey(sourceKey); if (source == null) { diff --git a/src/main/resources/resources/cohortsample/sql/deleteSampleElementsById.sql b/src/main/resources/resources/cohortsample/sql/deleteSampleElementsById.sql index 1331e60131..7b7f981a2a 100644 --- a/src/main/resources/resources/cohortsample/sql/deleteSampleElementsById.sql +++ b/src/main/resources/resources/cohortsample/sql/deleteSampleElementsById.sql @@ -1,3 +1,3 @@ DELETE FROM @results_schema.cohort_sample_element s -WHERE s.cohort_sample_id = @cohortSampleId +WHERE s.cohort_sample_id IN ( @cohortSampleId ) ; \ No newline at end of file From ccb005b6c49f652d0fd2bc7248f7b2f35f55dc0e Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 21 Jan 2020 13:26:00 +0100 Subject: [PATCH 16/33] Run sample deletion in tasklet --- .../CleanupCohortSamplesTasklet.java | 141 ++++++++++++++++++ .../cohortsample/CohortSamplingService.java | 52 +++---- .../service/CohortDefinitionService.java | 14 +- .../webapi/service/CohortSampleService.java | 4 +- 4 files changed, 180 insertions(+), 31 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/cohortsample/CleanupCohortSamplesTasklet.java diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CleanupCohortSamplesTasklet.java b/src/main/java/org/ohdsi/webapi/cohortsample/CleanupCohortSamplesTasklet.java new file mode 100644 index 0000000000..861f0d8c56 --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CleanupCohortSamplesTasklet.java @@ -0,0 +1,141 @@ +package org.ohdsi.webapi.cohortsample; + +import org.ohdsi.webapi.cohortdefinition.CleanupCohortTasklet; +import org.ohdsi.webapi.job.JobExecutionResource; +import org.ohdsi.webapi.job.JobTemplate; +import org.ohdsi.webapi.source.Source; +import org.ohdsi.webapi.source.SourceDaimon; +import org.ohdsi.webapi.source.SourceRepository; +import org.ohdsi.webapi.util.PreparedStatementRenderer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.StepContribution; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; +import org.springframework.batch.core.job.builder.SimpleJobBuilder; +import org.springframework.batch.core.scope.context.ChunkContext; +import org.springframework.batch.core.step.tasklet.Tasklet; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.transaction.support.TransactionTemplate; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.ohdsi.webapi.Constants.Params.COHORT_DEFINITION_ID; +import static org.ohdsi.webapi.Constants.Params.JOB_NAME; +import static org.ohdsi.webapi.Constants.Params.SOURCE_ID; + +public class CleanupCohortSamplesTasklet implements Tasklet { + private static final Logger log = LoggerFactory.getLogger(CleanupCohortTasklet.class); + + private final TransactionTemplate transactionTemplate; + private final SourceRepository sourceRepository; + private final CohortSamplingService samplingService; + private final CohortSampleRepository sampleRepository; + + public CleanupCohortSamplesTasklet( + final TransactionTemplate transactionTemplate, + final SourceRepository sourceRepository, + CohortSamplingService samplingService, + CohortSampleRepository sampleRepository + ) { + this.transactionTemplate = transactionTemplate; + this.sourceRepository = sourceRepository; + this.samplingService = samplingService; + this.sampleRepository = sampleRepository; + } + + private Integer doTask(ChunkContext chunkContext) { + Map jobParams = chunkContext.getStepContext().getJobParameters(); + int cohortDefinitionId = Integer.parseInt(jobParams.get(COHORT_DEFINITION_ID).toString()); + + if (jobParams.containsKey(SOURCE_ID)) { + int sourceId = Integer.parseInt(jobParams.get(SOURCE_ID).toString()); + Source source = this.sourceRepository.findOne(sourceId); + if (source != null) { + return mapSource(source, cohortDefinitionId); + } else { + return 0; + } + } else { + return this.sourceRepository.findAll().stream() + .filter(source-> source.getDaimons() + .stream() + .anyMatch(daimon -> daimon.getDaimonType() == SourceDaimon.DaimonType.Results)) + .mapToInt(source -> mapSource(source, cohortDefinitionId)) + .sum(); + } + } + + private int mapSource(Source source, int cohortDefinitionId) { + try { + String resultSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); + transactionTemplate.execute(transactionStatus -> { + List samples = sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, source.getId()); + sampleRepository.delete(samples); + String cohortSampleId = samples.stream() + .map(s -> s.getId().toString()) + .collect(Collectors.joining(",")); + + String sql = new PreparedStatementRenderer( + source, + "/resources/cohortsample/sql/deleteSampleElementsById.sql", + "results_schema", + resultSchema, + "cohortSampleId", + cohortSampleId).getSql(); + + samplingService.getSourceJdbcTemplate(source).update(sql, cohortSampleId); + return null; + }); + return 1; + } catch (Exception e) { + log.error("Error deleting samples for cohort: {}, cause: {}", cohortDefinitionId, e.getMessage()); + return 0; + } + } + + @Override + public RepeatStatus execute(final StepContribution contribution, final ChunkContext chunkContext) throws Exception { + this.transactionTemplate.execute(status -> doTask(chunkContext)); + + return RepeatStatus.FINISHED; + } + + public JobExecutionResource launch(JobBuilderFactory jobBuilders, StepBuilderFactory stepBuilders, JobTemplate jobTemplate, int cohortDefinitionId) { + JobParametersBuilder builder = new JobParametersBuilder(); + builder.addString(JOB_NAME, String.format("Cleanup cohort samples of cohort definition %d.", cohortDefinitionId)); + builder.addString(COHORT_DEFINITION_ID, String.valueOf(cohortDefinitionId)); + + log.info("Beginning cohort cleanup for cohort definition id: {}", cohortDefinitionId); + return launch(jobBuilders, stepBuilders, jobTemplate, builder.toJobParameters()); + } + + public JobExecutionResource launch(JobBuilderFactory jobBuilders, StepBuilderFactory stepBuilders, JobTemplate jobTemplate, int cohortDefinitionId, int sourceId) { + JobParametersBuilder builder = new JobParametersBuilder(); + builder.addString(JOB_NAME, String.format("Cleanup cohort samples of cohort definition %d.", cohortDefinitionId)); + builder.addString(COHORT_DEFINITION_ID, String.valueOf(cohortDefinitionId)); + builder.addString(SOURCE_ID, String.valueOf(sourceId)); + + log.info("Beginning cohort cleanup for cohort definition id {} and source ID {}", cohortDefinitionId, sourceId); + return launch(jobBuilders, stepBuilders, jobTemplate, builder.toJobParameters()); + } + + private JobExecutionResource launch(JobBuilderFactory jobBuilders, StepBuilderFactory stepBuilders, JobTemplate jobTemplate, JobParameters jobParameters) { + Step cleanupStep = stepBuilders.get("cohortSample.cleanupSamples") + .tasklet(this) + .build(); + + SimpleJobBuilder cleanupJobBuilder = jobBuilders.get("cleanupSamples") + .start(cleanupStep); + + Job cleanupCohortJob = cleanupJobBuilder.build(); + + return jobTemplate.launch(cleanupCohortJob, jobParameters); + } +} diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index daacfa9c74..9501b4d0e7 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -3,18 +3,20 @@ import org.ohdsi.webapi.cohortsample.dto.CohortSampleDTO; import org.ohdsi.webapi.cohortsample.dto.SampleElementDTO; import org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO; +import org.ohdsi.webapi.job.JobTemplate; import org.ohdsi.webapi.service.AbstractDaoService; import org.ohdsi.webapi.shiro.Entities.UserEntity; import org.ohdsi.webapi.source.Source; import org.ohdsi.webapi.source.SourceDaimon; import org.ohdsi.webapi.user.dto.UserDTO; import org.ohdsi.webapi.util.PreparedStatementRenderer; +import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; +import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Component; import org.springframework.transaction.support.TransactionCallback; -import org.springframework.transaction.support.TransactionTemplate; import javax.ws.rs.BadRequestException; import javax.ws.rs.NotFoundException; @@ -39,15 +41,21 @@ */ @Component public class CohortSamplingService extends AbstractDaoService { - private final TransactionTemplate transactionTemplate; private final CohortSampleRepository sampleRepository; + private final JobBuilderFactory jobBuilders; + private final StepBuilderFactory stepBuilders; + private final JobTemplate jobTemplate; @Autowired public CohortSamplingService( - TransactionTemplate transactionTemplate, - CohortSampleRepository sampleRepository) { - this.transactionTemplate = transactionTemplate; + CohortSampleRepository sampleRepository, + JobBuilderFactory jobBuilders, + StepBuilderFactory stepBuilders, + JobTemplate jobTemplate) { this.sampleRepository = sampleRepository; + this.jobBuilders = jobBuilders; + this.stepBuilders = stepBuilders; + this.jobTemplate = jobTemplate; } /** @@ -145,7 +153,7 @@ public CohortSampleDTO createSample(Source source, int cohortDefinitionId, Sampl sample.setSize(elements.size()); } - transactionTemplate.execute((TransactionCallback) transactionStatus -> { + getTransactionTemplate().execute((TransactionCallback) transactionStatus -> { log.debug("Saving {} sample elements for cohort {}", sample.getSize(), cohortDefinitionId); CohortSample updatedSample = sampleRepository.save(sample); insertSampledElements(source, jdbcTemplate, updatedSample.getId(), elements); @@ -394,35 +402,27 @@ public void deleteSample(int cohortDefinitionId, Source source, int sampleId) { throw new BadRequestException("Source " + sample.getSourceId() + " does not match provided source " + source.getId()); } - transactionTemplate.execute((TransactionCallback) transactionStatus -> { + getTransactionTemplate().execute((TransactionCallback) transactionStatus -> { sampleRepository.delete(sampleId); jdbcTemplate.update(sql, sampleId); return null; }); } - public void deleteSamples(int cohortDefinitionId, Source source) { - JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); - String resultsSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); + public void launchDeleteSamplesTasklet(int cohortDefinitionId) { + CleanupCohortSamplesTasklet tasklet = createDeleteSamplesTasklet(); - transactionTemplate.execute(transactionStatus -> { - List samples = sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, source.getId()); - sampleRepository.delete(samples); - String cohortSampleId = samples.stream() - .map(s -> s.getId().toString()) - .collect(Collectors.joining(",")) + tasklet.launch(jobBuilders, stepBuilders, jobTemplate, cohortDefinitionId); + } - String sql = new PreparedStatementRenderer( - source, - "/resources/cohortsample/sql/deleteSampleElementsById.sql", - "results_schema", - resultsSchema, - "cohortSampleId", - cohortSampleId).getSql(); + public void launchDeleteSamplesTasklet(int cohortDefinitionId, int sourceId) { + CleanupCohortSamplesTasklet tasklet = createDeleteSamplesTasklet(); - jdbcTemplate.update(sql, cohortSampleId); - return null; - }); + tasklet.launch(jobBuilders, stepBuilders, jobTemplate, cohortDefinitionId, sourceId); + } + + public CleanupCohortSamplesTasklet createDeleteSamplesTasklet() { + return new CleanupCohortSamplesTasklet(getTransactionTemplate(), getSourceRepository(), this, sampleRepository); } /** Maps a SQL result to a sample element. */ diff --git a/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java b/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java index 5637221bfd..e7cc109898 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java @@ -20,6 +20,7 @@ import org.ohdsi.webapi.cohortdefinition.*; import org.ohdsi.webapi.cohortdefinition.dto.CohortDTO; import org.ohdsi.webapi.cohortdefinition.dto.CohortMetadataDTO; +import org.ohdsi.webapi.cohortsample.CleanupCohortSamplesTasklet; import org.ohdsi.webapi.cohortsample.CohortSamplingService; import org.ohdsi.webapi.common.SourceMapKey; import org.ohdsi.webapi.common.generation.GenerateSqlResult; @@ -439,7 +440,7 @@ public CohortDTO saveCohortDefinition(@PathParam("id") final int id, CohortDTO d currentDefinition.setModifiedBy(modifier); currentDefinition.setModifiedDate(currentTime); - sourceService.getSources().forEach(source -> this.samplingService.deleteSamples(id, source)); + this.samplingService.launchDeleteSamplesTasklet(id); this.cohortDefinitionRepository.save(currentDefinition); return getCohortDefinition(id); @@ -563,6 +564,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { }); }); cohortDefinitionRepository.delete(def); + samplingService.launchDeleteSamplesTasklet(id); } else { log.warn("Failed to delete Cohort Definition with ID = {}", id); } @@ -583,8 +585,14 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { .tasklet(cleanupTasklet) .build(); - SimpleJobBuilder cleanupJobBuilder = jobBuilders.get("cleanupCohort") - .start(cleanupStep); + CleanupCohortSamplesTasklet cleanupSamplesTasklet = samplingService.createDeleteSamplesTasklet(); + Step cleanupSamplesStep = stepBuilders.get("cohortDefinition.cleanupSamples") + .tasklet(cleanupSamplesTasklet) + .build(); + + SimpleJobBuilder cleanupJobBuilder = jobBuilders.get("cleanupCohort") + .start(cleanupStep) + .next(cleanupSamplesStep); Job cleanupCohortJob = cleanupJobBuilder.build(); diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index fdef0085cb..0b3c939541 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -120,8 +120,8 @@ public Response deleteCohortSamples( if (cohortDefinitionRepository.findOne(cohortDefinitionId) == null) { throw new NotFoundException("Cohort definition " + cohortDefinitionId + " does not exist."); } - samplingService.deleteSamples(cohortDefinitionId, source); - return Response.status(Response.Status.NO_CONTENT).build(); + samplingService.launchDeleteSamplesTasklet(cohortDefinitionId, source.getId()); + return Response.status(Response.Status.ACCEPTED).build(); } private Source getSource(String sourceKey) { From f83d408d1a6bf5d5d7d77f8c278a9a9ab25e8467 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Tue, 21 Jan 2020 14:45:19 +0100 Subject: [PATCH 17/33] Don't run removal if no cohort samples are present --- .../CleanupCohortSamplesTasklet.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CleanupCohortSamplesTasklet.java b/src/main/java/org/ohdsi/webapi/cohortsample/CleanupCohortSamplesTasklet.java index 861f0d8c56..2746ec16f9 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CleanupCohortSamplesTasklet.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CleanupCohortSamplesTasklet.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.ohdsi.webapi.Constants.Params.COHORT_DEFINITION_ID; import static org.ohdsi.webapi.Constants.Params.JOB_NAME; @@ -75,25 +74,30 @@ private Integer doTask(ChunkContext chunkContext) { private int mapSource(Source source, int cohortDefinitionId) { try { String resultSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); - transactionTemplate.execute(transactionStatus -> { + return transactionTemplate.execute(transactionStatus -> { List samples = sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, source.getId()); + if (samples.isEmpty()) { + return 0; + } + sampleRepository.delete(samples); - String cohortSampleId = samples.stream() - .map(s -> s.getId().toString()) - .collect(Collectors.joining(",")); - String sql = new PreparedStatementRenderer( + int[] cohortSampleIds = samples.stream() + .mapToInt(CohortSample::getId) + .toArray(); + + PreparedStatementRenderer renderer = new PreparedStatementRenderer( source, "/resources/cohortsample/sql/deleteSampleElementsById.sql", "results_schema", resultSchema, "cohortSampleId", - cohortSampleId).getSql(); + cohortSampleIds); - samplingService.getSourceJdbcTemplate(source).update(sql, cohortSampleId); - return null; + samplingService.getSourceJdbcTemplate(source) + .update(renderer.getSql(), renderer.getOrderedParams()); + return cohortSampleIds.length; }); - return 1; } catch (Exception e) { log.error("Error deleting samples for cohort: {}, cause: {}", cohortDefinitionId, e.getMessage()); return 0; From 0b65c778dbeb15456ae4b67e0437a0c5dc5da2d9 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 23 Jan 2020 17:00:18 +0100 Subject: [PATCH 18/33] Fixes with authorization and SQL server --- .gitignore | 1 + pom.xml | 7 +++++++ .../cohortsample/CohortSampleRepository.java | 8 ++++++- .../cohortsample/CohortSamplingService.java | 21 +++++++++++++++++-- .../model/CohortSamplePermissionSchema.java | 21 +++++++++++++++++++ .../webapi/security/model/EntityType.java | 4 +++- .../webapi/service/AbstractDaoService.java | 4 +++- .../webapi/service/CohortSampleService.java | 15 ++----------- src/main/resources/application.properties | 10 ++++++--- ...0.20200109100200__cohort_sample_tables.sql | 8 +++---- ...3000__insert_cohort_sample_permissions.sql | 17 +++++++++++++++ .../ddl/results/cohort_sample_element.sql | 2 +- .../sql/deleteSampleElementsById.sql | 4 ++-- 13 files changed, 94 insertions(+), 28 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/security/model/CohortSamplePermissionSchema.java create mode 100644 src/main/resources/db/migration/sqlserver/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql diff --git a/.gitignore b/.gitignore index a98205f8c3..e613dc3e0c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +WebAPIConfig/ *application.properties .idea/ .metadata/ diff --git a/pom.xml b/pom.xml index b1df20557c..27fe0e1950 100644 --- a/pom.xml +++ b/pom.xml @@ -173,6 +173,8 @@ info info info + info + info warn 10 @@ -811,6 +813,11 @@ 8.3.1 jar + + log4j + log4j + 1.2.17 + diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java index eca3f3de3d..afd0b52992 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java @@ -1,6 +1,8 @@ package org.ohdsi.webapi.cohortsample; +import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Component; import java.util.List; @@ -10,5 +12,9 @@ */ @Component public interface CohortSampleRepository extends CrudRepository { - List findByCohortDefinitionIdAndSourceId(int cohortDefinitionId, int sourceId); + @Query("SELECT c FROM CohortSample c LEFT JOIN FETCH c.createdBy WHERE c.id = :id") + CohortSample findById(@Param("id") int cohortSampleId); + + @Query("SELECT c FROM CohortSample c LEFT JOIN FETCH c.createdBy WHERE c.cohortDefinitionId = :cohortDefinitionId AND c.sourceId = :sourceId") + List findByCohortDefinitionIdAndSourceId(@Param("cohortDefinitionId") int cohortDefinitionId, @Param("sourceId") int sourceId); } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index 9501b4d0e7..ea0189b34a 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -58,6 +58,23 @@ public CohortSamplingService( this.jobTemplate = jobTemplate; } + public List listSamples(int cohortDefinitionId, int sourceId) { + return sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, sourceId).stream() + .map(sample -> sampleToSampleDTO(sample, null)) + .collect(Collectors.toList()); + } + + + public CohortSampleDTO getSample(int sampleId, boolean withRecordCounts) { + CohortSample sample = sampleRepository.findById(sampleId); + if (sample == null) { + throw new NotFoundException("Cohort sample with ID " + sampleId + " not found"); + } + Source source = getSourceRepository().findBySourceId(sample.getSourceId()); + List sampleElements = findSampleElements(source, sample.getId(), withRecordCounts); + return sampleToSampleDTO(sample, sampleElements); + } + /** * Find all sample elements of a sample. * @param source Source to use @@ -65,7 +82,7 @@ public CohortSamplingService( * @param withRecordCounts whether to return record counts. This makes the query much slower. * @return list of elements. */ - public List findSampleElements(Source source, int cohortSampleId, boolean withRecordCounts) { + private List findSampleElements(Source source, int cohortSampleId, boolean withRecordCounts) { JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); PreparedStatementRenderer renderer; Collection optionalFields; @@ -165,7 +182,7 @@ public CohortSampleDTO createSample(Source source, int cohortDefinitionId, Sampl } /** Convert a given sample with given elements to a DTO. */ - public CohortSampleDTO sampleToSampleDTO(CohortSample sample, List elements) { + private CohortSampleDTO sampleToSampleDTO(CohortSample sample, List elements) { CohortSampleDTO sampleDTO = new CohortSampleDTO(); sampleDTO.setId(sample.getId()); sampleDTO.setName(sample.getName()); diff --git a/src/main/java/org/ohdsi/webapi/security/model/CohortSamplePermissionSchema.java b/src/main/java/org/ohdsi/webapi/security/model/CohortSamplePermissionSchema.java new file mode 100644 index 0000000000..12923eadff --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/security/model/CohortSamplePermissionSchema.java @@ -0,0 +1,21 @@ +package org.ohdsi.webapi.security.model; + +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; + +@Component +public class CohortSamplePermissionSchema extends EntityPermissionSchema { + private static Map readPermissions = new HashMap() {{ + put("cohortsample:*:*:%s:get", "Get Cohort Sample with ID = %s"); + }}; + + private static Map writePermissions = new HashMap() {{ + put("cohortsample:*:*:%s:delete", "Delete Cohort Sample with ID = %s"); + }}; + + public CohortSamplePermissionSchema() { + super(EntityType.COHORT_SAMPLE, readPermissions, writePermissions); + } +} diff --git a/src/main/java/org/ohdsi/webapi/security/model/EntityType.java b/src/main/java/org/ohdsi/webapi/security/model/EntityType.java index b48c895992..a87c4bdd85 100644 --- a/src/main/java/org/ohdsi/webapi/security/model/EntityType.java +++ b/src/main/java/org/ohdsi/webapi/security/model/EntityType.java @@ -2,6 +2,7 @@ import org.ohdsi.webapi.cohortcharacterization.domain.CohortCharacterizationEntity; import org.ohdsi.webapi.cohortdefinition.CohortDefinition; +import org.ohdsi.webapi.cohortsample.CohortSample; import org.ohdsi.webapi.conceptset.ConceptSet; import org.ohdsi.webapi.estimation.Estimation; import org.ohdsi.webapi.feanalysis.domain.FeAnalysisEntity; @@ -20,7 +21,8 @@ public enum EntityType { INCIDENCE_RATE(IncidenceRateAnalysis.class), SOURCE(Source.class), ESTIMATION(Estimation.class), - PREDICTION(PredictionAnalysis.class); + PREDICTION(PredictionAnalysis.class), + COHORT_SAMPLE(CohortSample.class); private final Class entityClass; diff --git a/src/main/java/org/ohdsi/webapi/service/AbstractDaoService.java b/src/main/java/org/ohdsi/webapi/service/AbstractDaoService.java index 849c94a5d3..18c72ceda9 100644 --- a/src/main/java/org/ohdsi/webapi/service/AbstractDaoService.java +++ b/src/main/java/org/ohdsi/webapi/service/AbstractDaoService.java @@ -8,6 +8,8 @@ import org.apache.commons.lang3.StringUtils; import org.ohdsi.webapi.GenerationStatus; import org.ohdsi.webapi.IExecutionInfo; +import org.ohdsi.webapi.cohortsample.CohortSample; +import org.ohdsi.webapi.cohortsample.dto.CohortSampleDTO; import org.ohdsi.webapi.common.sensitiveinfo.AbstractAdminService; import org.ohdsi.webapi.conceptset.ConceptSetItemRepository; import org.ohdsi.webapi.conceptset.ConceptSetRepository; @@ -29,6 +31,7 @@ import org.springframework.jdbc.datasource.DriverManagerDataSource; import org.springframework.transaction.support.TransactionTemplate; +import javax.ws.rs.NotFoundException; import java.io.File; import java.io.IOException; import java.sql.ResultSet; @@ -291,5 +294,4 @@ protected UserEntity getCurrentUser() { protected String getCurrentUserLogin() { return security.getSubject(); } - } diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index 0b3c939541..cc29ff98ad 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -1,10 +1,8 @@ package org.ohdsi.webapi.service; import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; -import org.ohdsi.webapi.cohortsample.CohortSample; import org.ohdsi.webapi.cohortsample.CohortSampleRepository; import org.ohdsi.webapi.cohortsample.CohortSamplingService; -import org.ohdsi.webapi.cohortsample.SampleElement; import org.ohdsi.webapi.cohortsample.dto.CohortSampleDTO; import org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO; import org.ohdsi.webapi.source.Source; @@ -26,7 +24,6 @@ import javax.ws.rs.core.Response; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; @Path("/cohortsample/{cohortDefinitionId}/{sourceKey}") @Component @@ -57,9 +54,7 @@ public List listCohortSamples( @PathParam("sourceKey") String sourceKey ) { Source source = getSource(sourceKey); - return this.sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, source.getId()).stream() - .map(s -> samplingService.sampleToSampleDTO(s, null)) - .collect(Collectors.toList()); + return this.samplingService.listSamples(cohortDefinitionId, source.getId()); } @Path("/{sampleId}") @@ -68,15 +63,9 @@ public CohortSampleDTO getCohortSample( @PathParam("sampleId") Integer sampleId, @DefaultValue("recordCount") @QueryParam("fields") String fields ) { - CohortSample sample = sampleRepository.findOne(sampleId); - if (sample == null) { - throw new NotFoundException("Cohort sample with ID " + sampleId + " not found"); - } List returnFields = Arrays.asList(fields.split(",")); boolean withRecordCounts = returnFields.contains("recordCount"); - Source source = sourceRepository.findBySourceId(sample.getSourceId()); - List elements = this.samplingService.findSampleElements(source, sampleId, withRecordCounts); - return samplingService.sampleToSampleDTO(sample, elements); + return this.samplingService.getSample(sampleId, withRecordCounts); } @Path("/") diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 42b2be4ccd..ba48975867 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -4,9 +4,13 @@ build.number=NA spring.profiles.active=${spring.profiles.active} # Logging -logging.level.org.springframework.web: OFF -logging.level.org.hibernate: OFF -logging.level.org.apache.shiro=TRACE +logging.level.org.springframework.web=${logging.level.org.springframework.web} +logging.level.org.hibernate=${logging.level.org.hibernate} +logging.level.root=${logging.level.root} +logging.level.org.ohdsi=${logging.level.org.ohdsi} +logging.level.org.springframework.orm=${logging.level.org.springframework.orm} +logging.level.org.springframework.jdbc=${logging.level.org.springframework.jdbc} +logging.level.org.apache.shiro=${logging.level.org.apache.shiro} #Primary DataSource datasource.driverClassName=${datasource.driverClassName} diff --git a/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql b/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql index 69b5a729e6..927913b4da 100644 --- a/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql +++ b/src/main/resources/db/migration/sqlserver/V2.8.0.20200109100200__cohort_sample_tables.sql @@ -1,13 +1,13 @@ CREATE SEQUENCE ${ohdsiSchema}.cohort_sample_sequence; CREATE TABLE ${ohdsiSchema}.cohort_sample( - id NUMBER(19) PRIMARY KEY CLUSTERED NOT NULL, + id BIGINT PRIMARY KEY CLUSTERED NOT NULL, name VARCHAR(255) NOT NULL, cohort_definition_id INTEGER NOT NULL, source_id INTEGER NOT NULL, - size NUMBER(9) NOT NULL, - age_min NUMBER(4) NULL, - age_max NUMBER(4) NULL, + size INT NOT NULL, + age_min SMALLINT NULL, + age_max SMALLINT NULL, age_mode VARCHAR(24), gender_concept_ids VARCHAR(255) NULL, created_by_id INTEGER, diff --git a/src/main/resources/db/migration/sqlserver/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql b/src/main/resources/db/migration/sqlserver/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql new file mode 100644 index 0000000000..8ca257a1ca --- /dev/null +++ b/src/main/resources/db/migration/sqlserver/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql @@ -0,0 +1,17 @@ +INSERT INTO ${ohdsiSchema}.sec_permission(id, value, description) VALUES +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:get', 'List cohort samples'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:get', 'Get single cohort samples'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:delete', 'Delete cohort sample'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:delete', 'Delete all cohort samples of a cohort.'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:post', 'Create cohort sample'); + +INSERT INTO ${ohdsiSchema}.sec_role_permission(id, role_id, permission_id) +SELECT NEXT VALUE FOR ${ohdsiSchema}.sec_role_permission_sequence, sr.id, sp.id +FROM ${ohdsiSchema}.sec_permission SP, ${ohdsiSchema}.sec_role sr +WHERE sp.value IN ( + 'cohortsample:*:*:get', + 'cohortsample:*:*:*:get', + 'cohortsample:*:*:*:delete', + 'cohortsample:*:*:delete', + 'cohortsample:*:*:post' + ) AND sr.name IN ('Atlas users'); diff --git a/src/main/resources/ddl/results/cohort_sample_element.sql b/src/main/resources/ddl/results/cohort_sample_element.sql index 902f80e02e..93a96c2189 100644 --- a/src/main/resources/ddl/results/cohort_sample_element.sql +++ b/src/main/resources/ddl/results/cohort_sample_element.sql @@ -1,4 +1,4 @@ -IF OBJECT_ID('@results_schema.cohort_sample', 'U') IS NULL +IF OBJECT_ID('@results_schema.cohort_sample_element', 'U') IS NULL CREATE TABLE @results_schema.cohort_sample_element( cohort_sample_id int NOT NULL, rank int NOT NULL, diff --git a/src/main/resources/resources/cohortsample/sql/deleteSampleElementsById.sql b/src/main/resources/resources/cohortsample/sql/deleteSampleElementsById.sql index 7b7f981a2a..bf6e68560c 100644 --- a/src/main/resources/resources/cohortsample/sql/deleteSampleElementsById.sql +++ b/src/main/resources/resources/cohortsample/sql/deleteSampleElementsById.sql @@ -1,3 +1,3 @@ -DELETE FROM @results_schema.cohort_sample_element s -WHERE s.cohort_sample_id IN ( @cohortSampleId ) +DELETE FROM @results_schema.cohort_sample_element +WHERE cohort_sample_id IN ( @cohortSampleId ) ; \ No newline at end of file From 85c793b3677d63168c3cf96d5450011f09c09c37 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Thu, 23 Jan 2020 17:34:28 +0100 Subject: [PATCH 19/33] Check cohort generation status before generating sample --- .../cohortsample/CohortSamplingService.java | 14 +++--- .../cohortsample/dto/CohortSampleDTO.java | 12 ++--- .../cohortsample/dto/CohortSampleListDTO.java | 44 +++++++++++++++++++ .../webapi/service/CohortSampleService.java | 35 ++++++++++++--- ...3000__insert_cohort_sample_permissions.sql | 17 +++++++ ...3000__insert_cohort_sample_permissions.sql | 17 +++++++ .../ddl/results/cohort_sample_element.sql | 2 - .../resources/ddl/results/create_index.sql | 3 +- 8 files changed, 122 insertions(+), 22 deletions(-) create mode 100644 src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleListDTO.java create mode 100644 src/main/resources/db/migration/oracle/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql create mode 100644 src/main/resources/db/migration/postgresql/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index ea0189b34a..60a0bb8b57 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -60,7 +60,7 @@ public CohortSamplingService( public List listSamples(int cohortDefinitionId, int sourceId) { return sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, sourceId).stream() - .map(sample -> sampleToSampleDTO(sample, null)) + .map(sample -> sampleToSampleDTO(sample, null, false)) .collect(Collectors.toList()); } @@ -72,7 +72,7 @@ public CohortSampleDTO getSample(int sampleId, boolean withRecordCounts) { } Source source = getSourceRepository().findBySourceId(sample.getSourceId()); List sampleElements = findSampleElements(source, sample.getId(), withRecordCounts); - return sampleToSampleDTO(sample, sampleElements); + return sampleToSampleDTO(sample, sampleElements, true); } /** @@ -178,17 +178,19 @@ public CohortSampleDTO createSample(Source source, int cohortDefinitionId, Sampl return null; }); - return sampleToSampleDTO(sample, elements); + return sampleToSampleDTO(sample, elements, true); } /** Convert a given sample with given elements to a DTO. */ - private CohortSampleDTO sampleToSampleDTO(CohortSample sample, List elements) { + private CohortSampleDTO sampleToSampleDTO(CohortSample sample, List elements, boolean includeIds) { CohortSampleDTO sampleDTO = new CohortSampleDTO(); sampleDTO.setId(sample.getId()); sampleDTO.setName(sample.getName()); sampleDTO.setSize(sample.getSize()); - sampleDTO.setCohortDefinitionId(sample.getCohortDefinitionId()); - sampleDTO.setSourceId(sample.getSourceId()); + if (includeIds) { + sampleDTO.setCohortDefinitionId(sample.getCohortDefinitionId()); + sampleDTO.setSourceId(sample.getSourceId()); + } sampleDTO.setCreatedDate(sample.getCreatedDate()); UserEntity createdBy = sample.getCreatedBy(); if (createdBy != null) { diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java index 33f5dce8ff..fb9796b495 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java @@ -31,11 +31,11 @@ public class CohortSampleDTO { /** * Cohort definition ID that was sampled. */ - private int cohortDefinitionId; + private Integer cohortDefinitionId; /** * Source ID that was sampled. */ - private int sourceId; + private Integer sourceId; /** * Age criteria used to create the sample. @@ -76,19 +76,19 @@ public void setCreatedBy(UserDTO createdBy) { this.createdBy = createdBy; } - public int getCohortDefinitionId() { + public Integer getCohortDefinitionId() { return cohortDefinitionId; } - public void setCohortDefinitionId(int cohortDefinitionId) { + public void setCohortDefinitionId(Integer cohortDefinitionId) { this.cohortDefinitionId = cohortDefinitionId; } - public int getSourceId() { + public Integer getSourceId() { return sourceId; } - public void setSourceId(int sourceId) { + public void setSourceId(Integer sourceId) { this.sourceId = sourceId; } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleListDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleListDTO.java new file mode 100644 index 0000000000..2dc85f2a4d --- /dev/null +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleListDTO.java @@ -0,0 +1,44 @@ +package org.ohdsi.webapi.cohortsample.dto; + +import org.ohdsi.webapi.GenerationStatus; + +import java.util.List; + +public class CohortSampleListDTO { + private int cohortDefinitionId; + private int sourceId; + private GenerationStatus generationStatus; + private List samples; + + public int getCohortDefinitionId() { + return cohortDefinitionId; + } + + public void setCohortDefinitionId(int cohortDefinitionId) { + this.cohortDefinitionId = cohortDefinitionId; + } + + public int getSourceId() { + return sourceId; + } + + public void setSourceId(int sourceId) { + this.sourceId = sourceId; + } + + public GenerationStatus getGenerationStatus() { + return generationStatus; + } + + public void setGenerationStatus(GenerationStatus generationStatus) { + this.generationStatus = generationStatus; + } + + public List getSamples() { + return samples; + } + + public void setSamples(List samples) { + this.samples = samples; + } +} diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index cc29ff98ad..b644920a71 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -1,15 +1,20 @@ package org.ohdsi.webapi.service; +import org.ohdsi.webapi.GenerationStatus; import org.ohdsi.webapi.cohortdefinition.CohortDefinitionRepository; -import org.ohdsi.webapi.cohortsample.CohortSampleRepository; +import org.ohdsi.webapi.cohortdefinition.CohortGenerationInfo; +import org.ohdsi.webapi.cohortdefinition.CohortGenerationInfoId; +import org.ohdsi.webapi.cohortdefinition.CohortGenerationInfoRepository; import org.ohdsi.webapi.cohortsample.CohortSamplingService; import org.ohdsi.webapi.cohortsample.dto.CohortSampleDTO; +import org.ohdsi.webapi.cohortsample.dto.CohortSampleListDTO; import org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO; import org.ohdsi.webapi.source.Source; import org.ohdsi.webapi.source.SourceRepository; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import javax.ws.rs.BadRequestException; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.DefaultValue; @@ -30,31 +35,42 @@ @Produces(MediaType.APPLICATION_JSON) public class CohortSampleService { private final CohortDefinitionRepository cohortDefinitionRepository; - private final CohortSampleRepository sampleRepository; + private final CohortGenerationInfoRepository generationInfoRepository; private final CohortSamplingService samplingService; private final SourceRepository sourceRepository; @Autowired public CohortSampleService( - CohortSampleRepository sampleRepository, CohortSamplingService samplingService, SourceRepository sourceRepository, - CohortDefinitionRepository cohortDefinitionRepository + CohortDefinitionRepository cohortDefinitionRepository, + CohortGenerationInfoRepository generationInfoRepository ) { - this.sampleRepository = sampleRepository; this.samplingService = samplingService; this.sourceRepository = sourceRepository; this.cohortDefinitionRepository = cohortDefinitionRepository; + this.generationInfoRepository = generationInfoRepository; } @Path("/") @GET - public List listCohortSamples( + public CohortSampleListDTO listCohortSamples( @PathParam("cohortDefinitionId") int cohortDefinitionId, @PathParam("sourceKey") String sourceKey ) { Source source = getSource(sourceKey); - return this.samplingService.listSamples(cohortDefinitionId, source.getId()); + CohortSampleListDTO result = new CohortSampleListDTO(); + + result.setCohortDefinitionId(cohortDefinitionId); + result.setSourceId(source.getId()); + + CohortGenerationInfo generationInfo = generationInfoRepository.findOne( + new CohortGenerationInfoId(cohortDefinitionId, source.getId())); + result.setGenerationStatus(generationInfo != null ? generationInfo.getStatus() : null); + + result.setSamples(this.samplingService.listSamples(cohortDefinitionId, source.getId())); + + return result; } @Path("/{sampleId}") @@ -81,6 +97,11 @@ public CohortSampleDTO createCohortSample( if (cohortDefinitionRepository.findOne(cohortDefinitionId) == null) { throw new NotFoundException("Cohort definition " + cohortDefinitionId + " does not exist."); } + CohortGenerationInfo generationInfo = generationInfoRepository.findOne( + new CohortGenerationInfoId(cohortDefinitionId, source.getId())); + if (generationInfo == null || generationInfo.getStatus() != GenerationStatus.COMPLETE) { + throw new BadRequestException("Cohort is not yet generated"); + } return samplingService.createSample(source, cohortDefinitionId, sampleParameters); } diff --git a/src/main/resources/db/migration/oracle/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql b/src/main/resources/db/migration/oracle/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql new file mode 100644 index 0000000000..8ca257a1ca --- /dev/null +++ b/src/main/resources/db/migration/oracle/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql @@ -0,0 +1,17 @@ +INSERT INTO ${ohdsiSchema}.sec_permission(id, value, description) VALUES +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:get', 'List cohort samples'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:get', 'Get single cohort samples'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:delete', 'Delete cohort sample'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:delete', 'Delete all cohort samples of a cohort.'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:post', 'Create cohort sample'); + +INSERT INTO ${ohdsiSchema}.sec_role_permission(id, role_id, permission_id) +SELECT NEXT VALUE FOR ${ohdsiSchema}.sec_role_permission_sequence, sr.id, sp.id +FROM ${ohdsiSchema}.sec_permission SP, ${ohdsiSchema}.sec_role sr +WHERE sp.value IN ( + 'cohortsample:*:*:get', + 'cohortsample:*:*:*:get', + 'cohortsample:*:*:*:delete', + 'cohortsample:*:*:delete', + 'cohortsample:*:*:post' + ) AND sr.name IN ('Atlas users'); diff --git a/src/main/resources/db/migration/postgresql/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql b/src/main/resources/db/migration/postgresql/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql new file mode 100644 index 0000000000..8ca257a1ca --- /dev/null +++ b/src/main/resources/db/migration/postgresql/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql @@ -0,0 +1,17 @@ +INSERT INTO ${ohdsiSchema}.sec_permission(id, value, description) VALUES +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:get', 'List cohort samples'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:get', 'Get single cohort samples'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:delete', 'Delete cohort sample'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:delete', 'Delete all cohort samples of a cohort.'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:post', 'Create cohort sample'); + +INSERT INTO ${ohdsiSchema}.sec_role_permission(id, role_id, permission_id) +SELECT NEXT VALUE FOR ${ohdsiSchema}.sec_role_permission_sequence, sr.id, sp.id +FROM ${ohdsiSchema}.sec_permission SP, ${ohdsiSchema}.sec_role sr +WHERE sp.value IN ( + 'cohortsample:*:*:get', + 'cohortsample:*:*:*:get', + 'cohortsample:*:*:*:delete', + 'cohortsample:*:*:delete', + 'cohortsample:*:*:post' + ) AND sr.name IN ('Atlas users'); diff --git a/src/main/resources/ddl/results/cohort_sample_element.sql b/src/main/resources/ddl/results/cohort_sample_element.sql index 93a96c2189..a26f3ef202 100644 --- a/src/main/resources/ddl/results/cohort_sample_element.sql +++ b/src/main/resources/ddl/results/cohort_sample_element.sql @@ -6,5 +6,3 @@ CREATE TABLE @results_schema.cohort_sample_element( age int, gender_concept_id int ); - -CREATE INDEX idx_cohort_sample_element_rank ON @results_schema.cohort_sample_element (cohort_sample_id, rank); diff --git a/src/main/resources/ddl/results/create_index.sql b/src/main/resources/ddl/results/create_index.sql index 95e9217bb6..af58a14a40 100644 --- a/src/main/resources/ddl/results/create_index.sql +++ b/src/main/resources/ddl/results/create_index.sql @@ -12,4 +12,5 @@ CREATE INDEX HH_IDX_COHORT_ID_ANALYSIS_ID ON @results_schema.HERACLES_HEEL_RESUL CREATE INDEX idx_heracles_periods_startdate ON @results_schema.heracles_periods (period_start_date); CREATE INDEX idx_heracles_periods_end_date ON @results_schema.heracles_periods (period_end_date); -CREATE INDEX idx_concept_id_varchar ON @vocab_schema.concept (CAST(concept_id AS VARCHAR)); \ No newline at end of file +CREATE INDEX idx_concept_id_varchar ON @vocab_schema.concept (CAST(concept_id AS VARCHAR)); +CREATE INDEX idx_cohort_sample_element_rank ON @results_schema.cohort_sample_element (cohort_sample_id, rank); From e192d4595d384c849bb6be1ad4250edbc52a7cd4 Mon Sep 17 00:00:00 2001 From: Joris Borgdorff Date: Mon, 27 Jan 2020 11:51:54 +0100 Subject: [PATCH 20/33] Fix Oracle and PostgreSQL migration syntax --- ...00122173000__insert_cohort_sample_permissions.sql | 10 +++++----- ...00122173000__insert_cohort_sample_permissions.sql | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/resources/db/migration/oracle/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql b/src/main/resources/db/migration/oracle/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql index 8ca257a1ca..ab534ba256 100644 --- a/src/main/resources/db/migration/oracle/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql +++ b/src/main/resources/db/migration/oracle/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql @@ -1,9 +1,9 @@ INSERT INTO ${ohdsiSchema}.sec_permission(id, value, description) VALUES -(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:get', 'List cohort samples'), -(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:get', 'Get single cohort samples'), -(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:delete', 'Delete cohort sample'), -(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:delete', 'Delete all cohort samples of a cohort.'), -(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:post', 'Create cohort sample'); +(${ohdsiSchema}.sec_permission_id_seq.nextval, 'cohortsample:*:*:get', 'List cohort samples'), +(${ohdsiSchema}.sec_permission_id_seq.nextval, 'cohortsample:*:*:*:get', 'Get single cohort samples'), +(${ohdsiSchema}.sec_permission_id_seq.nextval, 'cohortsample:*:*:*:delete', 'Delete cohort sample'), +(${ohdsiSchema}.sec_permission_id_seq.nextval, 'cohortsample:*:*:delete', 'Delete all cohort samples of a cohort.'), +(${ohdsiSchema}.sec_permission_id_seq.nextval, 'cohortsample:*:*:post', 'Create cohort sample'); INSERT INTO ${ohdsiSchema}.sec_role_permission(id, role_id, permission_id) SELECT NEXT VALUE FOR ${ohdsiSchema}.sec_role_permission_sequence, sr.id, sp.id diff --git a/src/main/resources/db/migration/postgresql/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql b/src/main/resources/db/migration/postgresql/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql index 8ca257a1ca..aebead7c84 100644 --- a/src/main/resources/db/migration/postgresql/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql +++ b/src/main/resources/db/migration/postgresql/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql @@ -1,12 +1,12 @@ INSERT INTO ${ohdsiSchema}.sec_permission(id, value, description) VALUES -(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:get', 'List cohort samples'), -(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:get', 'Get single cohort samples'), -(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:delete', 'Delete cohort sample'), -(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:delete', 'Delete all cohort samples of a cohort.'), -(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:post', 'Create cohort sample'); +(nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cohortsample:*:*:get', 'List cohort samples'), +(nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cohortsample:*:*:*:get', 'Get single cohort samples'), +(nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cohortsample:*:*:*:delete', 'Delete cohort sample'), +(nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cohortsample:*:*:delete', 'Delete all cohort samples of a cohort.'), +(nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cohortsample:*:*:post', 'Create cohort sample'); INSERT INTO ${ohdsiSchema}.sec_role_permission(id, role_id, permission_id) -SELECT NEXT VALUE FOR ${ohdsiSchema}.sec_role_permission_sequence, sr.id, sp.id +SELECT nextval('${ohdsiSchema}.sec_role_permission_sequence'), sr.id, sp.id FROM ${ohdsiSchema}.sec_permission SP, ${ohdsiSchema}.sec_role sr WHERE sp.value IN ( 'cohortsample:*:*:get', From 60da4e06ee44b336058081edf382ec5cf060671a Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Mon, 10 Feb 2020 20:10:21 +0100 Subject: [PATCH 21/33] add endpoint to check that samples exist for a cohort --- .../cohortsample/CohortSampleRepository.java | 3 +++ .../cohortsample/CohortSamplingService.java | 5 +++- .../webapi/service/CohortSampleService.java | 24 ++++++++++++------- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java index afd0b52992..319477e584 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java @@ -17,4 +17,7 @@ public interface CohortSampleRepository extends CrudRepository findByCohortDefinitionIdAndSourceId(@Param("cohortDefinitionId") int cohortDefinitionId, @Param("sourceId") int sourceId); + + @Query("SELECT count(c.id) FROM CohortSample c WHERE c.cohortDefinitionId = :cohortDefinitionId") + int countSamples(@Param("cohortDefinitionId") int cohortDefinitionId); } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index 60a0bb8b57..e2bb8cc4d4 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -64,7 +64,6 @@ public List listSamples(int cohortDefinitionId, int sourceId) { .collect(Collectors.toList()); } - public CohortSampleDTO getSample(int sampleId, boolean withRecordCounts) { CohortSample sample = sampleRepository.findById(sampleId); if (sample == null) { @@ -75,6 +74,10 @@ public CohortSampleDTO getSample(int sampleId, boolean withRecordCounts) { return sampleToSampleDTO(sample, sampleElements, true); } + public int countSamples(int cohortDefinitionId) { + return sampleRepository.countSamples(cohortDefinitionId); + } + /** * Find all sample elements of a sample. * @param source Source to use diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index b644920a71..a47672e9db 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -27,10 +27,9 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import java.util.Arrays; -import java.util.List; +import java.util.*; -@Path("/cohortsample/{cohortDefinitionId}/{sourceKey}") +@Path("/cohortsample") @Component @Produces(MediaType.APPLICATION_JSON) public class CohortSampleService { @@ -52,7 +51,7 @@ public CohortSampleService( this.generationInfoRepository = generationInfoRepository; } - @Path("/") + @Path("/{cohortDefinitionId}/{sourceKey}") @GET public CohortSampleListDTO listCohortSamples( @PathParam("cohortDefinitionId") int cohortDefinitionId, @@ -73,7 +72,7 @@ public CohortSampleListDTO listCohortSamples( return result; } - @Path("/{sampleId}") + @Path("/{cohortDefinitionId}/{sourceKey}/{sampleId}") @GET public CohortSampleDTO getCohortSample( @PathParam("sampleId") Integer sampleId, @@ -84,7 +83,16 @@ public CohortSampleDTO getCohortSample( return this.samplingService.getSample(sampleId, withRecordCounts); } - @Path("/") + @Path("/has-samples/{cohortDefinitionId}") + @GET + public Map hasSamples( + @PathParam("cohortDefinitionId") int cohortDefinitionId + ) { + int nSamples = this.samplingService.countSamples(cohortDefinitionId); + return Collections.singletonMap("hasSamples", nSamples > 0); + } + + @Path("/{cohortDefinitionId}/{sourceKey}") @POST @Consumes(MediaType.APPLICATION_JSON) public CohortSampleDTO createCohortSample( @@ -105,7 +113,7 @@ public CohortSampleDTO createCohortSample( return samplingService.createSample(source, cohortDefinitionId, sampleParameters); } - @Path("/{sampleId}") + @Path("/{cohortDefinitionId}/{sourceKey}/{sampleId}") @DELETE public Response deleteCohortSample( @PathParam("sourceKey") String sourceKey, @@ -120,7 +128,7 @@ public Response deleteCohortSample( return Response.status(Response.Status.NO_CONTENT).build(); } - @Path("/") + @Path("/{cohortDefinitionId}/{sourceKey}") @DELETE public Response deleteCohortSamples( @PathParam("sourceKey") String sourceKey, From a26f76560621c78fcfa91d406f0e89673fda892a Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Mon, 10 Feb 2020 21:50:17 +0100 Subject: [PATCH 22/33] calcuate age in subquery to allow usage in sample expression --- .../cohortsample/CohortSamplingService.java | 24 +++++++++---------- .../cohortsample/sql/generateSample.sql | 24 +++++++++++-------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index e2bb8cc4d4..289f235007 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -308,32 +308,32 @@ private List sampleElements(SampleParametersDTO sampleParametersD if (ageMode != null) { switch (ageMode) { case LESS_THAN: - expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) < @age"); - sqlVariables.put("age", sample.getAgeMax()); + expressionBuilder.append(" AND age < @age_max"); + sqlVariables.put("age_max", sample.getAgeMax()); break; case LESS_THAN_OR_EQUAL: - expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) <= @age"); - sqlVariables.put("age", sample.getAgeMax()); + expressionBuilder.append(" AND age <= @age_max"); + sqlVariables.put("age_max", sample.getAgeMax()); break; case GREATER_THAN: - expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) > @age"); - sqlVariables.put("age", sample.getAgeMin()); + expressionBuilder.append(" AND age > @age_min"); + sqlVariables.put("age_min", sample.getAgeMin()); break; case GREATER_THAN_OR_EQUAL: - expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) >= @age"); - sqlVariables.put("age", sample.getAgeMin()); + expressionBuilder.append(" AND age >= @age_min"); + sqlVariables.put("age_min", sample.getAgeMin()); break; case EQUAL_TO: - expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) = @age"); - sqlVariables.put("age", sample.getAgeMin()); + expressionBuilder.append(" AND age = @age_equal"); + sqlVariables.put("age_equal", sample.getAgeMin()); break; case BETWEEN: - expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) <= @age_max AND cast(year(c.cohort_start_date) - p.year_of_birth as int) >= @age_min"); + expressionBuilder.append(" AND age <= @age_max AND age >= @age_min"); sqlVariables.put("age_min", sample.getAgeMin()); sqlVariables.put("age_max", sample.getAgeMax()); break; case NOT_BETWEEN: - expressionBuilder.append(" AND cast(year(c.cohort_start_date) - p.year_of_birth as int) > @age_max AND cast(year(c.cohort_start_date) - p.year_of_birth as int) < @age_min"); + expressionBuilder.append(" AND age > @age_max OR age < @age_min"); sqlVariables.put("age_min", sample.getAgeMin()); sqlVariables.put("age_max", sample.getAgeMax()); break; diff --git a/src/main/resources/resources/cohortsample/sql/generateSample.sql b/src/main/resources/resources/cohortsample/sql/generateSample.sql index 472aeb08e8..9394de3923 100644 --- a/src/main/resources/resources/cohortsample/sql/generateSample.sql +++ b/src/main/resources/resources/cohortsample/sql/generateSample.sql @@ -1,11 +1,15 @@ -select p.person_id, - c.cohort_definition_id as cohort_definition_id, - p.gender_concept_id as gender_concept_id, - cast(year(c.cohort_start_date) - p.year_of_birth as int) as age -from @results_schema.cohort c -join @CDM_schema.person p -on p.person_id = c.subject_id -where c.cohort_definition_id = @cohort_definition_id +SELECT * +FROM ( + SELECT + p.person_id AS person_id, + c.cohort_definition_id AS cohort_definition_id, + p.gender_concept_id AS gender_concept_id, + cast(year(c.cohort_start_date) - p.year_of_birth AS INT) AS age + FROM @results_schema.cohort c + JOIN @CDM_schema.person p + ON p.person_id = c.subject_id +) c +WHERE c.cohort_definition_id = @cohort_definition_id @expression -order by RAND() -; +ORDER BY RAND() +; \ No newline at end of file From e381ec165cdd148a7d12e76bb8b81cfd77938ac4 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Wed, 26 Feb 2020 17:16:11 +0100 Subject: [PATCH 23/33] delete samples of a source upon triggering cohort generation of that source --- .../org/ohdsi/webapi/service/CohortDefinitionService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java b/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java index e7cc109898..846180a9ee 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java @@ -440,8 +440,6 @@ public CohortDTO saveCohortDefinition(@PathParam("id") final int id, CohortDTO d currentDefinition.setModifiedBy(modifier); currentDefinition.setModifiedDate(currentTime); - this.samplingService.launchDeleteSamplesTasklet(id); - this.cohortDefinitionRepository.save(currentDefinition); return getCohortDefinition(id); } @@ -461,6 +459,8 @@ public JobExecutionResource generateCohort(@PathParam("id") final int id, @PathP Source source = getSourceRepository().findBySourceKey(sourceKey); CohortDefinition currentDefinition = this.cohortDefinitionRepository.findOne(id); + this.samplingService.launchDeleteSamplesTasklet(id, source.getSourceId()); + return cohortGenerationService.generateCohortViaJob(currentDefinition, source, Objects.nonNull(includeFeatures)); } From dde9418f340283c6fa834439de1d688719fd904c Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Fri, 28 Feb 2020 14:33:20 +0100 Subject: [PATCH 24/33] fix sql for gender sample criteria --- .../cohortsample/CohortSamplingService.java | 8 ++++---- .../cohortsample/sql/generateSample.sql | 19 +++++++++---------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index 289f235007..33b324310e 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -345,24 +345,24 @@ private List sampleElements(SampleParametersDTO sampleParametersD List conceptIds = gender.getConceptIds(); if (gender.isOtherNonBinary()) { if (conceptIds.size() == 0) { - expressionBuilder.append(" AND p.gender_concept_id NOT IN (") + expressionBuilder.append(" AND gender_concept_id NOT IN (") .append(GENDER_MALE_CONCEPT_ID) .append(',') .append(GENDER_FEMALE_CONCEPT_ID) .append(')'); } else if (conceptIds.size() == 1) { if (conceptIds.get(0) == GENDER_FEMALE_CONCEPT_ID) { - expressionBuilder.append(" AND p.gender_concept_id <> ") + expressionBuilder.append(" AND gender_concept_id <> ") .append(GENDER_MALE_CONCEPT_ID); } else { - expressionBuilder.append(" AND p.gender_concept_id <> ") + expressionBuilder.append(" AND gender_concept_id <> ") .append(GENDER_FEMALE_CONCEPT_ID); } } // else: all genders are selected, no where statement needed } else if (!conceptIds.isEmpty()) { - expressionBuilder.append(" AND p.gender_concept_id IN ("); + expressionBuilder.append(" AND gender_concept_id IN ("); for (int i = 0; i < conceptIds.size(); i++) { if (i > 0) { expressionBuilder.append(','); diff --git a/src/main/resources/resources/cohortsample/sql/generateSample.sql b/src/main/resources/resources/cohortsample/sql/generateSample.sql index 9394de3923..309e36cbac 100644 --- a/src/main/resources/resources/cohortsample/sql/generateSample.sql +++ b/src/main/resources/resources/cohortsample/sql/generateSample.sql @@ -1,15 +1,14 @@ SELECT * FROM ( SELECT - p.person_id AS person_id, - c.cohort_definition_id AS cohort_definition_id, - p.gender_concept_id AS gender_concept_id, - cast(year(c.cohort_start_date) - p.year_of_birth AS INT) AS age - FROM @results_schema.cohort c - JOIN @CDM_schema.person p - ON p.person_id = c.subject_id -) c -WHERE c.cohort_definition_id = @cohort_definition_id -@expression + cohort.subject_id AS person_id, + cohort.cohort_definition_id AS cohort_definition_id, + person.gender_concept_id AS gender_concept_id, + CAST(year(cohort.cohort_start_date) - person.year_of_birth AS INT) AS age + FROM @results_schema.cohort + JOIN @CDM_schema.person ON person_id = subject_id +) AS cte +WHERE cohort_definition_id = @cohort_definition_id + @expression ORDER BY RAND() ; \ No newline at end of file From 10fd14d597742527a2352b3ca81c769e41c0378d Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Wed, 4 Mar 2020 10:48:33 +0100 Subject: [PATCH 25/33] add has-samples endpoint by sourceKey and cohortDefinitionId --- .../webapi/cohortsample/CohortSampleRepository.java | 3 +++ .../webapi/cohortsample/CohortSamplingService.java | 4 ++++ .../org/ohdsi/webapi/service/CohortSampleService.java | 11 +++++++++++ 3 files changed, 18 insertions(+) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java index 319477e584..944e739c6b 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java @@ -20,4 +20,7 @@ public interface CohortSampleRepository extends CrudRepository hasSamples( int nSamples = this.samplingService.countSamples(cohortDefinitionId); return Collections.singletonMap("hasSamples", nSamples > 0); } + + @Path("/has-samples/{cohortDefinitionId}/{sourceKey}") + @GET + public Map hasSamples( + @PathParam("sourceKey") String sourceKey, + @PathParam("cohortDefinitionId") int cohortDefinitionId + ) { + Source source = getSource(sourceKey); + int nSamples = this.samplingService.countSamples(cohortDefinitionId, source.getId()); + return Collections.singletonMap("hasSamples", nSamples > 0); + } @Path("/{cohortDefinitionId}/{sourceKey}") @POST From b4df22f7a349913e4742aff481a733bf0fb5c466 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Wed, 4 Mar 2020 16:19:11 +0100 Subject: [PATCH 26/33] add isValid attribute to cohortsample summary --- .../webapi/cohortsample/dto/CohortSampleListDTO.java | 9 +++++++++ .../org/ohdsi/webapi/service/CohortSampleService.java | 1 + 2 files changed, 10 insertions(+) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleListDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleListDTO.java index 2dc85f2a4d..4da870edf5 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleListDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleListDTO.java @@ -8,6 +8,7 @@ public class CohortSampleListDTO { private int cohortDefinitionId; private int sourceId; private GenerationStatus generationStatus; + private boolean isValid; private List samples; public int getCohortDefinitionId() { @@ -34,6 +35,14 @@ public void setGenerationStatus(GenerationStatus generationStatus) { this.generationStatus = generationStatus; } + public boolean isValid() { + return isValid; + } + + public void setIsValid(boolean valid) { + isValid = valid; + } + public List getSamples() { return samples; } diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index 581c24691d..23952dc3f3 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -66,6 +66,7 @@ public CohortSampleListDTO listCohortSamples( CohortGenerationInfo generationInfo = generationInfoRepository.findOne( new CohortGenerationInfoId(cohortDefinitionId, source.getId())); result.setGenerationStatus(generationInfo != null ? generationInfo.getStatus() : null); + result.setIsValid(generationInfo != null && generationInfo.isIsValid()); result.setSamples(this.samplingService.listSamples(cohortDefinitionId, source.getId())); From a25245bd0b894f588902d7aab1311fda723de119 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Wed, 14 Oct 2020 16:27:49 +0200 Subject: [PATCH 27/33] use tab-indentation --- .../CleanupCohortSamplesTasklet.java | 224 ++--- .../webapi/cohortsample/CohortSample.java | 178 ++-- .../cohortsample/CohortSampleRepository.java | 16 +- .../cohortsample/CohortSamplingService.java | 864 +++++++++--------- .../webapi/cohortsample/SampleElement.java | 84 +- .../cohortsample/dto/CohortSampleDTO.java | 246 ++--- .../cohortsample/dto/CohortSampleListDTO.java | 90 +- .../cohortsample/dto/SampleElementDTO.java | 150 +-- .../cohortsample/dto/SampleParametersDTO.java | 488 +++++----- .../webapi/service/AbstractDaoService.java | 4 +- .../service/CohortDefinitionService.java | 18 +- .../webapi/service/CohortSampleService.java | 256 +++--- 12 files changed, 1309 insertions(+), 1309 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CleanupCohortSamplesTasklet.java b/src/main/java/org/ohdsi/webapi/cohortsample/CleanupCohortSamplesTasklet.java index 2746ec16f9..269230f39a 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CleanupCohortSamplesTasklet.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CleanupCohortSamplesTasklet.java @@ -30,116 +30,116 @@ import static org.ohdsi.webapi.Constants.Params.SOURCE_ID; public class CleanupCohortSamplesTasklet implements Tasklet { - private static final Logger log = LoggerFactory.getLogger(CleanupCohortTasklet.class); - - private final TransactionTemplate transactionTemplate; - private final SourceRepository sourceRepository; - private final CohortSamplingService samplingService; - private final CohortSampleRepository sampleRepository; - - public CleanupCohortSamplesTasklet( - final TransactionTemplate transactionTemplate, - final SourceRepository sourceRepository, - CohortSamplingService samplingService, - CohortSampleRepository sampleRepository - ) { - this.transactionTemplate = transactionTemplate; - this.sourceRepository = sourceRepository; - this.samplingService = samplingService; - this.sampleRepository = sampleRepository; - } - - private Integer doTask(ChunkContext chunkContext) { - Map jobParams = chunkContext.getStepContext().getJobParameters(); - int cohortDefinitionId = Integer.parseInt(jobParams.get(COHORT_DEFINITION_ID).toString()); - - if (jobParams.containsKey(SOURCE_ID)) { - int sourceId = Integer.parseInt(jobParams.get(SOURCE_ID).toString()); - Source source = this.sourceRepository.findOne(sourceId); - if (source != null) { - return mapSource(source, cohortDefinitionId); - } else { - return 0; - } - } else { - return this.sourceRepository.findAll().stream() - .filter(source-> source.getDaimons() - .stream() - .anyMatch(daimon -> daimon.getDaimonType() == SourceDaimon.DaimonType.Results)) - .mapToInt(source -> mapSource(source, cohortDefinitionId)) - .sum(); - } - } - - private int mapSource(Source source, int cohortDefinitionId) { - try { - String resultSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); - return transactionTemplate.execute(transactionStatus -> { - List samples = sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, source.getId()); - if (samples.isEmpty()) { - return 0; - } - - sampleRepository.delete(samples); - - int[] cohortSampleIds = samples.stream() - .mapToInt(CohortSample::getId) - .toArray(); - - PreparedStatementRenderer renderer = new PreparedStatementRenderer( - source, - "/resources/cohortsample/sql/deleteSampleElementsById.sql", - "results_schema", - resultSchema, - "cohortSampleId", - cohortSampleIds); - - samplingService.getSourceJdbcTemplate(source) - .update(renderer.getSql(), renderer.getOrderedParams()); - return cohortSampleIds.length; - }); - } catch (Exception e) { - log.error("Error deleting samples for cohort: {}, cause: {}", cohortDefinitionId, e.getMessage()); - return 0; - } - } - - @Override - public RepeatStatus execute(final StepContribution contribution, final ChunkContext chunkContext) throws Exception { - this.transactionTemplate.execute(status -> doTask(chunkContext)); - - return RepeatStatus.FINISHED; - } - - public JobExecutionResource launch(JobBuilderFactory jobBuilders, StepBuilderFactory stepBuilders, JobTemplate jobTemplate, int cohortDefinitionId) { - JobParametersBuilder builder = new JobParametersBuilder(); - builder.addString(JOB_NAME, String.format("Cleanup cohort samples of cohort definition %d.", cohortDefinitionId)); - builder.addString(COHORT_DEFINITION_ID, String.valueOf(cohortDefinitionId)); - - log.info("Beginning cohort cleanup for cohort definition id: {}", cohortDefinitionId); - return launch(jobBuilders, stepBuilders, jobTemplate, builder.toJobParameters()); - } - - public JobExecutionResource launch(JobBuilderFactory jobBuilders, StepBuilderFactory stepBuilders, JobTemplate jobTemplate, int cohortDefinitionId, int sourceId) { - JobParametersBuilder builder = new JobParametersBuilder(); - builder.addString(JOB_NAME, String.format("Cleanup cohort samples of cohort definition %d.", cohortDefinitionId)); - builder.addString(COHORT_DEFINITION_ID, String.valueOf(cohortDefinitionId)); - builder.addString(SOURCE_ID, String.valueOf(sourceId)); - - log.info("Beginning cohort cleanup for cohort definition id {} and source ID {}", cohortDefinitionId, sourceId); - return launch(jobBuilders, stepBuilders, jobTemplate, builder.toJobParameters()); - } - - private JobExecutionResource launch(JobBuilderFactory jobBuilders, StepBuilderFactory stepBuilders, JobTemplate jobTemplate, JobParameters jobParameters) { - Step cleanupStep = stepBuilders.get("cohortSample.cleanupSamples") - .tasklet(this) - .build(); - - SimpleJobBuilder cleanupJobBuilder = jobBuilders.get("cleanupSamples") - .start(cleanupStep); - - Job cleanupCohortJob = cleanupJobBuilder.build(); - - return jobTemplate.launch(cleanupCohortJob, jobParameters); - } + private static final Logger log = LoggerFactory.getLogger(CleanupCohortTasklet.class); + + private final TransactionTemplate transactionTemplate; + private final SourceRepository sourceRepository; + private final CohortSamplingService samplingService; + private final CohortSampleRepository sampleRepository; + + public CleanupCohortSamplesTasklet( + final TransactionTemplate transactionTemplate, + final SourceRepository sourceRepository, + CohortSamplingService samplingService, + CohortSampleRepository sampleRepository + ) { + this.transactionTemplate = transactionTemplate; + this.sourceRepository = sourceRepository; + this.samplingService = samplingService; + this.sampleRepository = sampleRepository; + } + + private Integer doTask(ChunkContext chunkContext) { + Map jobParams = chunkContext.getStepContext().getJobParameters(); + int cohortDefinitionId = Integer.parseInt(jobParams.get(COHORT_DEFINITION_ID).toString()); + + if (jobParams.containsKey(SOURCE_ID)) { + int sourceId = Integer.parseInt(jobParams.get(SOURCE_ID).toString()); + Source source = this.sourceRepository.findOne(sourceId); + if (source != null) { + return mapSource(source, cohortDefinitionId); + } else { + return 0; + } + } else { + return this.sourceRepository.findAll().stream() + .filter(source-> source.getDaimons() + .stream() + .anyMatch(daimon -> daimon.getDaimonType() == SourceDaimon.DaimonType.Results)) + .mapToInt(source -> mapSource(source, cohortDefinitionId)) + .sum(); + } + } + + private int mapSource(Source source, int cohortDefinitionId) { + try { + String resultSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); + return transactionTemplate.execute(transactionStatus -> { + List samples = sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, source.getId()); + if (samples.isEmpty()) { + return 0; + } + + sampleRepository.delete(samples); + + int[] cohortSampleIds = samples.stream() + .mapToInt(CohortSample::getId) + .toArray(); + + PreparedStatementRenderer renderer = new PreparedStatementRenderer( + source, + "/resources/cohortsample/sql/deleteSampleElementsById.sql", + "results_schema", + resultSchema, + "cohortSampleId", + cohortSampleIds); + + samplingService.getSourceJdbcTemplate(source) + .update(renderer.getSql(), renderer.getOrderedParams()); + return cohortSampleIds.length; + }); + } catch (Exception e) { + log.error("Error deleting samples for cohort: {}, cause: {}", cohortDefinitionId, e.getMessage()); + return 0; + } + } + + @Override + public RepeatStatus execute(final StepContribution contribution, final ChunkContext chunkContext) throws Exception { + this.transactionTemplate.execute(status -> doTask(chunkContext)); + + return RepeatStatus.FINISHED; + } + + public JobExecutionResource launch(JobBuilderFactory jobBuilders, StepBuilderFactory stepBuilders, JobTemplate jobTemplate, int cohortDefinitionId) { + JobParametersBuilder builder = new JobParametersBuilder(); + builder.addString(JOB_NAME, String.format("Cleanup cohort samples of cohort definition %d.", cohortDefinitionId)); + builder.addString(COHORT_DEFINITION_ID, String.valueOf(cohortDefinitionId)); + + log.info("Beginning cohort cleanup for cohort definition id: {}", cohortDefinitionId); + return launch(jobBuilders, stepBuilders, jobTemplate, builder.toJobParameters()); + } + + public JobExecutionResource launch(JobBuilderFactory jobBuilders, StepBuilderFactory stepBuilders, JobTemplate jobTemplate, int cohortDefinitionId, int sourceId) { + JobParametersBuilder builder = new JobParametersBuilder(); + builder.addString(JOB_NAME, String.format("Cleanup cohort samples of cohort definition %d.", cohortDefinitionId)); + builder.addString(COHORT_DEFINITION_ID, String.valueOf(cohortDefinitionId)); + builder.addString(SOURCE_ID, String.valueOf(sourceId)); + + log.info("Beginning cohort cleanup for cohort definition id {} and source ID {}", cohortDefinitionId, sourceId); + return launch(jobBuilders, stepBuilders, jobTemplate, builder.toJobParameters()); + } + + private JobExecutionResource launch(JobBuilderFactory jobBuilders, StepBuilderFactory stepBuilders, JobTemplate jobTemplate, JobParameters jobParameters) { + Step cleanupStep = stepBuilders.get("cohortSample.cleanupSamples") + .tasklet(this) + .build(); + + SimpleJobBuilder cleanupJobBuilder = jobBuilders.get("cleanupSamples") + .start(cleanupStep); + + Job cleanupCohortJob = cleanupJobBuilder.build(); + + return jobTemplate.launch(cleanupCohortJob, jobParameters); + } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java index f55da0b494..4bacd82e4f 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSample.java @@ -18,122 +18,122 @@ @Entity(name = "CohortSample") @Table(name = "cohort_sample") public class CohortSample extends CommonEntity { - @Id - @GenericGenerator( - name = "cohort_sample_generator", - strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", - parameters = { - @Parameter(name = "sequence_name", value = "cohort_sample_sequence"), - @Parameter(name = "increment_size", value = "1") - } - ) - @GeneratedValue(generator = "cohort_sample_generator") - private Integer id; + @Id + @GenericGenerator( + name = "cohort_sample_generator", + strategy = "org.hibernate.id.enhanced.SequenceStyleGenerator", + parameters = { + @Parameter(name = "sequence_name", value = "cohort_sample_sequence"), + @Parameter(name = "increment_size", value = "1") + } + ) + @GeneratedValue(generator = "cohort_sample_generator") + private Integer id; - @Column - private String name; + @Column + private String name; - @Column(name = "cohort_definition_id") - private int cohortDefinitionId; + @Column(name = "cohort_definition_id") + private int cohortDefinitionId; - @Column(name = "source_id") - private int sourceId; + @Column(name = "source_id") + private int sourceId; - @Column(name = "age_mode") - private String ageMode; + @Column(name = "age_mode") + private String ageMode; - @Column(name = "age_min") - private Integer ageMin; + @Column(name = "age_min") + private Integer ageMin; - @Column(name = "age_max") - private Integer ageMax; + @Column(name = "age_max") + private Integer ageMax; - @Column(name = "gender_concept_ids") - private String genderConceptIds; + @Column(name = "gender_concept_ids") + private String genderConceptIds; - @Column - private int size; + @Column + private int size; - @Transient - private List elements; + @Transient + private List elements; - public Integer getId() { - return this.id; - } + public Integer getId() { + return this.id; + } - public void setId(Integer id) { - this.id = id; - } + public void setId(Integer id) { + this.id = id; + } - public int getCohortDefinitionId() { - return cohortDefinitionId; - } + public int getCohortDefinitionId() { + return cohortDefinitionId; + } - public void setCohortDefinitionId(int cohortDefinitionId) { - this.cohortDefinitionId = cohortDefinitionId; - } + public void setCohortDefinitionId(int cohortDefinitionId) { + this.cohortDefinitionId = cohortDefinitionId; + } - public List getElements() { - return elements; - } + public List getElements() { + return elements; + } - public void setElements(List elements) { - this.elements = elements; - } + public void setElements(List elements) { + this.elements = elements; + } - public int getSize() { - return size; - } + public int getSize() { + return size; + } - public void setSize(int size) { - this.size = size; - } + public void setSize(int size) { + this.size = size; + } - public Integer getAgeMin() { - return ageMin; - } + public Integer getAgeMin() { + return ageMin; + } - public void setAgeMin(Integer ageMin) { - this.ageMin = ageMin; - } + public void setAgeMin(Integer ageMin) { + this.ageMin = ageMin; + } - public Integer getAgeMax() { - return ageMax; - } + public Integer getAgeMax() { + return ageMax; + } - public void setAgeMax(Integer ageMax) { - this.ageMax = ageMax; - } + public void setAgeMax(Integer ageMax) { + this.ageMax = ageMax; + } - public String getGenderConceptIds() { - return genderConceptIds; - } + public String getGenderConceptIds() { + return genderConceptIds; + } - public void setGenderConceptIds(String genderConceptIds) { - this.genderConceptIds = genderConceptIds; - } + public void setGenderConceptIds(String genderConceptIds) { + this.genderConceptIds = genderConceptIds; + } - public int getSourceId() { - return sourceId; - } + public int getSourceId() { + return sourceId; + } - public void setSourceId(int sourceId) { - this.sourceId = sourceId; - } + public void setSourceId(int sourceId) { + this.sourceId = sourceId; + } - public void setName(String name) { - this.name = name; - } + public void setName(String name) { + this.name = name; + } - public String getName() { - return name; - } + public String getName() { + return name; + } - public String getAgeMode() { - return ageMode; - } + public String getAgeMode() { + return ageMode; + } - public void setAgeMode(String ageMode) { - this.ageMode = ageMode; - } + public void setAgeMode(String ageMode) { + this.ageMode = ageMode; + } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java index 944e739c6b..d04aa4d941 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSampleRepository.java @@ -12,15 +12,15 @@ */ @Component public interface CohortSampleRepository extends CrudRepository { - @Query("SELECT c FROM CohortSample c LEFT JOIN FETCH c.createdBy WHERE c.id = :id") - CohortSample findById(@Param("id") int cohortSampleId); + @Query("SELECT c FROM CohortSample c LEFT JOIN FETCH c.createdBy WHERE c.id = :id") + CohortSample findById(@Param("id") int cohortSampleId); - @Query("SELECT c FROM CohortSample c LEFT JOIN FETCH c.createdBy WHERE c.cohortDefinitionId = :cohortDefinitionId AND c.sourceId = :sourceId") - List findByCohortDefinitionIdAndSourceId(@Param("cohortDefinitionId") int cohortDefinitionId, @Param("sourceId") int sourceId); + @Query("SELECT c FROM CohortSample c LEFT JOIN FETCH c.createdBy WHERE c.cohortDefinitionId = :cohortDefinitionId AND c.sourceId = :sourceId") + List findByCohortDefinitionIdAndSourceId(@Param("cohortDefinitionId") int cohortDefinitionId, @Param("sourceId") int sourceId); - @Query("SELECT count(c.id) FROM CohortSample c WHERE c.cohortDefinitionId = :cohortDefinitionId") - int countSamples(@Param("cohortDefinitionId") int cohortDefinitionId); + @Query("SELECT count(c.id) FROM CohortSample c WHERE c.cohortDefinitionId = :cohortDefinitionId") + int countSamples(@Param("cohortDefinitionId") int cohortDefinitionId); - @Query("SELECT count(c.id) FROM CohortSample c WHERE c.cohortDefinitionId = :cohortDefinitionId AND c.sourceId = :sourceId") - int countSamples(@Param("cohortDefinitionId") int cohortDefinitionId, @Param("sourceId") int sourceId); + @Query("SELECT count(c.id) FROM CohortSample c WHERE c.cohortDefinitionId = :cohortDefinitionId AND c.sourceId = :sourceId") + int countSamples(@Param("cohortDefinitionId") int cohortDefinitionId, @Param("sourceId") int sourceId); } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index 91d30ba821..af944d5cd4 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -41,436 +41,436 @@ */ @Component public class CohortSamplingService extends AbstractDaoService { - private final CohortSampleRepository sampleRepository; - private final JobBuilderFactory jobBuilders; - private final StepBuilderFactory stepBuilders; - private final JobTemplate jobTemplate; - - @Autowired - public CohortSamplingService( - CohortSampleRepository sampleRepository, - JobBuilderFactory jobBuilders, - StepBuilderFactory stepBuilders, - JobTemplate jobTemplate) { - this.sampleRepository = sampleRepository; - this.jobBuilders = jobBuilders; - this.stepBuilders = stepBuilders; - this.jobTemplate = jobTemplate; - } - - public List listSamples(int cohortDefinitionId, int sourceId) { - return sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, sourceId).stream() - .map(sample -> sampleToSampleDTO(sample, null, false)) - .collect(Collectors.toList()); - } - - public CohortSampleDTO getSample(int sampleId, boolean withRecordCounts) { - CohortSample sample = sampleRepository.findById(sampleId); - if (sample == null) { - throw new NotFoundException("Cohort sample with ID " + sampleId + " not found"); - } - Source source = getSourceRepository().findBySourceId(sample.getSourceId()); - List sampleElements = findSampleElements(source, sample.getId(), withRecordCounts); - return sampleToSampleDTO(sample, sampleElements, true); - } - - public int countSamples(int cohortDefinitionId) { - return sampleRepository.countSamples(cohortDefinitionId); - } - - public int countSamples(int cohortDefinitionId, int sourceId) { - return sampleRepository.countSamples(cohortDefinitionId, sourceId); - } - - /** - * Find all sample elements of a sample. - * @param source Source to use - * @param cohortSampleId sample ID of the elements. - * @param withRecordCounts whether to return record counts. This makes the query much slower. - * @return list of elements. - */ - private List findSampleElements(Source source, int cohortSampleId, boolean withRecordCounts) { - JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); - PreparedStatementRenderer renderer; - Collection optionalFields; - - if (withRecordCounts) { - renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql", - new String[]{"results_schema", "CDM_schema"}, - new String[]{source.getTableQualifier(SourceDaimon.DaimonType.Results), source.getTableQualifier(SourceDaimon.DaimonType.CDM)}, - "cohortSampleId", cohortSampleId); - optionalFields = Collections.singleton("record_count"); - } else { - renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleId.sql", - "results_schema", - source.getTableQualifier(SourceDaimon.DaimonType.Results), - "cohortSampleId", cohortSampleId); - optionalFields = Collections.emptySet(); - } - return jdbcTemplate.query(renderer.getSql(), renderer.getOrderedParams(), new CohortSampleElementRowMapper(optionalFields)); - } - - /** - * Create a new sample in given source and cohort definition, using sample parameters. - * @param source Source to use - * @param cohortDefinitionId cohort definition ID to sample - * @param sampleParameters parameters to define the sample - * @return list of elements. - */ - public CohortSampleDTO createSample(Source source, int cohortDefinitionId, SampleParametersDTO sampleParameters) { - JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); - - CohortSample sample = new CohortSample(); - sample.setName(sampleParameters.getName()); - sample.setCohortDefinitionId(cohortDefinitionId); - sample.setSourceId(source.getId()); - sample.setSize(sampleParameters.getSize()); - - SampleParametersDTO.AgeDTO age = sampleParameters.getAge(); - if (age != null) { - switch (age.getMode()) { - case LESS_THAN: - case LESS_THAN_OR_EQUAL: - sample.setAgeMax(age.getValue()); - break; - case GREATER_THAN: - case GREATER_THAN_OR_EQUAL: - sample.setAgeMin(age.getValue()); - break; - case EQUAL_TO: - sample.setAgeMin(age.getValue()); - sample.setAgeMax(age.getValue()); - break; - case BETWEEN: - case NOT_BETWEEN: - sample.setAgeMin(age.getMin()); - sample.setAgeMax(age.getMax()); - break; - } - sample.setAgeMode(age.getMode().getSerialName()); - } - - SampleParametersDTO.GenderDTO gender = sampleParameters.getGender(); - if (gender != null) { - StringBuilder sb = new StringBuilder(12); - for (Integer conceptId : gender.getConceptIds()) { - if (sb.length() > 0) { - sb.append(','); - } - sb.append(conceptId); - } - if (gender.isOtherNonBinary()) { - if (sb.length() > 0) { - sb.append(','); - } - sb.append(-1); - } - sample.setGenderConceptIds(sb.toString()); - } - sample.setCreatedBy(getCurrentUser()); - sample.setCreatedDate(new Date()); - - log.info("Sampling {} elements for cohort {}", sampleParameters.getSize(), cohortDefinitionId); - final List elements = sampleElements(sampleParameters, sample, jdbcTemplate, source); - - if (elements.size() < sample.getSize()) { - sample.setSize(elements.size()); - } - - getTransactionTemplate().execute((TransactionCallback) transactionStatus -> { - log.debug("Saving {} sample elements for cohort {}", sample.getSize(), cohortDefinitionId); - CohortSample updatedSample = sampleRepository.save(sample); - insertSampledElements(source, jdbcTemplate, updatedSample.getId(), elements); - - return null; - }); - - return sampleToSampleDTO(sample, elements, true); - } - - /** Convert a given sample with given elements to a DTO. */ - private CohortSampleDTO sampleToSampleDTO(CohortSample sample, List elements, boolean includeIds) { - CohortSampleDTO sampleDTO = new CohortSampleDTO(); - sampleDTO.setId(sample.getId()); - sampleDTO.setName(sample.getName()); - sampleDTO.setSize(sample.getSize()); - if (includeIds) { - sampleDTO.setCohortDefinitionId(sample.getCohortDefinitionId()); - sampleDTO.setSourceId(sample.getSourceId()); - } - sampleDTO.setCreatedDate(sample.getCreatedDate()); - UserEntity createdBy = sample.getCreatedBy(); - if (createdBy != null) { - UserDTO userDto = new UserDTO(); - userDto.setId(createdBy.getId()); - userDto.setLogin(createdBy.getLogin()); - userDto.setName(createdBy.getName()); - sampleDTO.setCreatedBy(userDto); - } - - SampleParametersDTO.AgeMode ageMode = SampleParametersDTO.AgeMode.fromSerialName(sample.getAgeMode()); - if (ageMode != null) { - SampleParametersDTO.AgeDTO age = new SampleParametersDTO.AgeDTO(); - age.setMode(ageMode); - switch (ageMode) { - case LESS_THAN: - case LESS_THAN_OR_EQUAL: - case EQUAL_TO: - age.setValue(sample.getAgeMax()); - break; - case GREATER_THAN: - case GREATER_THAN_OR_EQUAL: - age.setValue(sample.getAgeMin()); - break; - case BETWEEN: - case NOT_BETWEEN: - age.setMin(sample.getAgeMin()); - age.setMax(sample.getAgeMax()); - break; - } - sampleDTO.setAge(age); - } - if (sample.getGenderConceptIds() != null && !sample.getGenderConceptIds().isEmpty()) { - List conceptIds = Arrays.stream(sample.getGenderConceptIds().split(",")) - .map(Integer::valueOf) - .collect(Collectors.toList()); - - SampleParametersDTO.GenderDTO genderDto = new SampleParametersDTO.GenderDTO(); - - if (conceptIds.remove(Integer.valueOf(-1))) { - genderDto.setOtherNonBinary(true); - } - - genderDto.setConceptIds(conceptIds); - sampleDTO.setGender(genderDto); - } - - sampleDTO.setElements(sampleElementToDTO(elements)); - return sampleDTO; - } - - /** Convert given sample elements DTOs. */ - private List sampleElementToDTO(List elements) { - if (elements == null) { - return null; - } - - return elements.stream() - .map(el -> { - SampleElementDTO elementDTO = new SampleElementDTO(); - elementDTO.setRank(el.getRank()); - elementDTO.setPersonId(el.getPersonId()); - elementDTO.setAge(el.getAge()); - elementDTO.setGenderConceptId(el.getGenderConceptId()); - elementDTO.setRecordCount(el.getRecordCount()); - return elementDTO; - }) - .collect(Collectors.toList()); - } - - /** Insert elements that have been sampled. */ - private void insertSampledElements(Source source, JdbcTemplate jdbcTemplate, int sampleId, List elements) { - if (elements.isEmpty()) { - return; - } - - String[] parameters = new String[] { "results_schema" }; - String[] parameterValues = new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results) }; - String[] sqlParameters = new String[] { "cohortSampleId", "rank", "personId", "age", "genderConceptId" }; - - String statement = null; - List variables = new ArrayList<>(elements.size()); - for (SampleElement element : elements) { - Object[] sqlValues = new Object[] { - sampleId, - element.getRank(), - element.getPersonId(), - element.getAge(), - element.getGenderConceptId() }; - - PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/insertSampleElement.sql", parameters, parameterValues, sqlParameters, sqlValues); - - if (statement == null) { - statement = renderer.getSql(); - } - - variables.add(renderer.getOrderedParams()); - } - - jdbcTemplate.batchUpdate(statement, variables); - } - - /** Sample elements based on parameters. */ - private List sampleElements(SampleParametersDTO sampleParametersDTO, CohortSample sample, JdbcTemplate jdbcTemplate, Source source) { - StringBuilder expressionBuilder = new StringBuilder(); - Map sqlVariables = new LinkedHashMap<>(); - - sqlVariables.put("cohort_definition_id", sample.getCohortDefinitionId()); - - SampleParametersDTO.AgeMode ageMode = SampleParametersDTO.AgeMode.fromSerialName(sample.getAgeMode()); - - if (ageMode != null) { - switch (ageMode) { - case LESS_THAN: - expressionBuilder.append(" AND age < @age_max"); - sqlVariables.put("age_max", sample.getAgeMax()); - break; - case LESS_THAN_OR_EQUAL: - expressionBuilder.append(" AND age <= @age_max"); - sqlVariables.put("age_max", sample.getAgeMax()); - break; - case GREATER_THAN: - expressionBuilder.append(" AND age > @age_min"); - sqlVariables.put("age_min", sample.getAgeMin()); - break; - case GREATER_THAN_OR_EQUAL: - expressionBuilder.append(" AND age >= @age_min"); - sqlVariables.put("age_min", sample.getAgeMin()); - break; - case EQUAL_TO: - expressionBuilder.append(" AND age = @age_equal"); - sqlVariables.put("age_equal", sample.getAgeMin()); - break; - case BETWEEN: - expressionBuilder.append(" AND age <= @age_max AND age >= @age_min"); - sqlVariables.put("age_min", sample.getAgeMin()); - sqlVariables.put("age_max", sample.getAgeMax()); - break; - case NOT_BETWEEN: - expressionBuilder.append(" AND age > @age_max OR age < @age_min"); - sqlVariables.put("age_min", sample.getAgeMin()); - sqlVariables.put("age_max", sample.getAgeMax()); - break; - } - } - - SampleParametersDTO.GenderDTO gender = sampleParametersDTO.getGender(); - if (gender != null) { - List conceptIds = gender.getConceptIds(); - if (gender.isOtherNonBinary()) { - if (conceptIds.size() == 0) { - expressionBuilder.append(" AND gender_concept_id NOT IN (") - .append(GENDER_MALE_CONCEPT_ID) - .append(',') - .append(GENDER_FEMALE_CONCEPT_ID) - .append(')'); - } else if (conceptIds.size() == 1) { - if (conceptIds.get(0) == GENDER_FEMALE_CONCEPT_ID) { - expressionBuilder.append(" AND gender_concept_id <> ") - .append(GENDER_MALE_CONCEPT_ID); - - } else { - expressionBuilder.append(" AND gender_concept_id <> ") - .append(GENDER_FEMALE_CONCEPT_ID); - } - } - // else: all genders are selected, no where statement needed - } else if (!conceptIds.isEmpty()) { - expressionBuilder.append(" AND gender_concept_id IN ("); - for (int i = 0; i < conceptIds.size(); i++) { - if (i > 0) { - expressionBuilder.append(','); - } - expressionBuilder.append(conceptIds.get(i)); - } - expressionBuilder.append(')'); - } - // else: all genders are selected, no where statement needed - } - - String[] parameterKeys = new String[] { "results_schema", "CDM_schema", "expression"}; - String[] parameterValues = new String[] { - source.getTableQualifier(SourceDaimon.DaimonType.Results), - source.getTableQualifier(SourceDaimon.DaimonType.CDM), - expressionBuilder.toString() }; - String[] sqlVariableKeys = sqlVariables.keySet().toArray(new String[0]); - Object[] sqlVariableValues = Stream.of(sqlVariableKeys) - .map(sqlVariables::get) - .toArray(Object[]::new); - - PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/generateSample.sql", - parameterKeys, - parameterValues, - sqlVariableKeys, - sqlVariableValues); - - jdbcTemplate.setMaxRows(sample.getSize()); - - return jdbcTemplate.query(renderer.getSql(), renderer.getOrderedParams(), (rs, rowNum) -> { - SampleElement element = new SampleElement(); - element.setRank(rowNum); - element.setAge(rs.getInt("age")); - element.setGenderConceptId(rs.getInt("gender_concept_id")); - element.setPersonId(rs.getLong("person_id")); - return element; - }); - } - - /** Delete a sample and its elements. */ - public void deleteSample(int cohortDefinitionId, Source source, int sampleId) { - JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); - String resultsSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); - String sql = new PreparedStatementRenderer( - source, - "/resources/cohortsample/sql/deleteSampleElementsById.sql", - "results_schema", - resultsSchema, - "cohortSampleId", - sampleId).getSql(); - CohortSample sample = sampleRepository.findOne(sampleId); - if (sample == null) { - throw new NotFoundException("Sample with ID " + sampleId + " does not exist"); - } - if (sample.getCohortDefinitionId() != cohortDefinitionId) { - throw new BadRequestException("Cohort definition ID " + sample.getCohortDefinitionId() + " does not match provided cohort definition id " + cohortDefinitionId); - } - if (sample.getSourceId() != source.getId()) { - throw new BadRequestException("Source " + sample.getSourceId() + " does not match provided source " + source.getId()); - } - - getTransactionTemplate().execute((TransactionCallback) transactionStatus -> { - sampleRepository.delete(sampleId); - jdbcTemplate.update(sql, sampleId); - return null; - }); - } - - public void launchDeleteSamplesTasklet(int cohortDefinitionId) { - CleanupCohortSamplesTasklet tasklet = createDeleteSamplesTasklet(); - - tasklet.launch(jobBuilders, stepBuilders, jobTemplate, cohortDefinitionId); - } - - public void launchDeleteSamplesTasklet(int cohortDefinitionId, int sourceId) { - CleanupCohortSamplesTasklet tasklet = createDeleteSamplesTasklet(); - - tasklet.launch(jobBuilders, stepBuilders, jobTemplate, cohortDefinitionId, sourceId); - } - - public CleanupCohortSamplesTasklet createDeleteSamplesTasklet() { - return new CleanupCohortSamplesTasklet(getTransactionTemplate(), getSourceRepository(), this, sampleRepository); - } - - /** Maps a SQL result to a sample element. */ - private static class CohortSampleElementRowMapper implements RowMapper { - private final Collection optionalFields; - - CohortSampleElementRowMapper(Collection optionalFields) { - this.optionalFields = optionalFields; - } - - @Override - public SampleElement mapRow(ResultSet rs, int rowNum) throws SQLException { - SampleElement sample = new SampleElement(); - sample.setRank(rs.getInt("rank")); - sample.setSampleId(rs.getInt("cohort_sample_id")); - sample.setPersonId(rs.getInt("person_id")); - sample.setGenderConceptId(rs.getInt("gender_concept_id")); - sample.setAge(rs.getInt("age")); - if (optionalFields.contains("record_count")) { - sample.setRecordCount(rs.getInt("record_count")); - } - return sample; - } - } + private final CohortSampleRepository sampleRepository; + private final JobBuilderFactory jobBuilders; + private final StepBuilderFactory stepBuilders; + private final JobTemplate jobTemplate; + + @Autowired + public CohortSamplingService( + CohortSampleRepository sampleRepository, + JobBuilderFactory jobBuilders, + StepBuilderFactory stepBuilders, + JobTemplate jobTemplate) { + this.sampleRepository = sampleRepository; + this.jobBuilders = jobBuilders; + this.stepBuilders = stepBuilders; + this.jobTemplate = jobTemplate; + } + + public List listSamples(int cohortDefinitionId, int sourceId) { + return sampleRepository.findByCohortDefinitionIdAndSourceId(cohortDefinitionId, sourceId).stream() + .map(sample -> sampleToSampleDTO(sample, null, false)) + .collect(Collectors.toList()); + } + + public CohortSampleDTO getSample(int sampleId, boolean withRecordCounts) { + CohortSample sample = sampleRepository.findById(sampleId); + if (sample == null) { + throw new NotFoundException("Cohort sample with ID " + sampleId + " not found"); + } + Source source = getSourceRepository().findBySourceId(sample.getSourceId()); + List sampleElements = findSampleElements(source, sample.getId(), withRecordCounts); + return sampleToSampleDTO(sample, sampleElements, true); + } + + public int countSamples(int cohortDefinitionId) { + return sampleRepository.countSamples(cohortDefinitionId); + } + + public int countSamples(int cohortDefinitionId, int sourceId) { + return sampleRepository.countSamples(cohortDefinitionId, sourceId); + } + + /** + * Find all sample elements of a sample. + * @param source Source to use + * @param cohortSampleId sample ID of the elements. + * @param withRecordCounts whether to return record counts. This makes the query much slower. + * @return list of elements. + */ + private List findSampleElements(Source source, int cohortSampleId, boolean withRecordCounts) { + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + PreparedStatementRenderer renderer; + Collection optionalFields; + + if (withRecordCounts) { + renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql", + new String[]{"results_schema", "CDM_schema"}, + new String[]{source.getTableQualifier(SourceDaimon.DaimonType.Results), source.getTableQualifier(SourceDaimon.DaimonType.CDM)}, + "cohortSampleId", cohortSampleId); + optionalFields = Collections.singleton("record_count"); + } else { + renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/findElementsByCohortSampleId.sql", + "results_schema", + source.getTableQualifier(SourceDaimon.DaimonType.Results), + "cohortSampleId", cohortSampleId); + optionalFields = Collections.emptySet(); + } + return jdbcTemplate.query(renderer.getSql(), renderer.getOrderedParams(), new CohortSampleElementRowMapper(optionalFields)); + } + + /** + * Create a new sample in given source and cohort definition, using sample parameters. + * @param source Source to use + * @param cohortDefinitionId cohort definition ID to sample + * @param sampleParameters parameters to define the sample + * @return list of elements. + */ + public CohortSampleDTO createSample(Source source, int cohortDefinitionId, SampleParametersDTO sampleParameters) { + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + + CohortSample sample = new CohortSample(); + sample.setName(sampleParameters.getName()); + sample.setCohortDefinitionId(cohortDefinitionId); + sample.setSourceId(source.getId()); + sample.setSize(sampleParameters.getSize()); + + SampleParametersDTO.AgeDTO age = sampleParameters.getAge(); + if (age != null) { + switch (age.getMode()) { + case LESS_THAN: + case LESS_THAN_OR_EQUAL: + sample.setAgeMax(age.getValue()); + break; + case GREATER_THAN: + case GREATER_THAN_OR_EQUAL: + sample.setAgeMin(age.getValue()); + break; + case EQUAL_TO: + sample.setAgeMin(age.getValue()); + sample.setAgeMax(age.getValue()); + break; + case BETWEEN: + case NOT_BETWEEN: + sample.setAgeMin(age.getMin()); + sample.setAgeMax(age.getMax()); + break; + } + sample.setAgeMode(age.getMode().getSerialName()); + } + + SampleParametersDTO.GenderDTO gender = sampleParameters.getGender(); + if (gender != null) { + StringBuilder sb = new StringBuilder(12); + for (Integer conceptId : gender.getConceptIds()) { + if (sb.length() > 0) { + sb.append(','); + } + sb.append(conceptId); + } + if (gender.isOtherNonBinary()) { + if (sb.length() > 0) { + sb.append(','); + } + sb.append(-1); + } + sample.setGenderConceptIds(sb.toString()); + } + sample.setCreatedBy(getCurrentUser()); + sample.setCreatedDate(new Date()); + + log.info("Sampling {} elements for cohort {}", sampleParameters.getSize(), cohortDefinitionId); + final List elements = sampleElements(sampleParameters, sample, jdbcTemplate, source); + + if (elements.size() < sample.getSize()) { + sample.setSize(elements.size()); + } + + getTransactionTemplate().execute((TransactionCallback) transactionStatus -> { + log.debug("Saving {} sample elements for cohort {}", sample.getSize(), cohortDefinitionId); + CohortSample updatedSample = sampleRepository.save(sample); + insertSampledElements(source, jdbcTemplate, updatedSample.getId(), elements); + + return null; + }); + + return sampleToSampleDTO(sample, elements, true); + } + + /** Convert a given sample with given elements to a DTO. */ + private CohortSampleDTO sampleToSampleDTO(CohortSample sample, List elements, boolean includeIds) { + CohortSampleDTO sampleDTO = new CohortSampleDTO(); + sampleDTO.setId(sample.getId()); + sampleDTO.setName(sample.getName()); + sampleDTO.setSize(sample.getSize()); + if (includeIds) { + sampleDTO.setCohortDefinitionId(sample.getCohortDefinitionId()); + sampleDTO.setSourceId(sample.getSourceId()); + } + sampleDTO.setCreatedDate(sample.getCreatedDate()); + UserEntity createdBy = sample.getCreatedBy(); + if (createdBy != null) { + UserDTO userDto = new UserDTO(); + userDto.setId(createdBy.getId()); + userDto.setLogin(createdBy.getLogin()); + userDto.setName(createdBy.getName()); + sampleDTO.setCreatedBy(userDto); + } + + SampleParametersDTO.AgeMode ageMode = SampleParametersDTO.AgeMode.fromSerialName(sample.getAgeMode()); + if (ageMode != null) { + SampleParametersDTO.AgeDTO age = new SampleParametersDTO.AgeDTO(); + age.setMode(ageMode); + switch (ageMode) { + case LESS_THAN: + case LESS_THAN_OR_EQUAL: + case EQUAL_TO: + age.setValue(sample.getAgeMax()); + break; + case GREATER_THAN: + case GREATER_THAN_OR_EQUAL: + age.setValue(sample.getAgeMin()); + break; + case BETWEEN: + case NOT_BETWEEN: + age.setMin(sample.getAgeMin()); + age.setMax(sample.getAgeMax()); + break; + } + sampleDTO.setAge(age); + } + if (sample.getGenderConceptIds() != null && !sample.getGenderConceptIds().isEmpty()) { + List conceptIds = Arrays.stream(sample.getGenderConceptIds().split(",")) + .map(Integer::valueOf) + .collect(Collectors.toList()); + + SampleParametersDTO.GenderDTO genderDto = new SampleParametersDTO.GenderDTO(); + + if (conceptIds.remove(Integer.valueOf(-1))) { + genderDto.setOtherNonBinary(true); + } + + genderDto.setConceptIds(conceptIds); + sampleDTO.setGender(genderDto); + } + + sampleDTO.setElements(sampleElementToDTO(elements)); + return sampleDTO; + } + + /** Convert given sample elements DTOs. */ + private List sampleElementToDTO(List elements) { + if (elements == null) { + return null; + } + + return elements.stream() + .map(el -> { + SampleElementDTO elementDTO = new SampleElementDTO(); + elementDTO.setRank(el.getRank()); + elementDTO.setPersonId(el.getPersonId()); + elementDTO.setAge(el.getAge()); + elementDTO.setGenderConceptId(el.getGenderConceptId()); + elementDTO.setRecordCount(el.getRecordCount()); + return elementDTO; + }) + .collect(Collectors.toList()); + } + + /** Insert elements that have been sampled. */ + private void insertSampledElements(Source source, JdbcTemplate jdbcTemplate, int sampleId, List elements) { + if (elements.isEmpty()) { + return; + } + + String[] parameters = new String[] { "results_schema" }; + String[] parameterValues = new String[] { source.getTableQualifier(SourceDaimon.DaimonType.Results) }; + String[] sqlParameters = new String[] { "cohortSampleId", "rank", "personId", "age", "genderConceptId" }; + + String statement = null; + List variables = new ArrayList<>(elements.size()); + for (SampleElement element : elements) { + Object[] sqlValues = new Object[] { + sampleId, + element.getRank(), + element.getPersonId(), + element.getAge(), + element.getGenderConceptId() }; + + PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/insertSampleElement.sql", parameters, parameterValues, sqlParameters, sqlValues); + + if (statement == null) { + statement = renderer.getSql(); + } + + variables.add(renderer.getOrderedParams()); + } + + jdbcTemplate.batchUpdate(statement, variables); + } + + /** Sample elements based on parameters. */ + private List sampleElements(SampleParametersDTO sampleParametersDTO, CohortSample sample, JdbcTemplate jdbcTemplate, Source source) { + StringBuilder expressionBuilder = new StringBuilder(); + Map sqlVariables = new LinkedHashMap<>(); + + sqlVariables.put("cohort_definition_id", sample.getCohortDefinitionId()); + + SampleParametersDTO.AgeMode ageMode = SampleParametersDTO.AgeMode.fromSerialName(sample.getAgeMode()); + + if (ageMode != null) { + switch (ageMode) { + case LESS_THAN: + expressionBuilder.append(" AND age < @age_max"); + sqlVariables.put("age_max", sample.getAgeMax()); + break; + case LESS_THAN_OR_EQUAL: + expressionBuilder.append(" AND age <= @age_max"); + sqlVariables.put("age_max", sample.getAgeMax()); + break; + case GREATER_THAN: + expressionBuilder.append(" AND age > @age_min"); + sqlVariables.put("age_min", sample.getAgeMin()); + break; + case GREATER_THAN_OR_EQUAL: + expressionBuilder.append(" AND age >= @age_min"); + sqlVariables.put("age_min", sample.getAgeMin()); + break; + case EQUAL_TO: + expressionBuilder.append(" AND age = @age_equal"); + sqlVariables.put("age_equal", sample.getAgeMin()); + break; + case BETWEEN: + expressionBuilder.append(" AND age <= @age_max AND age >= @age_min"); + sqlVariables.put("age_min", sample.getAgeMin()); + sqlVariables.put("age_max", sample.getAgeMax()); + break; + case NOT_BETWEEN: + expressionBuilder.append(" AND age > @age_max OR age < @age_min"); + sqlVariables.put("age_min", sample.getAgeMin()); + sqlVariables.put("age_max", sample.getAgeMax()); + break; + } + } + + SampleParametersDTO.GenderDTO gender = sampleParametersDTO.getGender(); + if (gender != null) { + List conceptIds = gender.getConceptIds(); + if (gender.isOtherNonBinary()) { + if (conceptIds.size() == 0) { + expressionBuilder.append(" AND gender_concept_id NOT IN (") + .append(GENDER_MALE_CONCEPT_ID) + .append(',') + .append(GENDER_FEMALE_CONCEPT_ID) + .append(')'); + } else if (conceptIds.size() == 1) { + if (conceptIds.get(0) == GENDER_FEMALE_CONCEPT_ID) { + expressionBuilder.append(" AND gender_concept_id <> ") + .append(GENDER_MALE_CONCEPT_ID); + + } else { + expressionBuilder.append(" AND gender_concept_id <> ") + .append(GENDER_FEMALE_CONCEPT_ID); + } + } + // else: all genders are selected, no where statement needed + } else if (!conceptIds.isEmpty()) { + expressionBuilder.append(" AND gender_concept_id IN ("); + for (int i = 0; i < conceptIds.size(); i++) { + if (i > 0) { + expressionBuilder.append(','); + } + expressionBuilder.append(conceptIds.get(i)); + } + expressionBuilder.append(')'); + } + // else: all genders are selected, no where statement needed + } + + String[] parameterKeys = new String[] { "results_schema", "CDM_schema", "expression"}; + String[] parameterValues = new String[] { + source.getTableQualifier(SourceDaimon.DaimonType.Results), + source.getTableQualifier(SourceDaimon.DaimonType.CDM), + expressionBuilder.toString() }; + String[] sqlVariableKeys = sqlVariables.keySet().toArray(new String[0]); + Object[] sqlVariableValues = Stream.of(sqlVariableKeys) + .map(sqlVariables::get) + .toArray(Object[]::new); + + PreparedStatementRenderer renderer = new PreparedStatementRenderer(source, "/resources/cohortsample/sql/generateSample.sql", + parameterKeys, + parameterValues, + sqlVariableKeys, + sqlVariableValues); + + jdbcTemplate.setMaxRows(sample.getSize()); + + return jdbcTemplate.query(renderer.getSql(), renderer.getOrderedParams(), (rs, rowNum) -> { + SampleElement element = new SampleElement(); + element.setRank(rowNum); + element.setAge(rs.getInt("age")); + element.setGenderConceptId(rs.getInt("gender_concept_id")); + element.setPersonId(rs.getLong("person_id")); + return element; + }); + } + + /** Delete a sample and its elements. */ + public void deleteSample(int cohortDefinitionId, Source source, int sampleId) { + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + String resultsSchema = source.getTableQualifier(SourceDaimon.DaimonType.Results); + String sql = new PreparedStatementRenderer( + source, + "/resources/cohortsample/sql/deleteSampleElementsById.sql", + "results_schema", + resultsSchema, + "cohortSampleId", + sampleId).getSql(); + CohortSample sample = sampleRepository.findOne(sampleId); + if (sample == null) { + throw new NotFoundException("Sample with ID " + sampleId + " does not exist"); + } + if (sample.getCohortDefinitionId() != cohortDefinitionId) { + throw new BadRequestException("Cohort definition ID " + sample.getCohortDefinitionId() + " does not match provided cohort definition id " + cohortDefinitionId); + } + if (sample.getSourceId() != source.getId()) { + throw new BadRequestException("Source " + sample.getSourceId() + " does not match provided source " + source.getId()); + } + + getTransactionTemplate().execute((TransactionCallback) transactionStatus -> { + sampleRepository.delete(sampleId); + jdbcTemplate.update(sql, sampleId); + return null; + }); + } + + public void launchDeleteSamplesTasklet(int cohortDefinitionId) { + CleanupCohortSamplesTasklet tasklet = createDeleteSamplesTasklet(); + + tasklet.launch(jobBuilders, stepBuilders, jobTemplate, cohortDefinitionId); + } + + public void launchDeleteSamplesTasklet(int cohortDefinitionId, int sourceId) { + CleanupCohortSamplesTasklet tasklet = createDeleteSamplesTasklet(); + + tasklet.launch(jobBuilders, stepBuilders, jobTemplate, cohortDefinitionId, sourceId); + } + + public CleanupCohortSamplesTasklet createDeleteSamplesTasklet() { + return new CleanupCohortSamplesTasklet(getTransactionTemplate(), getSourceRepository(), this, sampleRepository); + } + + /** Maps a SQL result to a sample element. */ + private static class CohortSampleElementRowMapper implements RowMapper { + private final Collection optionalFields; + + CohortSampleElementRowMapper(Collection optionalFields) { + this.optionalFields = optionalFields; + } + + @Override + public SampleElement mapRow(ResultSet rs, int rowNum) throws SQLException { + SampleElement sample = new SampleElement(); + sample.setRank(rs.getInt("rank")); + sample.setSampleId(rs.getInt("cohort_sample_id")); + sample.setPersonId(rs.getInt("person_id")); + sample.setGenderConceptId(rs.getInt("gender_concept_id")); + sample.setAge(rs.getInt("age")); + if (optionalFields.contains("record_count")) { + sample.setRecordCount(rs.getInt("record_count")); + } + return sample; + } + } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java b/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java index c022e1ae71..7a2d506899 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/SampleElement.java @@ -2,63 +2,63 @@ /** A single person that is part of a given sample. */ public class SampleElement { - private int sampleId; + private int sampleId; - private int rank; + private int rank; - private long personId; + private long personId; - private long genderConceptId; + private long genderConceptId; - private int age; + private int age; - private Integer recordCount; + private Integer recordCount; - public int getSampleId() { - return sampleId; - } + public int getSampleId() { + return sampleId; + } - public void setSampleId(int sampleId) { - this.sampleId = sampleId; - } + public void setSampleId(int sampleId) { + this.sampleId = sampleId; + } - public int getRank() { - return rank; - } + public int getRank() { + return rank; + } - public void setRank(int rank) { - this.rank = rank; - } + public void setRank(int rank) { + this.rank = rank; + } - public long getPersonId() { - return personId; - } + public long getPersonId() { + return personId; + } - public void setPersonId(long personId) { - this.personId = personId; - } + public void setPersonId(long personId) { + this.personId = personId; + } - public long getGenderConceptId() { - return genderConceptId; - } + public long getGenderConceptId() { + return genderConceptId; + } - public void setGenderConceptId(long genderConceptId) { - this.genderConceptId = genderConceptId; - } + public void setGenderConceptId(long genderConceptId) { + this.genderConceptId = genderConceptId; + } - public int getAge() { - return age; - } + public int getAge() { + return age; + } - public void setAge(int age) { - this.age = age; - } + public void setAge(int age) { + this.age = age; + } - public Integer getRecordCount() { - return recordCount; - } + public Integer getRecordCount() { + return recordCount; + } - public void setRecordCount(Integer recordCount) { - this.recordCount = recordCount; - } + public void setRecordCount(Integer recordCount) { + this.recordCount = recordCount; + } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java index fb9796b495..0ef3b9fa5f 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleDTO.java @@ -8,127 +8,127 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class CohortSampleDTO { - /** Cohort sample ID. */ - private int id; - /** Cohort sample name. */ - private String name; - /** - * Actual sample size. This may be different from the size specified by the user if not enough - * persons could be found matching the criteria. - */ - private int size; - - /** - * Date that the sample was created. - */ - private Date createdDate; - - /** - * User that created the sample. If no login system is used, this is null. - */ - private UserDTO createdBy; - - /** - * Cohort definition ID that was sampled. - */ - private Integer cohortDefinitionId; - /** - * Source ID that was sampled. - */ - private Integer sourceId; - - /** - * Age criteria used to create the sample. - */ - private SampleParametersDTO.AgeDTO age; - - /** - * Gender criteria used to create the sample. - */ - private SampleParametersDTO.GenderDTO gender; - - /** - * Actually sampled elements. - */ - private List elements; - - public int getId() { - return id; - } - - public void setId(int id) { - this.id = id; - } - - public Date getCreatedDate() { - return createdDate; - } - - public void setCreatedDate(Date createdDate) { - this.createdDate = createdDate; - } - - public UserDTO getCreatedBy() { - return createdBy; - } - - public void setCreatedBy(UserDTO createdBy) { - this.createdBy = createdBy; - } - - public Integer getCohortDefinitionId() { - return cohortDefinitionId; - } - - public void setCohortDefinitionId(Integer cohortDefinitionId) { - this.cohortDefinitionId = cohortDefinitionId; - } - - public Integer getSourceId() { - return sourceId; - } - - public void setSourceId(Integer sourceId) { - this.sourceId = sourceId; - } - - public int getSize() { - return size; - } - - public void setSize(int size) { - this.size = size; - } - - public List getElements() { - return elements; - } - - public void setElements(List elements) { - this.elements = elements; - } - - public SampleParametersDTO.AgeDTO getAge() { - return age; - } - - public void setAge(SampleParametersDTO.AgeDTO age) { - this.age = age; - } - - public SampleParametersDTO.GenderDTO getGender() { - return gender; - } - - public void setGender(SampleParametersDTO.GenderDTO gender) { - this.gender = gender; - } - - public void setName(String name) { - this.name = name; - } - - public String getName() { - return name; - } + /** Cohort sample ID. */ + private int id; + /** Cohort sample name. */ + private String name; + /** + * Actual sample size. This may be different from the size specified by the user if not enough + * persons could be found matching the criteria. + */ + private int size; + + /** + * Date that the sample was created. + */ + private Date createdDate; + + /** + * User that created the sample. If no login system is used, this is null. + */ + private UserDTO createdBy; + + /** + * Cohort definition ID that was sampled. + */ + private Integer cohortDefinitionId; + /** + * Source ID that was sampled. + */ + private Integer sourceId; + + /** + * Age criteria used to create the sample. + */ + private SampleParametersDTO.AgeDTO age; + + /** + * Gender criteria used to create the sample. + */ + private SampleParametersDTO.GenderDTO gender; + + /** + * Actually sampled elements. + */ + private List elements; + + public int getId() { + return id; + } + + public void setId(int id) { + this.id = id; + } + + public Date getCreatedDate() { + return createdDate; + } + + public void setCreatedDate(Date createdDate) { + this.createdDate = createdDate; + } + + public UserDTO getCreatedBy() { + return createdBy; + } + + public void setCreatedBy(UserDTO createdBy) { + this.createdBy = createdBy; + } + + public Integer getCohortDefinitionId() { + return cohortDefinitionId; + } + + public void setCohortDefinitionId(Integer cohortDefinitionId) { + this.cohortDefinitionId = cohortDefinitionId; + } + + public Integer getSourceId() { + return sourceId; + } + + public void setSourceId(Integer sourceId) { + this.sourceId = sourceId; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public List getElements() { + return elements; + } + + public void setElements(List elements) { + this.elements = elements; + } + + public SampleParametersDTO.AgeDTO getAge() { + return age; + } + + public void setAge(SampleParametersDTO.AgeDTO age) { + this.age = age; + } + + public SampleParametersDTO.GenderDTO getGender() { + return gender; + } + + public void setGender(SampleParametersDTO.GenderDTO gender) { + this.gender = gender; + } + + public void setName(String name) { + this.name = name; + } + + public String getName() { + return name; + } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleListDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleListDTO.java index 4da870edf5..b16b90a0e3 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleListDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/CohortSampleListDTO.java @@ -5,49 +5,49 @@ import java.util.List; public class CohortSampleListDTO { - private int cohortDefinitionId; - private int sourceId; - private GenerationStatus generationStatus; - private boolean isValid; - private List samples; - - public int getCohortDefinitionId() { - return cohortDefinitionId; - } - - public void setCohortDefinitionId(int cohortDefinitionId) { - this.cohortDefinitionId = cohortDefinitionId; - } - - public int getSourceId() { - return sourceId; - } - - public void setSourceId(int sourceId) { - this.sourceId = sourceId; - } - - public GenerationStatus getGenerationStatus() { - return generationStatus; - } - - public void setGenerationStatus(GenerationStatus generationStatus) { - this.generationStatus = generationStatus; - } - - public boolean isValid() { - return isValid; - } - - public void setIsValid(boolean valid) { - isValid = valid; - } - - public List getSamples() { - return samples; - } - - public void setSamples(List samples) { - this.samples = samples; - } + private int cohortDefinitionId; + private int sourceId; + private GenerationStatus generationStatus; + private boolean isValid; + private List samples; + + public int getCohortDefinitionId() { + return cohortDefinitionId; + } + + public void setCohortDefinitionId(int cohortDefinitionId) { + this.cohortDefinitionId = cohortDefinitionId; + } + + public int getSourceId() { + return sourceId; + } + + public void setSourceId(int sourceId) { + this.sourceId = sourceId; + } + + public GenerationStatus getGenerationStatus() { + return generationStatus; + } + + public void setGenerationStatus(GenerationStatus generationStatus) { + this.generationStatus = generationStatus; + } + + public boolean isValid() { + return isValid; + } + + public void setIsValid(boolean valid) { + isValid = valid; + } + + public List getSamples() { + return samples; + } + + public void setSamples(List samples) { + this.samples = samples; + } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java index aea5872196..9956f23640 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java @@ -4,79 +4,79 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class SampleElementDTO { - /** - * Sample ID that this element belongs to. May be null if this object is part of a - * {@link CohortSampleDTO} object. - */ - private Integer sampleId; - - /** - * Rank of the object within the sample. This establishes order between elements. - */ - private int rank; - - /** - * Person ID of the element. - */ - private long personId; - - /** - * Gender ID of the person. - */ - private long genderConceptId; - - /** - * Age of the person. - */ - private int age; - - private Integer recordCount; - - public Integer getSampleId() { - return sampleId; - } - - public void setSampleId(Integer sampleId) { - this.sampleId = sampleId; - } - - public int getRank() { - return rank; - } - - public void setRank(int rank) { - this.rank = rank; - } - - public long getPersonId() { - return personId; - } - - public void setPersonId(long personId) { - this.personId = personId; - } - - public long getGenderConceptId() { - return genderConceptId; - } - - public void setGenderConceptId(long genderConceptId) { - this.genderConceptId = genderConceptId; - } - - public int getAge() { - return age; - } - - public void setAge(int age) { - this.age = age; - } - - public Integer getRecordCount() { - return recordCount; - } - - public void setRecordCount(Integer recordCount) { - this.recordCount = recordCount; - } + /** + * Sample ID that this element belongs to. May be null if this object is part of a + * {@link CohortSampleDTO} object. + */ + private Integer sampleId; + + /** + * Rank of the object within the sample. This establishes order between elements. + */ + private int rank; + + /** + * Person ID of the element. + */ + private long personId; + + /** + * Gender ID of the person. + */ + private long genderConceptId; + + /** + * Age of the person. + */ + private int age; + + private Integer recordCount; + + public Integer getSampleId() { + return sampleId; + } + + public void setSampleId(Integer sampleId) { + this.sampleId = sampleId; + } + + public int getRank() { + return rank; + } + + public void setRank(int rank) { + this.rank = rank; + } + + public long getPersonId() { + return personId; + } + + public void setPersonId(long personId) { + this.personId = personId; + } + + public long getGenderConceptId() { + return genderConceptId; + } + + public void setGenderConceptId(long genderConceptId) { + this.genderConceptId = genderConceptId; + } + + public int getAge() { + return age; + } + + public void setAge(int age) { + this.age = age; + } + + public Integer getRecordCount() { + return recordCount; + } + + public void setRecordCount(Integer recordCount) { + this.recordCount = recordCount; + } } diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java index ab850c490e..d60322591a 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleParametersDTO.java @@ -10,248 +10,248 @@ import java.util.stream.Stream; public class SampleParametersDTO { - private static final int SIZE_MAX = 500; - private static final int AGE_MAX = 500; - - /** Sample size. */ - private int size; - /** Sample name. */ - private String name; - - /** Gender criteria. */ - private GenderDTO gender; - - /** Age criteria. */ - private AgeDTO age; - - /** - * Validate this DTO. - * @throws BadRequestException if the DTO is not valid. - */ - public void validate() { - if (name == null) { - throw new BadRequestException("Sample must have a name"); - } - if (size <= 0) { - throw new BadRequestException("sample parameter size must fall in the range (1, " + SIZE_MAX + ")"); - } - if (size > SIZE_MAX) { - throw new BadRequestException("sample parameter size must fall in the range (1, " + SIZE_MAX + ")"); - } - if (age != null && !age.validate()) { - age = null; - } - if (gender != null && !gender.validate()) { - gender = null; - } - } - - public AgeDTO getAge() { - return age; - } - - public void setAge(AgeDTO age) { - this.age = age; - } - - public GenderDTO getGender() { - return gender; - } - - public void setGender(GenderDTO gender) { - this.gender = gender; - } - - public int getSize() { - return size; - } - - public void setSize(int size) { - this.size = size; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public enum AgeMode { - LESS_THAN("lessThan"), - LESS_THAN_OR_EQUAL("lessThanOrEqual"), - GREATER_THAN("greaterThan"), - GREATER_THAN_OR_EQUAL("greaterThanOrEqual"), - EQUAL_TO("equalTo"), - BETWEEN("between"), - NOT_BETWEEN("notBetween"); - - private final String serialName; - - AgeMode(String serialName) { - this.serialName = serialName; - } - - @JsonValue - public String getSerialName() { - return serialName; - } - - public static AgeMode fromSerialName(String name) { - return Stream.of(SampleParametersDTO.AgeMode.values()) - .filter(mode -> mode.getSerialName().equals(name)) - .findFirst() - .orElse(null); - } - } - - @JsonInclude(JsonInclude.Include.NON_NULL) - public static class GenderDTO { - public final static int GENDER_MALE_CONCEPT_ID = 8507; - public final static int GENDER_FEMALE_CONCEPT_ID = 8532; - - private Integer conceptId; - - private List conceptIds; - - private boolean otherNonBinary = false; - - /** - * Validate this DTO. - * @return true if this DTO contains any information, false otherwise. - */ - public boolean validate() { - if (conceptIds == null) { - conceptIds = new ArrayList<>(); - } else if (conceptIds.contains(null)) { - conceptIds.removeIf(Objects::isNull); - } - if (conceptId != null) { - conceptIds.add(conceptId); - conceptId = null; - } - - if (!isOtherNonBinary() && conceptIds.isEmpty()) { - return false; - } - - if (isOtherNonBinary()) { - conceptIds.removeIf(i -> i != GENDER_MALE_CONCEPT_ID && i != GENDER_FEMALE_CONCEPT_ID); - } - return true; - } - - public Integer getConceptId() { - return conceptId; - } - - public void setConceptId(Integer conceptId) { - this.conceptId = conceptId; - } - - public List getConceptIds() { - return conceptIds; - } - - public void setConceptIds(List conceptIds) { - this.conceptIds = conceptIds; - } - - public boolean isOtherNonBinary() { - return otherNonBinary; - } - - public void setOtherNonBinary(boolean otherNonBinary) { - this.otherNonBinary = otherNonBinary; - } - } - - @JsonInclude(JsonInclude.Include.NON_NULL) - public static class AgeDTO { - private Integer min; - private Integer max; - private Integer value; - private AgeMode mode; - - /** - * Validate this DTO. - * @return true if this DTO contains any information, false otherwise. - * @throws BadRequestException if the DTO is not valid. - */ - public boolean validate() { - if (mode == null) { - if (min != null || max != null || value != null) { - throw new BadRequestException("Cannot specify age without a mode to use age with."); - } else { - return false; - } - } - switch (mode) { - case LESS_THAN: - case LESS_THAN_OR_EQUAL: - case GREATER_THAN: - case GREATER_THAN_OR_EQUAL: - case EQUAL_TO: - if (value == null) { - throw new BadRequestException("Cannot use single age comparison mode " + mode.getSerialName() + " without age property."); - } - if (min != null || max != null) { - throw new BadRequestException("Cannot use age range property with comparison mode " + mode.getSerialName() + "."); - } - break; - case BETWEEN: - case NOT_BETWEEN: - if (min == null || max == null) { - throw new BadRequestException("Cannot use age range comparison mode " + mode.getSerialName() + " without ageMin and ageMax properties."); - } - if (value != null) { - throw new BadRequestException("Cannot use single age property with comparison mode " + mode.getSerialName() + "."); - } - if (min < 0) { - throw new BadRequestException("Minimum age may not be less than 0"); - } - if (max >= AGE_MAX) { - throw new BadRequestException("Maximum age must be smaller than " + AGE_MAX); - } - if (min > max) { - throw new BadRequestException("Maximum age " + max + " may not be less than minimum age " + min); - } - break; - } - return true; - } - - public Integer getMin() { - return min; - } - - public void setMin(Integer min) { - this.min = min; - } - - public Integer getMax() { - return max; - } - - public void setMax(Integer max) { - this.max = max; - } - - public Integer getValue() { - return value; - } - - public void setValue(Integer value) { - this.value = value; - } - - public AgeMode getMode() { - return mode; - } - - public void setMode(AgeMode mode) { - this.mode = mode; - } - } + private static final int SIZE_MAX = 500; + private static final int AGE_MAX = 500; + + /** Sample size. */ + private int size; + /** Sample name. */ + private String name; + + /** Gender criteria. */ + private GenderDTO gender; + + /** Age criteria. */ + private AgeDTO age; + + /** + * Validate this DTO. + * @throws BadRequestException if the DTO is not valid. + */ + public void validate() { + if (name == null) { + throw new BadRequestException("Sample must have a name"); + } + if (size <= 0) { + throw new BadRequestException("sample parameter size must fall in the range (1, " + SIZE_MAX + ")"); + } + if (size > SIZE_MAX) { + throw new BadRequestException("sample parameter size must fall in the range (1, " + SIZE_MAX + ")"); + } + if (age != null && !age.validate()) { + age = null; + } + if (gender != null && !gender.validate()) { + gender = null; + } + } + + public AgeDTO getAge() { + return age; + } + + public void setAge(AgeDTO age) { + this.age = age; + } + + public GenderDTO getGender() { + return gender; + } + + public void setGender(GenderDTO gender) { + this.gender = gender; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public enum AgeMode { + LESS_THAN("lessThan"), + LESS_THAN_OR_EQUAL("lessThanOrEqual"), + GREATER_THAN("greaterThan"), + GREATER_THAN_OR_EQUAL("greaterThanOrEqual"), + EQUAL_TO("equalTo"), + BETWEEN("between"), + NOT_BETWEEN("notBetween"); + + private final String serialName; + + AgeMode(String serialName) { + this.serialName = serialName; + } + + @JsonValue + public String getSerialName() { + return serialName; + } + + public static AgeMode fromSerialName(String name) { + return Stream.of(SampleParametersDTO.AgeMode.values()) + .filter(mode -> mode.getSerialName().equals(name)) + .findFirst() + .orElse(null); + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class GenderDTO { + public final static int GENDER_MALE_CONCEPT_ID = 8507; + public final static int GENDER_FEMALE_CONCEPT_ID = 8532; + + private Integer conceptId; + + private List conceptIds; + + private boolean otherNonBinary = false; + + /** + * Validate this DTO. + * @return true if this DTO contains any information, false otherwise. + */ + public boolean validate() { + if (conceptIds == null) { + conceptIds = new ArrayList<>(); + } else if (conceptIds.contains(null)) { + conceptIds.removeIf(Objects::isNull); + } + if (conceptId != null) { + conceptIds.add(conceptId); + conceptId = null; + } + + if (!isOtherNonBinary() && conceptIds.isEmpty()) { + return false; + } + + if (isOtherNonBinary()) { + conceptIds.removeIf(i -> i != GENDER_MALE_CONCEPT_ID && i != GENDER_FEMALE_CONCEPT_ID); + } + return true; + } + + public Integer getConceptId() { + return conceptId; + } + + public void setConceptId(Integer conceptId) { + this.conceptId = conceptId; + } + + public List getConceptIds() { + return conceptIds; + } + + public void setConceptIds(List conceptIds) { + this.conceptIds = conceptIds; + } + + public boolean isOtherNonBinary() { + return otherNonBinary; + } + + public void setOtherNonBinary(boolean otherNonBinary) { + this.otherNonBinary = otherNonBinary; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class AgeDTO { + private Integer min; + private Integer max; + private Integer value; + private AgeMode mode; + + /** + * Validate this DTO. + * @return true if this DTO contains any information, false otherwise. + * @throws BadRequestException if the DTO is not valid. + */ + public boolean validate() { + if (mode == null) { + if (min != null || max != null || value != null) { + throw new BadRequestException("Cannot specify age without a mode to use age with."); + } else { + return false; + } + } + switch (mode) { + case LESS_THAN: + case LESS_THAN_OR_EQUAL: + case GREATER_THAN: + case GREATER_THAN_OR_EQUAL: + case EQUAL_TO: + if (value == null) { + throw new BadRequestException("Cannot use single age comparison mode " + mode.getSerialName() + " without age property."); + } + if (min != null || max != null) { + throw new BadRequestException("Cannot use age range property with comparison mode " + mode.getSerialName() + "."); + } + break; + case BETWEEN: + case NOT_BETWEEN: + if (min == null || max == null) { + throw new BadRequestException("Cannot use age range comparison mode " + mode.getSerialName() + " without ageMin and ageMax properties."); + } + if (value != null) { + throw new BadRequestException("Cannot use single age property with comparison mode " + mode.getSerialName() + "."); + } + if (min < 0) { + throw new BadRequestException("Minimum age may not be less than 0"); + } + if (max >= AGE_MAX) { + throw new BadRequestException("Maximum age must be smaller than " + AGE_MAX); + } + if (min > max) { + throw new BadRequestException("Maximum age " + max + " may not be less than minimum age " + min); + } + break; + } + return true; + } + + public Integer getMin() { + return min; + } + + public void setMin(Integer min) { + this.min = min; + } + + public Integer getMax() { + return max; + } + + public void setMax(Integer max) { + this.max = max; + } + + public Integer getValue() { + return value; + } + + public void setValue(Integer value) { + this.value = value; + } + + public AgeMode getMode() { + return mode; + } + + public void setMode(AgeMode mode) { + this.mode = mode; + } + } } diff --git a/src/main/java/org/ohdsi/webapi/service/AbstractDaoService.java b/src/main/java/org/ohdsi/webapi/service/AbstractDaoService.java index 18c72ceda9..849c94a5d3 100644 --- a/src/main/java/org/ohdsi/webapi/service/AbstractDaoService.java +++ b/src/main/java/org/ohdsi/webapi/service/AbstractDaoService.java @@ -8,8 +8,6 @@ import org.apache.commons.lang3.StringUtils; import org.ohdsi.webapi.GenerationStatus; import org.ohdsi.webapi.IExecutionInfo; -import org.ohdsi.webapi.cohortsample.CohortSample; -import org.ohdsi.webapi.cohortsample.dto.CohortSampleDTO; import org.ohdsi.webapi.common.sensitiveinfo.AbstractAdminService; import org.ohdsi.webapi.conceptset.ConceptSetItemRepository; import org.ohdsi.webapi.conceptset.ConceptSetRepository; @@ -31,7 +29,6 @@ import org.springframework.jdbc.datasource.DriverManagerDataSource; import org.springframework.transaction.support.TransactionTemplate; -import javax.ws.rs.NotFoundException; import java.io.File; import java.io.IOException; import java.sql.ResultSet; @@ -294,4 +291,5 @@ protected UserEntity getCurrentUser() { protected String getCurrentUserLogin() { return security.getSubject(); } + } diff --git a/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java b/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java index 025a0e245b..01be555430 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortDefinitionService.java @@ -151,8 +151,8 @@ public class CohortDefinitionService extends AbstractDaoService { @Autowired private ObjectMapper objectMapper; - @Autowired - private CohortSamplingService samplingService; + @Autowired + private CohortSamplingService samplingService; @Autowired private ApplicationEventPublisher eventPublisher; @@ -600,7 +600,7 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { }); }); cohortDefinitionRepository.delete(def); - samplingService.launchDeleteSamplesTasklet(id); + samplingService.launchDeleteSamplesTasklet(id); } else { log.warn("Failed to delete Cohort Definition with ID = {}", id); } @@ -621,13 +621,15 @@ public void doInTransactionWithoutResult(final TransactionStatus status) { .tasklet(cleanupTasklet) .build(); - CleanupCohortSamplesTasklet cleanupSamplesTasklet = samplingService.createDeleteSamplesTasklet(); - Step cleanupSamplesStep = stepBuilders.get("cohortDefinition.cleanupSamples") - .tasklet(cleanupSamplesTasklet) - .build(); + CleanupCohortSamplesTasklet cleanupSamplesTasklet = samplingService.createDeleteSamplesTasklet(); + + Step cleanupSamplesStep = stepBuilders.get("cohortDefinition.cleanupSamples") + .tasklet(cleanupSamplesTasklet) + .build(); SimpleJobBuilder cleanupJobBuilder = jobBuilders.get("cleanupCohort") - .start(cleanupStep); + .start(cleanupStep) + .next(cleanupSamplesStep); Job cleanupCohortJob = cleanupJobBuilder.build(); diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index 23952dc3f3..63952ba62f 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -33,132 +33,132 @@ @Component @Produces(MediaType.APPLICATION_JSON) public class CohortSampleService { - private final CohortDefinitionRepository cohortDefinitionRepository; - private final CohortGenerationInfoRepository generationInfoRepository; - private final CohortSamplingService samplingService; - private final SourceRepository sourceRepository; - - @Autowired - public CohortSampleService( - CohortSamplingService samplingService, - SourceRepository sourceRepository, - CohortDefinitionRepository cohortDefinitionRepository, - CohortGenerationInfoRepository generationInfoRepository - ) { - this.samplingService = samplingService; - this.sourceRepository = sourceRepository; - this.cohortDefinitionRepository = cohortDefinitionRepository; - this.generationInfoRepository = generationInfoRepository; - } - - @Path("/{cohortDefinitionId}/{sourceKey}") - @GET - public CohortSampleListDTO listCohortSamples( - @PathParam("cohortDefinitionId") int cohortDefinitionId, - @PathParam("sourceKey") String sourceKey - ) { - Source source = getSource(sourceKey); - CohortSampleListDTO result = new CohortSampleListDTO(); - - result.setCohortDefinitionId(cohortDefinitionId); - result.setSourceId(source.getId()); - - CohortGenerationInfo generationInfo = generationInfoRepository.findOne( - new CohortGenerationInfoId(cohortDefinitionId, source.getId())); - result.setGenerationStatus(generationInfo != null ? generationInfo.getStatus() : null); - result.setIsValid(generationInfo != null && generationInfo.isIsValid()); - - result.setSamples(this.samplingService.listSamples(cohortDefinitionId, source.getId())); - - return result; - } - - @Path("/{cohortDefinitionId}/{sourceKey}/{sampleId}") - @GET - public CohortSampleDTO getCohortSample( - @PathParam("sampleId") Integer sampleId, - @DefaultValue("recordCount") @QueryParam("fields") String fields - ) { - List returnFields = Arrays.asList(fields.split(",")); - boolean withRecordCounts = returnFields.contains("recordCount"); - return this.samplingService.getSample(sampleId, withRecordCounts); - } - - @Path("/has-samples/{cohortDefinitionId}") - @GET - public Map hasSamples( - @PathParam("cohortDefinitionId") int cohortDefinitionId - ) { - int nSamples = this.samplingService.countSamples(cohortDefinitionId); - return Collections.singletonMap("hasSamples", nSamples > 0); - } - - @Path("/has-samples/{cohortDefinitionId}/{sourceKey}") - @GET - public Map hasSamples( - @PathParam("sourceKey") String sourceKey, - @PathParam("cohortDefinitionId") int cohortDefinitionId - ) { - Source source = getSource(sourceKey); - int nSamples = this.samplingService.countSamples(cohortDefinitionId, source.getId()); - return Collections.singletonMap("hasSamples", nSamples > 0); - } - - @Path("/{cohortDefinitionId}/{sourceKey}") - @POST - @Consumes(MediaType.APPLICATION_JSON) - public CohortSampleDTO createCohortSample( - @PathParam("sourceKey") String sourceKey, - @PathParam("cohortDefinitionId") int cohortDefinitionId, - SampleParametersDTO sampleParameters - ) { - sampleParameters.validate(); - Source source = getSource(sourceKey); - if (cohortDefinitionRepository.findOne(cohortDefinitionId) == null) { - throw new NotFoundException("Cohort definition " + cohortDefinitionId + " does not exist."); - } - CohortGenerationInfo generationInfo = generationInfoRepository.findOne( - new CohortGenerationInfoId(cohortDefinitionId, source.getId())); - if (generationInfo == null || generationInfo.getStatus() != GenerationStatus.COMPLETE) { - throw new BadRequestException("Cohort is not yet generated"); - } - return samplingService.createSample(source, cohortDefinitionId, sampleParameters); - } - - @Path("/{cohortDefinitionId}/{sourceKey}/{sampleId}") - @DELETE - public Response deleteCohortSample( - @PathParam("sourceKey") String sourceKey, - @PathParam("cohortDefinitionId") int cohortDefinitionId, - @PathParam("sampleId") int sampleId - ) { - Source source = getSource(sourceKey); - if (cohortDefinitionRepository.findOne(cohortDefinitionId) == null) { - throw new NotFoundException("Cohort definition " + cohortDefinitionId + " does not exist."); - } - samplingService.deleteSample(cohortDefinitionId, source, sampleId); - return Response.status(Response.Status.NO_CONTENT).build(); - } - - @Path("/{cohortDefinitionId}/{sourceKey}") - @DELETE - public Response deleteCohortSamples( - @PathParam("sourceKey") String sourceKey, - @PathParam("cohortDefinitionId") int cohortDefinitionId - ) { - Source source = getSource(sourceKey); - if (cohortDefinitionRepository.findOne(cohortDefinitionId) == null) { - throw new NotFoundException("Cohort definition " + cohortDefinitionId + " does not exist."); - } - samplingService.launchDeleteSamplesTasklet(cohortDefinitionId, source.getId()); - return Response.status(Response.Status.ACCEPTED).build(); - } - - private Source getSource(String sourceKey) { - Source source = sourceRepository.findBySourceKey(sourceKey); - if (source == null) { - throw new NotFoundException("Source " + sourceKey + " does not exist"); - } - return source; - } + private final CohortDefinitionRepository cohortDefinitionRepository; + private final CohortGenerationInfoRepository generationInfoRepository; + private final CohortSamplingService samplingService; + private final SourceRepository sourceRepository; + + @Autowired + public CohortSampleService( + CohortSamplingService samplingService, + SourceRepository sourceRepository, + CohortDefinitionRepository cohortDefinitionRepository, + CohortGenerationInfoRepository generationInfoRepository + ) { + this.samplingService = samplingService; + this.sourceRepository = sourceRepository; + this.cohortDefinitionRepository = cohortDefinitionRepository; + this.generationInfoRepository = generationInfoRepository; + } + + @Path("/{cohortDefinitionId}/{sourceKey}") + @GET + public CohortSampleListDTO listCohortSamples( + @PathParam("cohortDefinitionId") int cohortDefinitionId, + @PathParam("sourceKey") String sourceKey + ) { + Source source = getSource(sourceKey); + CohortSampleListDTO result = new CohortSampleListDTO(); + + result.setCohortDefinitionId(cohortDefinitionId); + result.setSourceId(source.getId()); + + CohortGenerationInfo generationInfo = generationInfoRepository.findOne( + new CohortGenerationInfoId(cohortDefinitionId, source.getId())); + result.setGenerationStatus(generationInfo != null ? generationInfo.getStatus() : null); + result.setIsValid(generationInfo != null && generationInfo.isIsValid()); + + result.setSamples(this.samplingService.listSamples(cohortDefinitionId, source.getId())); + + return result; + } + + @Path("/{cohortDefinitionId}/{sourceKey}/{sampleId}") + @GET + public CohortSampleDTO getCohortSample( + @PathParam("sampleId") Integer sampleId, + @DefaultValue("recordCount") @QueryParam("fields") String fields + ) { + List returnFields = Arrays.asList(fields.split(",")); + boolean withRecordCounts = returnFields.contains("recordCount"); + return this.samplingService.getSample(sampleId, withRecordCounts); + } + + @Path("/has-samples/{cohortDefinitionId}") + @GET + public Map hasSamples( + @PathParam("cohortDefinitionId") int cohortDefinitionId + ) { + int nSamples = this.samplingService.countSamples(cohortDefinitionId); + return Collections.singletonMap("hasSamples", nSamples > 0); + } + + @Path("/has-samples/{cohortDefinitionId}/{sourceKey}") + @GET + public Map hasSamples( + @PathParam("sourceKey") String sourceKey, + @PathParam("cohortDefinitionId") int cohortDefinitionId + ) { + Source source = getSource(sourceKey); + int nSamples = this.samplingService.countSamples(cohortDefinitionId, source.getId()); + return Collections.singletonMap("hasSamples", nSamples > 0); + } + + @Path("/{cohortDefinitionId}/{sourceKey}") + @POST + @Consumes(MediaType.APPLICATION_JSON) + public CohortSampleDTO createCohortSample( + @PathParam("sourceKey") String sourceKey, + @PathParam("cohortDefinitionId") int cohortDefinitionId, + SampleParametersDTO sampleParameters + ) { + sampleParameters.validate(); + Source source = getSource(sourceKey); + if (cohortDefinitionRepository.findOne(cohortDefinitionId) == null) { + throw new NotFoundException("Cohort definition " + cohortDefinitionId + " does not exist."); + } + CohortGenerationInfo generationInfo = generationInfoRepository.findOne( + new CohortGenerationInfoId(cohortDefinitionId, source.getId())); + if (generationInfo == null || generationInfo.getStatus() != GenerationStatus.COMPLETE) { + throw new BadRequestException("Cohort is not yet generated"); + } + return samplingService.createSample(source, cohortDefinitionId, sampleParameters); + } + + @Path("/{cohortDefinitionId}/{sourceKey}/{sampleId}") + @DELETE + public Response deleteCohortSample( + @PathParam("sourceKey") String sourceKey, + @PathParam("cohortDefinitionId") int cohortDefinitionId, + @PathParam("sampleId") int sampleId + ) { + Source source = getSource(sourceKey); + if (cohortDefinitionRepository.findOne(cohortDefinitionId) == null) { + throw new NotFoundException("Cohort definition " + cohortDefinitionId + " does not exist."); + } + samplingService.deleteSample(cohortDefinitionId, source, sampleId); + return Response.status(Response.Status.NO_CONTENT).build(); + } + + @Path("/{cohortDefinitionId}/{sourceKey}") + @DELETE + public Response deleteCohortSamples( + @PathParam("sourceKey") String sourceKey, + @PathParam("cohortDefinitionId") int cohortDefinitionId + ) { + Source source = getSource(sourceKey); + if (cohortDefinitionRepository.findOne(cohortDefinitionId) == null) { + throw new NotFoundException("Cohort definition " + cohortDefinitionId + " does not exist."); + } + samplingService.launchDeleteSamplesTasklet(cohortDefinitionId, source.getId()); + return Response.status(Response.Status.ACCEPTED).build(); + } + + private Source getSource(String sourceKey) { + Source source = sourceRepository.findBySourceKey(sourceKey); + if (source == null) { + throw new NotFoundException("Source " + sourceKey + " does not exist"); + } + return source; + } } From 1019d913f8ab486931ba5d4aa7fc307f5f489d16 Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Wed, 28 Oct 2020 11:17:14 +0100 Subject: [PATCH 28/33] Make person ID a String for JS compatibility --- .../ohdsi/webapi/cohortsample/CohortSamplingService.java | 2 +- .../org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index af944d5cd4..e2d4f0943e 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -259,7 +259,7 @@ private List sampleElementToDTO(List elements) .map(el -> { SampleElementDTO elementDTO = new SampleElementDTO(); elementDTO.setRank(el.getRank()); - elementDTO.setPersonId(el.getPersonId()); + elementDTO.setPersonId(String.valueOf(el.getPersonId())); elementDTO.setAge(el.getAge()); elementDTO.setGenderConceptId(el.getGenderConceptId()); elementDTO.setRecordCount(el.getRecordCount()); diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java index 9956f23640..4191f01569 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/dto/SampleElementDTO.java @@ -18,7 +18,7 @@ public class SampleElementDTO { /** * Person ID of the element. */ - private long personId; + private String personId; /** * Gender ID of the person. @@ -48,11 +48,11 @@ public void setRank(int rank) { this.rank = rank; } - public long getPersonId() { + public String getPersonId() { return personId; } - public void setPersonId(long personId) { + public void setPersonId(String personId) { this.personId = personId; } From 681619b688aea96de0b34da0fa8bd3edc22f159d Mon Sep 17 00:00:00 2001 From: Maxim Moinat Date: Fri, 30 Oct 2020 09:35:21 +0100 Subject: [PATCH 29/33] fix corrupt bigint person_id --- .../org/ohdsi/webapi/cohortsample/CohortSamplingService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index e2d4f0943e..f2ad516cb9 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -464,7 +464,7 @@ public SampleElement mapRow(ResultSet rs, int rowNum) throws SQLException { SampleElement sample = new SampleElement(); sample.setRank(rs.getInt("rank")); sample.setSampleId(rs.getInt("cohort_sample_id")); - sample.setPersonId(rs.getInt("person_id")); + sample.setPersonId(rs.getLong("person_id")); sample.setGenderConceptId(rs.getInt("gender_concept_id")); sample.setAge(rs.getInt("age")); if (optionalFields.contains("record_count")) { From 329b717e0a4ad27ecebbdf07f37f3db62a3d715d Mon Sep 17 00:00:00 2001 From: Chris Knoll Date: Wed, 4 Nov 2020 16:21:25 -0500 Subject: [PATCH 30/33] Rename rank to rank_value. Remove default to include event counts. --- .../org/ohdsi/webapi/cohortsample/CohortSamplingService.java | 2 +- src/main/java/org/ohdsi/webapi/service/CohortSampleService.java | 2 +- src/main/resources/ddl/results/cohort_sample_element.sql | 2 +- .../resources/cohortsample/sql/findElementsByCohortSampleId.sql | 2 +- .../cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql | 2 +- .../resources/cohortsample/sql/insertSampleElement.sql | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index f2ad516cb9..4815df5978 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -462,7 +462,7 @@ private static class CohortSampleElementRowMapper implements RowMapper returnFields = Arrays.asList(fields.split(",")); boolean withRecordCounts = returnFields.contains("recordCount"); diff --git a/src/main/resources/ddl/results/cohort_sample_element.sql b/src/main/resources/ddl/results/cohort_sample_element.sql index a26f3ef202..568c057214 100644 --- a/src/main/resources/ddl/results/cohort_sample_element.sql +++ b/src/main/resources/ddl/results/cohort_sample_element.sql @@ -1,7 +1,7 @@ IF OBJECT_ID('@results_schema.cohort_sample_element', 'U') IS NULL CREATE TABLE @results_schema.cohort_sample_element( cohort_sample_id int NOT NULL, - rank int NOT NULL, + rank_value int NOT NULL, person_id bigint NOT NULL, age int, gender_concept_id int diff --git a/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql index 4a9d0005f6..04d35949d9 100644 --- a/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql +++ b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleId.sql @@ -1,5 +1,5 @@ SELECT * FROM @results_schema.cohort_sample_element s WHERE s.cohort_sample_id = @cohortSampleId -ORDER BY s.rank +ORDER BY s.rank_value ; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql index fe8753aabb..f1cb7b4e87 100644 --- a/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql +++ b/src/main/resources/resources/cohortsample/sql/findElementsByCohortSampleIdWithCounts.sql @@ -40,5 +40,5 @@ SELECT *, where c.person_id = s.person_id)) as record_count FROM @results_schema.cohort_sample_element s WHERE s.cohort_sample_id = @cohortSampleId -ORDER BY s.rank +ORDER BY s.rank_value ; \ No newline at end of file diff --git a/src/main/resources/resources/cohortsample/sql/insertSampleElement.sql b/src/main/resources/resources/cohortsample/sql/insertSampleElement.sql index d3f2bf80ad..ddaaff250e 100644 --- a/src/main/resources/resources/cohortsample/sql/insertSampleElement.sql +++ b/src/main/resources/resources/cohortsample/sql/insertSampleElement.sql @@ -1,3 +1,3 @@ -INSERT INTO @results_schema.cohort_sample_element (cohort_sample_id, rank, person_id, age, gender_concept_id) +INSERT INTO @results_schema.cohort_sample_element (cohort_sample_id, rank_value, person_id, age, gender_concept_id) VALUES (@cohortSampleId, @rank, @personId, @age, @genderConceptId) ; From 0dbb9f8fab9d49b0b93912995f4ed859360fe7b7 Mon Sep 17 00:00:00 2001 From: Chris Knoll Date: Fri, 6 Nov 2020 16:45:05 -0500 Subject: [PATCH 31/33] Added refresh function. --- .../cohortsample/CohortSamplingService.java | 35 +++++++++++++++++++ .../webapi/service/CohortSampleService.java | 6 +++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java index 4815df5978..eb47bd851e 100644 --- a/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java +++ b/src/main/java/org/ohdsi/webapi/cohortsample/CohortSamplingService.java @@ -32,9 +32,11 @@ import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.ohdsi.sql.SqlTranslate; import static org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO.GenderDTO.GENDER_FEMALE_CONCEPT_ID; import static org.ohdsi.webapi.cohortsample.dto.SampleParametersDTO.GenderDTO.GENDER_MALE_CONCEPT_ID; +import org.ohdsi.webapi.util.SourceUtils; /** * Service to do manage samples of a cohort definition. @@ -188,6 +190,39 @@ public CohortSampleDTO createSample(Source source, int cohortDefinitionId, Sampl return sampleToSampleDTO(sample, elements, true); } + /** + * Create a new sample in given source and cohort definition, using sample parameters. + * @param sampleId The sample to refresh + */ + public void refreshSample(Integer sampleId) { + + CohortSample sample = sampleRepository.findById(sampleId); + if (sample == null) { + throw new NotFoundException("Cohort sample with ID " + sampleId + " not found"); + } + Source source = getSourceRepository().findBySourceId(sample.getSourceId()); + + CohortSampleDTO sampleDto = sampleToSampleDTO(sample, null, true); + SampleParametersDTO sampleParamaters = new SampleParametersDTO(); + sampleParamaters.setAge(sampleDto.getAge()); + sampleParamaters.setGender(sampleDto.getGender()); + sampleParamaters.setSize(sampleDto.getSize()); + log.info("Sampling {} elements for cohort {}", sampleParamaters.getSize(), sample.getCohortDefinitionId()); + JdbcTemplate jdbcTemplate = getSourceJdbcTemplate(source); + final List elements = sampleElements(sampleParamaters, sample, jdbcTemplate, source); + + getTransactionTemplate().execute((TransactionCallback) transactionStatus -> { + String deleteSql = String.format( + "DELETE FROM %s.cohort_sample_element WHERE cohort_sample_id = %d;", + source.getTableQualifier(SourceDaimon.DaimonType.Results), + sample.getId()); + String translatedDeleteSql = SqlTranslate.translateSql(deleteSql, source.getSourceDialect(), null, null); + jdbcTemplate.update(translatedDeleteSql); + insertSampledElements(source, jdbcTemplate, sample.getId(), elements); + return null; + }); + } + /** Convert a given sample with given elements to a DTO. */ private CohortSampleDTO sampleToSampleDTO(CohortSample sample, List elements, boolean includeIds) { CohortSampleDTO sampleDTO = new CohortSampleDTO(); diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index 5fbf5e2117..e818eb8276 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -77,10 +77,14 @@ public CohortSampleListDTO listCohortSamples( @GET public CohortSampleDTO getCohortSample( @PathParam("sampleId") Integer sampleId, - @DefaultValue("") @QueryParam("fields") String fields + @DefaultValue("") @QueryParam("fields") String fields, + @QueryParam("refresh") boolean refresh ) { List returnFields = Arrays.asList(fields.split(",")); boolean withRecordCounts = returnFields.contains("recordCount"); + if (refresh) { + this.samplingService.refreshSample(sampleId); + } return this.samplingService.getSample(sampleId, withRecordCounts); } From 5f178702f935779e50ae779b98b503d1feb09338 Mon Sep 17 00:00:00 2001 From: Chris Knoll Date: Sun, 8 Nov 2020 22:24:42 -0500 Subject: [PATCH 32/33] Made refreshCohortSample POST. --- .../webapi/service/CohortSampleService.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java index e818eb8276..7120d17b1e 100644 --- a/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java +++ b/src/main/java/org/ohdsi/webapi/service/CohortSampleService.java @@ -77,14 +77,22 @@ public CohortSampleListDTO listCohortSamples( @GET public CohortSampleDTO getCohortSample( @PathParam("sampleId") Integer sampleId, - @DefaultValue("") @QueryParam("fields") String fields, - @QueryParam("refresh") boolean refresh + @DefaultValue("") @QueryParam("fields") String fields ) { List returnFields = Arrays.asList(fields.split(",")); boolean withRecordCounts = returnFields.contains("recordCount"); - if (refresh) { - this.samplingService.refreshSample(sampleId); - } + return this.samplingService.getSample(sampleId, withRecordCounts); + } + + @Path("/{cohortDefinitionId}/{sourceKey}/{sampleId}/refresh") + @POST + public CohortSampleDTO refreshCohortSample( + @PathParam("sampleId") Integer sampleId, + @DefaultValue("") @QueryParam("fields") String fields + ) { + List returnFields = Arrays.asList(fields.split(",")); + boolean withRecordCounts = returnFields.contains("recordCount"); + this.samplingService.refreshSample(sampleId); return this.samplingService.getSample(sampleId, withRecordCounts); } From bdea2c10ca2a7ae24d42afb8189c9e4b1e397f2b Mon Sep 17 00:00:00 2001 From: Chris Knoll Date: Mon, 9 Nov 2020 04:10:03 -0500 Subject: [PATCH 33/33] Added security records for refresh to migration scripts. --- ...8.0.20200122173000__insert_cohort_sample_permissions.sql | 6 ++++-- ...8.0.20200122173000__insert_cohort_sample_permissions.sql | 6 ++++-- ...8.0.20200122173000__insert_cohort_sample_permissions.sql | 6 ++++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/main/resources/db/migration/oracle/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql b/src/main/resources/db/migration/oracle/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql index ab534ba256..e019c66c3d 100644 --- a/src/main/resources/db/migration/oracle/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql +++ b/src/main/resources/db/migration/oracle/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql @@ -3,7 +3,8 @@ INSERT INTO ${ohdsiSchema}.sec_permission(id, value, description) VALUES (${ohdsiSchema}.sec_permission_id_seq.nextval, 'cohortsample:*:*:*:get', 'Get single cohort samples'), (${ohdsiSchema}.sec_permission_id_seq.nextval, 'cohortsample:*:*:*:delete', 'Delete cohort sample'), (${ohdsiSchema}.sec_permission_id_seq.nextval, 'cohortsample:*:*:delete', 'Delete all cohort samples of a cohort.'), -(${ohdsiSchema}.sec_permission_id_seq.nextval, 'cohortsample:*:*:post', 'Create cohort sample'); +(${ohdsiSchema}.sec_permission_id_seq.nextval, 'cohortsample:*:*:post', 'Create cohort sample') +(${ohdsiSchema}.sec_permission_id_seq.nextval, 'cohortsample:*:*:*:refresh:post', 'Refresh cohort sample'); INSERT INTO ${ohdsiSchema}.sec_role_permission(id, role_id, permission_id) SELECT NEXT VALUE FOR ${ohdsiSchema}.sec_role_permission_sequence, sr.id, sp.id @@ -13,5 +14,6 @@ WHERE sp.value IN ( 'cohortsample:*:*:*:get', 'cohortsample:*:*:*:delete', 'cohortsample:*:*:delete', - 'cohortsample:*:*:post' + 'cohortsample:*:*:post', + 'cohortsample:*:*:*:refresh:post' ) AND sr.name IN ('Atlas users'); diff --git a/src/main/resources/db/migration/postgresql/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql b/src/main/resources/db/migration/postgresql/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql index aebead7c84..27a2bc586b 100644 --- a/src/main/resources/db/migration/postgresql/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql +++ b/src/main/resources/db/migration/postgresql/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql @@ -3,7 +3,8 @@ INSERT INTO ${ohdsiSchema}.sec_permission(id, value, description) VALUES (nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cohortsample:*:*:*:get', 'Get single cohort samples'), (nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cohortsample:*:*:*:delete', 'Delete cohort sample'), (nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cohortsample:*:*:delete', 'Delete all cohort samples of a cohort.'), -(nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cohortsample:*:*:post', 'Create cohort sample'); +(nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cohortsample:*:*:post', 'Create cohort sample'), +(nextval('${ohdsiSchema}.sec_permission_id_seq'), 'cohortsample:*:*:*:refresh:post', 'Refresh cohort sample'); INSERT INTO ${ohdsiSchema}.sec_role_permission(id, role_id, permission_id) SELECT nextval('${ohdsiSchema}.sec_role_permission_sequence'), sr.id, sp.id @@ -13,5 +14,6 @@ WHERE sp.value IN ( 'cohortsample:*:*:*:get', 'cohortsample:*:*:*:delete', 'cohortsample:*:*:delete', - 'cohortsample:*:*:post' + 'cohortsample:*:*:post', + 'cohortsample:*:*:*:refresh:post' ) AND sr.name IN ('Atlas users'); diff --git a/src/main/resources/db/migration/sqlserver/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql b/src/main/resources/db/migration/sqlserver/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql index 8ca257a1ca..73751c94a1 100644 --- a/src/main/resources/db/migration/sqlserver/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql +++ b/src/main/resources/db/migration/sqlserver/V2.8.0.20200122173000__insert_cohort_sample_permissions.sql @@ -3,7 +3,8 @@ INSERT INTO ${ohdsiSchema}.sec_permission(id, value, description) VALUES (NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:get', 'Get single cohort samples'), (NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:delete', 'Delete cohort sample'), (NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:delete', 'Delete all cohort samples of a cohort.'), -(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:post', 'Create cohort sample'); +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:post', 'Create cohort sample'), +(NEXT VALUE FOR ${ohdsiSchema}.sec_permission_id_seq, 'cohortsample:*:*:*:refresh:post', 'Refresh cohort sample'); INSERT INTO ${ohdsiSchema}.sec_role_permission(id, role_id, permission_id) SELECT NEXT VALUE FOR ${ohdsiSchema}.sec_role_permission_sequence, sr.id, sp.id @@ -13,5 +14,6 @@ WHERE sp.value IN ( 'cohortsample:*:*:*:get', 'cohortsample:*:*:*:delete', 'cohortsample:*:*:delete', - 'cohortsample:*:*:post' + 'cohortsample:*:*:post', + 'cohortsample:*:*:*:refresh:post' ) AND sr.name IN ('Atlas users');