diff --git a/cellbase-core/pom.xml b/cellbase-core/pom.xml index 1232a17874..83e89593ba 100644 --- a/cellbase-core/pom.xml +++ b/cellbase-core/pom.xml @@ -23,7 +23,6 @@ org.opencb.biodata biodata-tools - org.glassfish.jersey.core jersey-client @@ -32,7 +31,21 @@ org.slf4j slf4j-log4j12 - + + org.redisson + redisson + 3.5.3 + + + com.esotericsoftware.kryo + kryo + 2.24.0 + + + commons-codec + commons-codec + 1.10 + junit junit @@ -125,9 +138,11 @@ protobuf-java directly, you will be transitively depending on the protobuf-java version that grpc depends on. --> - com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + grpc-java - io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + io.grpc:protoc-gen-grpc-java:${grpc.version}:exe:${os.detected.classifier} + diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/api/DBAdaptorFactory.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/api/DBAdaptorFactory.java index 29af3d5f0f..1101185845 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/api/DBAdaptorFactory.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/api/DBAdaptorFactory.java @@ -17,7 +17,6 @@ package org.opencb.cellbase.core.api; import org.opencb.cellbase.core.config.CellBaseConfiguration; -import org.opencb.cellbase.core.config.Species; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -105,7 +104,7 @@ public void setConfiguration(CellBaseConfiguration cellBaseConfiguration) { public abstract ConservationDBAdaptor getConservationDBAdaptor(String species, String assembly); - protected Species getSpecies(String speciesName) { +/* protected Species getSpecies(String speciesName) { Species species = null; for (Species sp : cellBaseConfiguration.getAllSpecies()) { if (speciesName.equalsIgnoreCase(sp.getId()) || speciesName.equalsIgnoreCase(sp.getScientificName())) { @@ -128,6 +127,6 @@ protected String getAssembly(Species species, String assemblyName) { } } return assembly; - } + }*/ } diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java new file mode 100644 index 0000000000..3f54ed85bb --- /dev/null +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/cache/CacheManager.java @@ -0,0 +1,145 @@ +package org.opencb.cellbase.core.cache; + +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.StringUtils; +import org.opencb.cellbase.core.config.CacheProperties; +import org.opencb.cellbase.core.config.CellBaseConfiguration; +import org.opencb.commons.datastore.core.Query; +import org.opencb.commons.datastore.core.QueryOptions; +import org.opencb.commons.datastore.core.QueryResult; +import org.redisson.Redisson; +import org.redisson.api.RKeys; +import org.redisson.api.RMap; +import org.redisson.api.RedissonClient; +import org.redisson.client.RedisConnectionException; +import org.redisson.codec.JsonJacksonCodec; +import org.redisson.codec.KryoCodec; +import org.redisson.config.Config; + +import java.util.*; +import java.util.regex.Pattern; + + +/** + * Created by imedina on 20/10/16. + */ +public class CacheManager { + + private final String DATABASE = "cb:"; + private CellBaseConfiguration cellBaseConfiguration; + private Config redissonConfig; + private static RedissonClient redissonClient; + private CacheProperties cache; + private boolean redisState; + + public CacheManager() { + } + + public CacheManager(CellBaseConfiguration configuration) { + + if (configuration != null && configuration.getCache() != null) { + this.cellBaseConfiguration = configuration; + cache = configuration.getCache(); + redissonConfig = new Config(); + redisState = true; + String host = (StringUtils.isNotEmpty(cache.getHost())) + ? cache.getHost() : cache.DEFAULT_HOST; + redissonConfig.useSingleServer().setAddress("redis://" + host); + String codec = (StringUtils.isNotEmpty(cache.getSerialization())) + ? cache.getSerialization() : cache.DEFAULT_SERIALIZATION; + if ("Kryo".equalsIgnoreCase(codec)) { + redissonConfig.setCodec(new KryoCodec()); + } else if ("JSON".equalsIgnoreCase(codec)) { + redissonConfig.setCodec(new JsonJacksonCodec()); + } + } + } + + public QueryResult get(String key, Class clazz) { + + QueryResult queryResult = new QueryResult(); + if (isActive()) { + long start = System.currentTimeMillis(); + RMap> map = getRedissonClient().getMap(key); + Map> result = new HashMap>(); + Set set = new HashSet(Arrays.asList(0)); + + try { + result = map.getAll(set); + } catch (RedisConnectionException e) { + redisState = false; + queryResult.setWarningMsg("Unable to connect to Redis Cache, Please query WITHOUT Cache (Falling back to Database)"); + return queryResult; + } + if (!result.isEmpty()) { + queryResult = (QueryResult) result.get(0).get("result"); + // we are getting two objects, second one is query, not used at the moment + queryResult.setWarningMsg("Data is originated from Redis Cache !!!"); + queryResult.setDbTime((int) (System.currentTimeMillis() - start)); + } + } + return queryResult; + } + + public void set(String key, Query query, QueryResult queryResult) { + + if (isActive() && queryResult.getDbTime() > cache.getSlowThreshold()) { + RMap> map = getRedissonClient().getMap(key); + Map record = new HashMap(); + record.put("query", query); + record.put("result", queryResult); + try { + map.fastPut(0, record); + } catch (RedisConnectionException e) { + redisState = false; + queryResult.setWarningMsg("Unable to connect to Redis Cache, Please query WITHOUT Cache (Falling back to Database)"); + } + } + } + + public String createKey(String species, Query query, QueryOptions queryOptions) { + + queryOptions.remove("cache"); + StringBuilder key = new StringBuilder(DATABASE); + key.append(cellBaseConfiguration.getVersion()).append(":").append(species); + SortedMap> map = new TreeMap>(); + for (String item : query.keySet()) { + map.put(item.toLowerCase(), new TreeSet(query.getAsStringList(item))); + } + for (String item : queryOptions.keySet()) { + map.put(item.toLowerCase(), new TreeSet(queryOptions.getAsStringList(item))); + } + String sha1 = DigestUtils.sha1Hex(map.toString()); + key.append(":").append(sha1); + queryOptions.add("cache", "true"); + return key.toString(); + } + + public boolean isActive() { + return cache.isActive() && redisState; + } + + public void clear() { + RKeys redisKeys = getRedissonClient().getKeys(); + redisKeys.deleteByPattern(DATABASE + "*"); + } + + public void clear(Pattern pattern) { + RKeys redisKeys = getRedissonClient().getKeys(); + redisKeys.deleteByPattern(pattern.toString()); + } + + public void close() { + if (redissonClient != null) { + redissonClient.shutdown(); + redissonClient = null; + } + } + + private synchronized RedissonClient getRedissonClient() { + if (redissonClient == null) { + redissonClient = Redisson.create(redissonConfig); + } + return redissonClient; + } +} diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java new file mode 100644 index 0000000000..d7668ecd9f --- /dev/null +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CacheProperties.java @@ -0,0 +1,82 @@ +package org.opencb.cellbase.core.config; + +/** + * Created by imedina on 20/10/16. + */ +public class CacheProperties { + + /** + * This field contain the host and port, ie. host[:port]. + */ + private String host; + private boolean active; + + /** + * Accepted values are: JSON, Kryo. + */ + private String serialization; + + private int slowThreshold; + + public static final boolean DEFAULT_ACTVE = true; + public static final String DEFAULT_SERIALIZATION = "JSON"; + public static final String DEFAULT_HOST = "localhost:6379"; + + public CacheProperties() { + this("", DEFAULT_ACTVE, DEFAULT_SERIALIZATION, 50); + } + + public CacheProperties(String host, boolean active, String serialization, int slowThreshold) { + this.host = host; + this.active = active; + this.serialization = serialization; + this.slowThreshold = slowThreshold; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CacheProperties{"); + sb.append("host='").append(host).append('\''); + sb.append(", active=").append(active); + sb.append(", serialization='").append(serialization).append('\''); + sb.append(", slowThreshold=").append(slowThreshold); + sb.append('}'); + return sb.toString(); + } + + public String getHost() { + return host; + } + + public CacheProperties setHost(String host) { + this.host = host; + return this; + } + + public boolean isActive() { + return active; + } + + public CacheProperties setActive(boolean active) { + this.active = active; + return this; + } + + public String getSerialization() { + return serialization; + } + + public CacheProperties setSerialization(String serialization) { + this.serialization = serialization; + return this; + } + + public int getSlowThreshold() { + return slowThreshold; + } + + public CacheProperties setSlowThreshold(int slowThreshold) { + this.slowThreshold = slowThreshold; + return this; + } +} diff --git a/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CellBaseConfiguration.java b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CellBaseConfiguration.java index fdd4f67c01..1dbbb03a80 100644 --- a/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CellBaseConfiguration.java +++ b/cellbase-core/src/main/java/org/opencb/cellbase/core/config/CellBaseConfiguration.java @@ -33,6 +33,7 @@ public class CellBaseConfiguration { private String wiki; private String defaultOutdir; private Databases databases; + private CacheProperties cache; private DownloadProperties download; private SpeciesProperties species; @@ -42,28 +43,66 @@ public static CellBaseConfiguration load(InputStream configurationInputStream) t return jsonMapper.readValue(configurationInputStream, CellBaseConfiguration.class); } + /* public List getAllSpecies() { + List allSpecies = new ArrayList<>(); + allSpecies.addAll(species.getVertebrates()); + allSpecies.addAll(species.getMetazoa()); + allSpecies.addAll(species.getFungi()); + allSpecies.addAll(species.getProtist()); + allSpecies.addAll(species.getPlants()); + + return allSpecies; + }*/ + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("CellBaseConfiguration{"); + sb.append("version='").append(version).append('\''); + sb.append(", apiVersion='").append(apiVersion).append('\''); + sb.append(", wiki='").append(wiki).append('\''); + sb.append(", defaultOutdir='").append(defaultOutdir).append('\''); + sb.append(", databases=").append(databases); + sb.append(", cache=").append(cache); + sb.append(", download=").append(download); + sb.append(", species=").append(species); + sb.append('}'); + return sb.toString(); + } + public String getVersion() { return version; } - public void setVersion(String version) { + public CellBaseConfiguration setVersion(String version) { this.version = version; + return this; } public String getApiVersion() { return apiVersion; } - public void setApiVersion(String apiVersion) { + public CellBaseConfiguration setApiVersion(String apiVersion) { this.apiVersion = apiVersion; + return this; } public String getWiki() { return wiki; } - public void setWiki(String wiki) { + public CellBaseConfiguration setWiki(String wiki) { this.wiki = wiki; + return this; + } + + public String getDefaultOutdir() { + return defaultOutdir; + } + + public CellBaseConfiguration setDefaultOutdir(String defaultOutdir) { + this.defaultOutdir = defaultOutdir; + return this; } public Databases getDatabases() { @@ -75,28 +114,31 @@ public CellBaseConfiguration setDatabases(Databases databases) { return this; } - public String getDefaultOutdir() { - return defaultOutdir; + public CacheProperties getCache() { + return cache; } - public void setDefaultOutdir(String defaultOutdir) { - this.defaultOutdir = defaultOutdir; + public CellBaseConfiguration setCache(CacheProperties cache) { + this.cache = cache; + return this; } public DownloadProperties getDownload() { return download; } - public void setDownload(DownloadProperties download) { + public CellBaseConfiguration setDownload(DownloadProperties download) { this.download = download; + return this; } public SpeciesProperties getSpecies() { return species; } - public void setSpecies(SpeciesProperties species) { + public CellBaseConfiguration setSpecies(SpeciesProperties species) { this.species = species; + return this; } public List getAllSpecies() { diff --git a/cellbase-core/src/main/resources/configuration.json b/cellbase-core/src/main/resources/configuration.json index fc15b8e8ed..787c8889f4 100644 --- a/cellbase-core/src/main/resources/configuration.json +++ b/cellbase-core/src/main/resources/configuration.json @@ -25,9 +25,17 @@ "user": "${CELLBASE.DB.USER}", "password": "${CELLBASE.DB.PASSWORD}" } + } }, "defaultOutdir": "/tmp", + "cache": { + "host": "localhost:6379", + "active": true, + "serialization": "kryo", + "slowThreshold": 50 + }, + "download": { "ensembl": { "database": { diff --git a/cellbase-core/src/test/java/org/opencb/cellbase/core/config/CellBaseConfigurationTest.java b/cellbase-core/src/test/java/org/opencb/cellbase/core/config/CellBaseConfigurationTest.java new file mode 100644 index 0000000000..6ea8abbfa2 --- /dev/null +++ b/cellbase-core/src/test/java/org/opencb/cellbase/core/config/CellBaseConfigurationTest.java @@ -0,0 +1,25 @@ +/* +package org.opencb.cellbase.core.config; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.junit.Test; + +*/ +/** + * Created by imedina on 20/10/16. + *//* + +public class CellBaseConfigurationTest { + + + @Test + public void load() throws Exception { + + CellBaseConfiguration cellBaseConfiguration = CellBaseConfiguration + .load(getClass().getResourceAsStream("/configuration_test.json")); + + ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); + System.out.println(objectMapper.writer().withDefaultPrettyPrinter().writeValueAsString(cellBaseConfiguration)); + } +}*/ diff --git a/cellbase-core/src/test/resources/configuration_test.json b/cellbase-core/src/test/resources/configuration_test.json new file mode 100644 index 0000000000..afb2dfeca7 --- /dev/null +++ b/cellbase-core/src/test/resources/configuration_test.json @@ -0,0 +1,908 @@ +{ + "version": "v4", + "apiVersion": "${cellbase.version}", + "wiki": "https://github.com/opencb/cellbase/wiki", + "defaultOutdir": "/tmp", + "databases": { + "mongodb": { + "host": "${CELLBASE.DB.MONGODB.HOST}", + "user": "${CELLBASE.DB.USER}", + "password": "${CELLBASE.DB.PASSWORD}", + "options": { + "authenticationDatabase": "${CELLBASE.DB.MONGODB.AUTHENTICATIONDATABASE}", + "readPreference": "${CELLBASE.DB.MONGODB.READPREFERENCE}", + "replicaSet": "${CELLBASE.DB.MONGODB.REPLICASET}" + } + }, + "neo4j": { + "hsapiens": { + "host": "${CELLBASE.DB.NEO4J.HOST}", + "user": "${CELLBASE.DB.USER}", + "password": "${CELLBASE.DB.PASSWORD}" + }, + "mmusculus": { + "host": "${CELLBASE.DB.NEO4J.HOST}", + "user": "${CELLBASE.DB.USER}", + "password": "${CELLBASE.DB.PASSWORD}" + } + } + }, + "cache": { + "host": "localhost:6379", + "active": true, + "serialization": "kryo" + }, + "download": { + "ensembl": { + "database": { + "host": "ensembldb.ensembl.org:3306", + "user": "anonymous", + "password": "" + }, + "libs": "${CELLBASE.ENSEMBL.LIBS}", + "url": { + "host": "ftp://ftp.ensembl.org/pub" + } + }, + "ensemblGenomes":{ + "database":{ + "host": "mysql-eg-publicsql.ebi.ac.uk:4157", + "user": "anonymous", + "password": "" + }, + "libs": "${CELLBASE.ENSEMBL.LIBS}", + "url":{ + "host":"ftp://ftp.ensemblgenomes.org/pub" + } + }, + "geneUniprotXref": { + "host": "ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/" + }, + "geneExpressionAtlas": { + "host": "ftp://ftp.ebi.ac.uk/pub/databases/microarray/data/gxa/allgenes_updown_in_organism_part_2.0.14.tab.gz" + }, + "mirbase": { + "host": "ftp://mirbase.org/pub/mirbase/CURRENT/" + }, + "mirbaseReadme": { + "host": "ftp://mirbase.org/pub/mirbase/CURRENT/README" + }, + "targetScan": { + "host": "http://hgdownload.cse.ucsc.edu/goldenPath/" + }, + "miRTarBase": { + "host": "http://mirtarbase.mbc.nctu.edu.tw/cache/download/4.5/" + }, + "uniprot": { + "host": "ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/complete/uniprot_sprot.xml.gz" + }, + "uniprotRelNotes": { + "host": "ftp://ftp.uniprot.org/pub/databases/uniprot/relnotes.txt" + }, + "intact": { + "host": "ftp://ftp.ebi.ac.uk/pub/databases/intact/current/psimitab/intact.txt" + }, + "interpro": { + "host": "ftp://ftp.ebi.ac.uk/pub/databases/interpro/current/protein2ipr.dat.gz" + }, + "interproRelNotes": { + "host": "ftp://ftp.ebi.ac.uk/pub/databases/interpro/current/release_notes.txt" + }, + "conservation": { + "host": "ftp://hgdownload.cse.ucsc.edu/goldenPath/" + }, + "gerp": { + "host": "http://mendel.stanford.edu/SidowLab/downloads/gerp/hg19.GERP_scores.tar.gz" + }, + "clinvar": { + "host": "ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/xml/ClinVarFullRelease_2016-10.xml.gz" + }, + "clinvarSummary": { + "host": "ftp://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/variant_summary.txt.gz" + }, + "clinvarEfoTerms": { + "host": "ftp://ftp.ebi.ac.uk/pub/databases/eva/ClinVar/ClinVar_Traits_EFO_Names_260615.csv" + }, + "gwasCatalog": { + "host": "ftp://ftp.ebi.ac.uk/pub/databases/gwas/releases/2016/09/28/gwas-catalog-associations.tsv" + }, + "hpo": { + "host": "http://compbio.charite.de/hudson/job/hpo.annotations.monthly/lastStableBuild/artifact/annotation/ALL_SOURCES_ALL_FREQUENCIES_diseases_to_genes_to_phenotypes.txt" + }, + "disgenet": { + "host": "http://www.disgenet.org/ds/DisGeNET/results/all_gene_disease_associations.tsv.gz" + }, + "disgenetReadme": { + "host": "http://www.disgenet.org/ds/DisGeNET/results/readme.txt" + }, + "dgidb": { + "host": "http://dgidb.genome.wustl.edu/downloads/interactions.tsv" + }, + "cadd": { + "host": "http://krishna.gs.washington.edu/download/CADD/v1.3/whole_genome_SNVs.tsv.gz" + }, + "reactome": { + "host": "http://www.reactome.org/download/current/biopax.zip" + } + }, + "species": { + "vertebrates": [ + { + "id": "hsapiens", + "scientificName": "Homo sapiens", + "assemblies": [ + { + "ensemblVersion":"82_37", + "name":"GRCh37" + }, + { + "ensemblVersion":"86_38", + "name":"GRCh38" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "gene_disease_association", + "variation", + "variation_functional_score", + "regulation", + "protein", + "conservation", + "clinical" + ] + }, + { + "id":"mmusculus", + "scientificName":"Mus musculus", + "assemblies":[ + { + "ensemblVersion":"82_38", + "name":"GRCm38" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "regulation", + "protein", + "conservation" + ] + }, + { + "id":"drerio", + "scientificName":"Danio rerio", + "assemblies":[ + { + "ensemblVersion":"82_10", + "name":"GRCz10" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"rnorvegicus", + "scientificName":"Rattus norvegicus", + "assemblies":[ + { + "ensemblVersion":"82_6", + "name":"Rnor_6.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"ptroglodytes", + "scientificName":"Pan troglodytes", + "assemblies":[ + { + "ensemblVersion":"82_214", + "name":"CHIMP2.1.4" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"ggorilla", + "scientificName":"Gorilla gorilla", + "assemblies":[ + { + "ensemblVersion":"82_31", + "name":"gorGor3.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"pabelii", + "scientificName":"Pongo abelii", + "assemblies":[ + { + "ensemblVersion":"82_1", + "name":"PPYG2" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"mmulatta", + "scientificName":"Macaca mulatta", + "assemblies":[ + { + "ensemblVersion":"82_10", + "name":"MMUL_1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"csabaeus", + "scientificName":"Chlorocebus sabaeus", + "assemblies":[ + { + "ensemblVersion":"82_1", + "name":"ChlSab1.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"sscrofa", + "scientificName":"Sus scrofa", + "assemblies":[ + { + "ensemblVersion":"82_102", + "name":"Sscrofa10.2" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"cfamiliaris", + "scientificName":"Canis familiaris", + "assemblies":[ + { + "ensemblVersion":"82_31", + "name":"CanFam3.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"ecaballus", + "scientificName":"Equus caballus", + "assemblies":[ + { + "ensemblVersion":"82_2", + "name":"EquCab2" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"ocuniculus", + "scientificName":"Oryctolagus cuniculus", + "assemblies":[ + { + "ensemblVersion":"82_2", + "name":"OryCun2.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"ggallus", + "scientificName":"Gallus gallus", + "assemblies":[ + { + "ensemblVersion":"82_4", + "name":"Galgal4" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"btaurus", + "scientificName":"Bos taurus", + "assemblies":[ + { + "ensemblVersion":"82_31", + "name":"UMD3.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"fcatus", + "scientificName":"Felis catus", + "assemblies":[ + { + "ensemblVersion":"82_62", + "name":"Felis_catus_6.2" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"cintestinalis", + "scientificName":"Ciona intestinalis", + "assemblies":[ + { + "ensemblVersion":"82_3", + "name":"KH" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"oaries", + "scientificName":"Ovis aries", + "assemblies":[ + { + "ensemblVersion":"82_31", + "name":"Oar_v3.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"olatipes", + "scientificName":"Oryzias latipes", + "assemblies":[ + { + "ensemblVersion":"82_1", + "name":"HdrR" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"ttruncatus", + "scientificName":"Tursiops truncatus", + "assemblies":[ + { + "ensemblVersion":"82_1", + "name":"turTru1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"lafricana", + "scientificName":"Loxodonta africana", + "assemblies":[ + { + "ensemblVersion":"82_3", + "name":"Loxafr3.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"aplatyrhynchos", + "scientificName":"Anas platyrhynchos", + "assemblies":[ + { + "ensemblVersion":"82_1", + "name":"BGI_duck_1.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + } + ], + "metazoa":[ + { + "id":"celegans", + "scientificName":"Caenorhabditis elegans", + "assemblies":[ + { + "ensemblVersion":"29_82_245", + "name":"WBcel235" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"dmelanogaster", + "scientificName":"Drosophila melanogaster", + "assemblies":[ + { + "ensemblVersion":"29_82_6", + "name":"BDGP6" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"dsimulans", + "scientificName":"Drosophila simulans", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"GCA_000259055.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"agambiae", + "scientificName":"Anopheles gambiae", + "assemblies":[ + { + "ensemblVersion":"29_82_4", + "name":"AgamP4" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"bmori", + "scientificName":"Bombyx mori", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"GCA_000151625.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + } + ], + "fungi":[ + { + "id":"scerevisiae", + "scientificName":"Saccharomyces cerevisiae", + "assemblies":[ + { + "ensemblVersion":"29_82_4", + "name":"R64-1-1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"spombe", + "scientificName":"Schizosaccharomyces pombe", + "assemblies":[ + { + "ensemblVersion":"29_82_2", + "name":"ASM294v2" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"afumigatus", + "scientificName":"Aspergillus fumigatus", + "assemblies":[ + { + "ensemblVersion":"29_82_2", + "name":"CADRE" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"aniger", + "scientificName":"Aspergillus niger", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"CADRE" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"anidulans", + "scientificName":"Aspergillus nidulans", + "assemblies":[ + { + "ensemblVersion":"29_82_6", + "name":"ASM1142v1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"moryzae", + "scientificName":"Magnaporthe oryzae", + "assemblies":[ + { + "ensemblVersion":"29_82_8", + "name":"MG8" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + } + ], + "protist":[ + { + "id":"pfalciparum", + "scientificName":"Plasmodium falciparum", + "assemblies":[ + { + "ensemblVersion":"29_82_3", + "name":"ASM276v1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"lmajor", + "scientificName":"Leishmania major", + "assemblies":[ + { + "ensemblVersion":"29_82_2", + "name":"ASM272v2" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"ddiscoideum", + "scientificName":"Dictyostelium discoideum", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"dictybase.01" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"glamblia", + "scientificName":"Giardia lamblia", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"GCA_000002435.1" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"pultimum", + "scientificName":"Pythium ultimum", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"pug" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + } + ], + "plants":[ + { + "id":"athaliana", + "scientificName":"Arabidopsis thaliana", + "assemblies":[ + { + "ensemblVersion":"29_82_10", + "name":"TAIR10" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation", + "protein" + ] + }, + { + "id":"alyrata", + "scientificName":"Arabidopsis lyrata", + "assemblies":[ + { + "ensemblVersion":"29_82_10", + "name":"v.1.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"osativa", + "scientificName":"Oryza sativa", + "assemblies":[ + { + "ensemblVersion":"29_82_7", + "name":"IRGSP-1.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"gmax", + "scientificName":"Glycine max", + "assemblies":[ + { + "ensemblVersion":"29_82_1", + "name":"V1.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + }, + { + "id":"vvinifera", + "scientificName":"Vitis vinifera", + "assemblies":[ + { + "ensemblVersion":"29_82_3", + "name":"IGGP_12x" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"zmays", + "scientificName":"Zea mays", + "assemblies":[ + { + "ensemblVersion":"29_82_6", + "name":"AGPv3" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"hvulgare", + "scientificName":"Hordeum vulgare", + "assemblies":[ + { + "ensemblVersion":"29_82_2", + "name":"082214v1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"sbicolor", + "scientificName":"Sorghum bicolor", + "assemblies":[ + { + "ensemblVersion":"29_82_14", + "name":"Sorbi1" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"slycopersicum", + "scientificName":"Solanum lycopersicum", + "assemblies":[ + { + "ensemblVersion":"29_82_250", + "name":"GCA_000188115.2" + } + ], + "data":[ + "genome", + "genome_info", + "gene", + "variation" + ] + }, + { + "id":"stuberosum", + "scientificName":"Solanum tuberosum", + "assemblies":[ + { + "ensemblVersion":"29_82_4", + "name":"3.0" + } + ], + "data":[ + "genome", + "genome_info", + "gene" + ] + } + ] + } +} diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalLegacyMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalLegacyMongoDBAdaptor.java index ea0de2d6a9..afb2a2fdb5 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalLegacyMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalLegacyMongoDBAdaptor.java @@ -24,10 +24,10 @@ import org.opencb.biodata.models.variant.avro.Gwas; import org.opencb.cellbase.core.api.ClinicalDBAdaptor; import org.opencb.cellbase.core.common.clinical.ClinicalVariant; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -39,8 +39,8 @@ @Deprecated public class ClinicalLegacyMongoDBAdaptor extends MongoDBAdaptor implements ClinicalDBAdaptor { - public ClinicalLegacyMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public ClinicalLegacyMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("clinical"); logger.debug("ClinicalLegacyMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java index 1efaf279d3..5670964810 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ClinicalMongoDBAdaptor.java @@ -8,10 +8,10 @@ import org.opencb.biodata.models.variant.Variant; import org.opencb.biodata.models.variant.avro.*; import org.opencb.cellbase.core.api.ClinicalDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -26,10 +26,10 @@ public class ClinicalMongoDBAdaptor extends MongoDBAdaptor implements ClinicalDB private static final String PRIVATE_CLINICAL_FIELDS = "_featureXrefs,_traits"; private static final String SEPARATOR = ","; - public ClinicalMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public ClinicalMongoDBAdaptor(String species, String assembly, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("clinical_variants"); - logger.debug("ClinicalMongoDBAdaptor: in 'constructor'"); } @@ -75,13 +75,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson bson = parseQuery(query); - return mongoDBCollection.count(bson); + return count(bson, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson bson = parseQuery(query); - return mongoDBCollection.distinct(field, bson); + return distinct(field, bson, mongoDBCollection); } @Override @@ -95,7 +95,7 @@ public QueryResult get(Query query, QueryOptions options) { QueryOptions parsedOptions = parseQueryOptions(options, query); parsedOptions = addPrivateExcludeOptions(parsedOptions, PRIVATE_CLINICAL_FIELDS); logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()).toJson()); - return mongoDBCollection.find(bson, null, Variant.class, parsedOptions); + return executeBsonQuery(bson, null, query, parsedOptions, mongoDBCollection, Variant.class); } @Override @@ -104,7 +104,7 @@ public QueryResult nativeGet(Query query, QueryOptions options) { QueryOptions parsedOptions = parseQueryOptions(options, query); parsedOptions = addPrivateExcludeOptions(parsedOptions, PRIVATE_CLINICAL_FIELDS); logger.info("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()).toJson()); - return mongoDBCollection.find(bson, parsedOptions); + return executeBsonQuery(bson, null, query, parsedOptions, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ConservationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ConservationMongoDBAdaptor.java index ea08efd890..1eb7d645b6 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ConservationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ConservationMongoDBAdaptor.java @@ -25,11 +25,11 @@ import org.opencb.biodata.models.variant.avro.Score; import org.opencb.cellbase.core.api.ConservationDBAdaptor; import org.opencb.biodata.models.core.GenomicScoreRegion; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -40,8 +40,9 @@ @Deprecated public class ConservationMongoDBAdaptor extends MongoDBAdaptor implements ConservationDBAdaptor { - public ConservationMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public ConservationMongoDBAdaptor(String species, String assembly, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("conservation"); logger.debug("ConservationMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java index 019351b90b..23acbc95be 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GeneMongoDBAdaptor.java @@ -31,12 +31,12 @@ import org.opencb.biodata.models.core.Gene; import org.opencb.biodata.models.core.Region; import org.opencb.cellbase.core.api.GeneDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; import org.opencb.commons.datastore.mongodb.MongoDBCollection; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.io.IOException; import java.util.*; @@ -49,14 +49,14 @@ */ public class GeneMongoDBAdaptor extends MongoDBAdaptor implements GeneDBAdaptor { + private static final String TRANSCRIPTS = "transcripts"; private static final String GENE = "gene"; private static final String ANNOTATION_FLAGS = "annotationFlags"; - public GeneMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public GeneMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection(GENE); - logger.debug("GeneMongoDBAdaptor: in 'constructor'"); } @@ -88,13 +88,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson bsonDocument = parseQuery(query); - return mongoDBCollection.count(bsonDocument); + return count(bsonDocument, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson bsonDocument = parseQuery(query); - return mongoDBCollection.distinct(field, bsonDocument); + return distinct(field, bsonDocument, mongoDBCollection); } @Override @@ -105,10 +105,10 @@ public QueryResult stats(Query query) { @Override public QueryResult get(Query query, QueryOptions options) { Bson bson = parseQuery(query); - options = addPrivateExcludeOptions(options); if (postDBFilteringParametersEnabled(query)) { - QueryResult nativeQueryResult = postDBFiltering(query, mongoDBCollection.find(bson, options)); + QueryResult nativeQueryResult = postDBFiltering(query, + executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class)); QueryResult queryResult = new QueryResult<>(nativeQueryResult.getId(), nativeQueryResult.getDbTime(), nativeQueryResult.getNumResults(), nativeQueryResult.getNumTotalResults(), nativeQueryResult.getWarningMsg(), @@ -131,16 +131,16 @@ public QueryResult get(Query query, QueryOptions options) { }).collect(Collectors.toList())); return queryResult; } else { - logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()) .toJson()); - return mongoDBCollection.find(bson, null, Gene.class, options); + logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()).toJson()); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Gene.class); } } @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); - logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()) .toJson()); - return postDBFiltering(query, mongoDBCollection.find(bson, options)); + logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()).toJson()); + return postDBFiltering(query, executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class)); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java index f646f762a8..185ab3eb8e 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/GenomeMongoDBAdaptor.java @@ -20,18 +20,18 @@ import com.mongodb.client.model.Filters; import org.bson.Document; import org.bson.conversions.Bson; +import org.opencb.biodata.models.core.GenomeSequenceFeature; +import org.opencb.biodata.models.core.GenomicScoreRegion; import org.opencb.biodata.models.core.Region; import org.opencb.biodata.models.variant.avro.Cytoband; import org.opencb.cellbase.core.api.GenomeDBAdaptor; import org.opencb.cellbase.core.common.DNASequenceUtils; -import org.opencb.biodata.models.core.GenomeSequenceFeature; -import org.opencb.biodata.models.core.GenomicScoreRegion; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; import org.opencb.commons.datastore.mongodb.MongoDBCollection; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -51,8 +51,8 @@ public class GenomeMongoDBAdaptor extends MongoDBAdaptor implements GenomeDBAdap private static final Object CHROMOSOMES = "chromosomes"; private Document genomeInfo = null; - public GenomeMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public GenomeMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); genomeInfoMongoDBCollection = mongoDataStore.getCollection("genome_info"); mongoDBCollection = mongoDataStore.getCollection("genome_sequence"); @@ -195,7 +195,7 @@ public QueryResult getSequence(Region region, QueryOption String sequence = stringBuilder.toString().substring(startIndex, startIndex + length); String strand = "1"; - String queryStrand= (query.getString("strand") != null) ? query.getString("strand") : "1"; + String queryStrand = (query.getString("strand") != null) ? query.getString("strand") : "1"; if (queryStrand.equals("-1") || queryStrand.equals("-")) { sequence = DNASequenceUtils.reverseComplement(sequence); strand = "-1"; @@ -328,13 +328,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson bson = parseQuery(query); - return mongoDBCollection.count(bson); + return count(bson, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson bson = parseQuery(query); - return mongoDBCollection.distinct(field, bson); + return distinct(field, bson, mongoDBCollection); } @Override @@ -350,7 +350,7 @@ public QueryResult get(Query query, QueryOptions options) { @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return mongoDBCollection.find(bson, options); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MetaMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MetaMongoDBAdaptor.java index dffc18a7c4..8fb4391193 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MetaMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MetaMongoDBAdaptor.java @@ -3,10 +3,10 @@ import org.bson.BsonDocument; import org.bson.Document; import org.opencb.cellbase.core.api.CellBaseDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.Iterator; import java.util.List; @@ -17,8 +17,8 @@ */ public class MetaMongoDBAdaptor extends MongoDBAdaptor implements CellBaseDBAdaptor { - public MetaMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public MetaMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("metadata"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java index dbc6a5432d..ea7dc5d295 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptor.java @@ -19,50 +19,69 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.mongodb.QueryBuilder; import com.mongodb.client.MongoCursor; -import com.mongodb.client.model.*; +import com.mongodb.client.model.Accumulators; +import com.mongodb.client.model.Aggregates; +import com.mongodb.client.model.Filters; +import com.mongodb.client.model.Projections; import org.bson.*; import org.bson.conversions.Bson; import org.opencb.biodata.models.core.Region; +import org.opencb.cellbase.core.cache.CacheManager; import org.opencb.cellbase.core.common.IntervalFeatureFrequency; +import org.opencb.cellbase.core.config.CellBaseConfiguration; +import org.opencb.cellbase.core.config.DatabaseCredentials; +import org.opencb.cellbase.core.config.Species; +import org.opencb.commons.datastore.core.DataStoreServerAddress; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; import org.opencb.commons.datastore.mongodb.MongoDBCollection; +import org.opencb.commons.datastore.mongodb.MongoDBConfiguration; import org.opencb.commons.datastore.mongodb.MongoDataStore; +import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.math.BigInteger; +import java.security.InvalidParameterException; import java.util.*; +import java.util.stream.Collectors; public class MongoDBAdaptor { + private static final String CELLBASE_DB_MONGODB_REPLICASET = "CELLBASE.DB.MONGODB.REPLICASET"; + enum QueryValueType {INTEGER, STRING} protected String species; protected String assembly; + protected static MongoDataStoreManager mongoDataStoreManager; protected MongoDataStore mongoDataStore; protected MongoDBCollection mongoDBCollection; + protected CellBaseConfiguration cellBaseConfiguration; + private static CacheManager cacheManager; protected Logger logger = LoggerFactory.getLogger(this.getClass()); protected ObjectMapper objectMapper; - public MongoDBAdaptor(MongoDataStore mongoDataStore) { - this("", "", mongoDataStore); + public MongoDBAdaptor(CellBaseConfiguration cellBaseConfiguration) { + this("", "", cellBaseConfiguration); } - public MongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { + public MongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { this.species = species; this.assembly = assembly; - this.mongoDataStore = mongoDataStore; + this.cellBaseConfiguration = cellBaseConfiguration; logger = LoggerFactory.getLogger(this.getClass().toString()); objectMapper = new ObjectMapper(); initSpeciesAssembly(species, assembly); -// jsonObjectMapper = new ObjectMapper(); + init(); + this.mongoDataStore = createMongoDBDatastore(species, assembly); + this.cacheManager = new CacheManager(cellBaseConfiguration); } private void initSpeciesAssembly(String species, String assembly) { @@ -74,6 +93,148 @@ private void initSpeciesAssembly(String species, String assembly) { } } + private void init() { + if (mongoDataStoreManager == null) { +// String[] hosts = cellBaseConfiguration.getDatabases().get("mongodb").getHost().split(","); + String[] hosts = cellBaseConfiguration.getDatabases().getMongodb().getHost().split(","); + List dataStoreServerAddresses = new ArrayList<>(hosts.length); + for (String host : hosts) { + String[] hostPort = host.split(":"); + if (hostPort.length == 1) { + dataStoreServerAddresses.add(new DataStoreServerAddress(hostPort[0], 27017)); + } else { + dataStoreServerAddresses.add(new DataStoreServerAddress(hostPort[0], Integer.parseInt(hostPort[1]))); + } + } + mongoDataStoreManager = new MongoDataStoreManager(dataStoreServerAddresses); + logger.debug("MongoDBAdaptorFactory constructor, this should be only be printed once"); + } + +// logger = LoggerFactory.getLogger(this.getClass()); + } + + private MongoDataStore createMongoDBDatastore(String species, String assembly) { + /** + Database name has the following pattern in lower case and with no '.' in the name: + cellbase_speciesId_assembly_cellbaseVersion + Example: + cellbase_hsapiens_grch37_v3 + **/ + + DatabaseCredentials mongodbCredentials = cellBaseConfiguration.getDatabases().getMongodb(); + + // We need to look for the species object in the configuration + Species speciesObject = getSpecies(species); + if (speciesObject != null) { + species = speciesObject.getId(); + String cellbaseAssembly = getAssembly(speciesObject, assembly); + + if (species != null && !species.isEmpty() && cellbaseAssembly != null && !cellbaseAssembly.isEmpty()) { + cellbaseAssembly = cellbaseAssembly.toLowerCase(); + // Database name is built following the above pattern + String database = "cellbase" + "_" + species + "_" + cellbaseAssembly.replaceAll("\\.", "").replaceAll("-", "") + .replaceAll("_", "") + "_" + cellBaseConfiguration.getVersion(); + logger.debug("Database for the species is '{}'", database); + + MongoDBConfiguration mongoDBConfiguration; + MongoDBConfiguration.Builder builder = MongoDBConfiguration.builder(); + + // For authenticated databases + if (!mongodbCredentials.getUser().isEmpty() && !mongodbCredentials.getPassword().isEmpty()) { + // MongoDB could authenticate against different databases + builder.setUserPassword(mongodbCredentials.getUser(), mongodbCredentials.getPassword()); + if (mongodbCredentials.getOptions().containsKey(MongoDBConfiguration.AUTHENTICATION_DATABASE)) { + builder.setAuthenticationDatabase(mongodbCredentials.getOptions() + .get(MongoDBConfiguration.AUTHENTICATION_DATABASE)); + } + } + + if (mongodbCredentials.getOptions().get(MongoDBConfiguration.READ_PREFERENCE) != null + && !mongodbCredentials.getOptions().get(MongoDBConfiguration.READ_PREFERENCE).isEmpty()) { + builder.add(MongoDBConfiguration.READ_PREFERENCE, + mongodbCredentials.getOptions().get(MongoDBConfiguration.READ_PREFERENCE)); + } + + String replicaSet = mongodbCredentials.getOptions().get(MongoDBConfiguration.REPLICA_SET); + if (replicaSet != null && !replicaSet.isEmpty() && !replicaSet.contains(CELLBASE_DB_MONGODB_REPLICASET)) { + builder.setReplicaSet(mongodbCredentials.getOptions().get(MongoDBConfiguration.REPLICA_SET)); + } + + String connectionsPerHost = mongodbCredentials.getOptions().get(MongoDBConfiguration.CONNECTIONS_PER_HOST); + if (connectionsPerHost != null && !connectionsPerHost.isEmpty()) { + builder.setConnectionsPerHost(Integer.valueOf(mongodbCredentials.getOptions() + .get(MongoDBConfiguration.CONNECTIONS_PER_HOST))); + } + + mongoDBConfiguration = builder.build(); + + logger.debug("*************************************************************************************"); + logger.debug("MongoDataStore configuration parameters: "); + logger.debug("{} = {}", MongoDBConfiguration.AUTHENTICATION_DATABASE, + mongoDBConfiguration.get(MongoDBConfiguration.AUTHENTICATION_DATABASE)); + logger.debug("{} = {}", MongoDBConfiguration.READ_PREFERENCE, + mongoDBConfiguration.get(MongoDBConfiguration.READ_PREFERENCE)); + logger.debug("{} = {}", MongoDBConfiguration.REPLICA_SET, + mongoDBConfiguration.get(MongoDBConfiguration.REPLICA_SET)); + logger.debug("{} = {}", MongoDBConfiguration.CONNECTIONS_PER_HOST, + mongoDBConfiguration.get(MongoDBConfiguration.CONNECTIONS_PER_HOST)); + logger.debug("*************************************************************************************"); +// } else { +// mongoDBConfiguration = MongoDBConfiguration.builder().init().build(); +// } + + // A MongoDataStore to this host and database is returned + MongoDataStore mongoDatastore = mongoDataStoreManager.get(database, mongoDBConfiguration); + + // we return the MongoDataStore object + return mongoDatastore; + } else { + logger.error("Assembly is not valid, assembly '{}'. Valid assemblies: {}", assembly, + String.join(",", speciesObject.getAssemblies().stream().map((assemblyObject) + -> assemblyObject.getName()).collect(Collectors.toList()))); + throw new InvalidParameterException("Assembly is not valid, assembly '" + assembly + + "'. Please provide one of supported assemblies: {" + + String.join(",", speciesObject.getAssemblies().stream().map((assemblyObject) + -> assemblyObject.getName()).collect(Collectors.toList())) + "}"); + } + } else { + logger.error("Species name is not valid: '{}'. Valid species: {}", species, + String.join(",", cellBaseConfiguration.getAllSpecies().stream().map((tmpSpeciesObject) + -> (tmpSpeciesObject.getCommonName() + "|" + tmpSpeciesObject.getScientificName())) + .collect(Collectors.toList()))); + throw new InvalidParameterException("Species name is not valid: '" + species + "'. Please provide one" + + " of supported species: {" + + String.join(",", cellBaseConfiguration.getAllSpecies().stream().map((tmpSpeciesObject) + -> (tmpSpeciesObject.getCommonName() + "|" + tmpSpeciesObject.getScientificName())) + .collect(Collectors.toList())) + "}"); + } + } + + protected Species getSpecies(String speciesName) { + Species species = null; + for (Species sp : cellBaseConfiguration.getAllSpecies()) { + if (speciesName.equalsIgnoreCase(sp.getId()) || speciesName.equalsIgnoreCase(sp.getScientificName())) { + species = sp; + break; + } + } + return species; + } + + protected String getAssembly(Species species, String assemblyName) { + String assembly = null; + if (assemblyName == null || assemblyName.isEmpty()) { + assembly = species.getAssemblies().get(0).getName(); + } else { + for (Species.Assembly assembly1 : species.getAssemblies()) { + if (assemblyName.equalsIgnoreCase(assembly1.getName())) { + assembly = assembly1.getName(); + } + } + } + return assembly; + } + protected QueryOptions addPrivateExcludeOptions(QueryOptions options) { return addPrivateExcludeOptions(options, "_id,_chunkIds"); } @@ -265,7 +426,6 @@ protected QueryResult groupBy(Bson query, List groupByField, String feat } - public QueryResult getIntervalFrequencies(Bson query, Region region, int intervalSize, QueryOptions options) { // MONGO QUERY TO IMPLEMENT // db.variation.aggregate({$match: {$and: [{chromosome: "1"}, {start: {$gt: 251391, $lt: 2701391}}]}}, {$group: @@ -393,39 +553,66 @@ public QueryResult getIntervalFrequencies(Bson query, Region region, int interva return queryResult; } + protected QueryResult count(Bson query, MongoDBCollection mongoDBCollection) { +// QueryResult result = cacheManager.get(); + QueryResult result = null; + if (result == null) { + result = mongoDBCollection.count(query); +// cacheManager.set(result); + } + return result; + } + protected QueryResult distinct(String field, Bson query, MongoDBCollection mongoDBCollection) { +// QueryResult result = cacheManager.get(); + QueryResult result = null; + if (result == null) { + result = mongoDBCollection.distinct(field, query); +// cacheManager.set(result); + } + return result; + } + protected QueryResult executeBsonQuery(Bson bsonQuery, Bson projection, Query query, QueryOptions options, + MongoDBCollection mongoDBCollection, Class clazz) { + QueryResult result = null; - - protected QueryResult executeDistinct(Object id, String fields, Document query) { -// long dbTimeStart, dbTimeEnd; -// dbTimeStart = System.currentTimeMillis(); - QueryResult queryResult = mongoDBCollection.distinct(fields, query); -// List dbObjectList = new LinkedList<>(); -// while (cursor.hasNext()) { -// dbObjectList.add(cursor.next()); -// } -// dbTimeEnd = System.currentTimeMillis(); - // setting queryResult fields - queryResult.setId(id.toString()); -// queryResult.setDbTime(Long.valueOf(dbTimeEnd - dbTimeStart).intValue()); -// queryResult.setNumResults(dbObjectList.size()); - return queryResult; + if (options.getBoolean("cache") && cacheManager.isActive()) { + String key = cacheManager.createKey(this.species, query, options); + result = cacheManager.get(key, clazz); + if (result.getResult().size() != 0) { + return result; + } else { + options.replace("cache", true, false); + options = addPrivateExcludeOptions(options); + result = mongoDBCollection.find(bsonQuery, projection, clazz, options); + cacheManager.set(key, query, result); + } + } else { + options = addPrivateExcludeOptions(options); + result = mongoDBCollection.find(bsonQuery, projection, clazz, options); + } + return result; } + + @Deprecated protected QueryResult executeQuery(Object id, Document query, QueryOptions options) { return executeQueryList2(Arrays.asList(id), Arrays.asList(query), options, mongoDBCollection).get(0); } + @Deprecated protected QueryResult executeQuery(Object id, Document query, QueryOptions options, MongoDBCollection mongoDBCollection2) { return executeQueryList2(Arrays.asList(id), Arrays.asList(query), options, mongoDBCollection2).get(0); } + @Deprecated protected List executeQueryList2(List ids, List queries, QueryOptions options) { return executeQueryList2(ids, queries, options, mongoDBCollection); } + @Deprecated protected List executeQueryList2(List ids, List queries, QueryOptions options, MongoDBCollection mongoDBCollection2) { List queryResults = new ArrayList<>(ids.size()); @@ -527,9 +714,6 @@ private int getChunkEnd(int id, int chunkSize) { } - - - public QueryResult next(String chromosome, int position, QueryOptions options, MongoDBCollection mongoDBCollection) { QueryBuilder builder; if (options.getString("strand") == null || options.getString("strand").equals("") @@ -696,9 +880,6 @@ protected Document getReturnFields(QueryOptions options) { // } - - - /* * For histograms */ diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptorFactory.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptorFactory.java index e4fb074ebd..8e6ccf3a8e 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptorFactory.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/MongoDBAdaptorFactory.java @@ -19,149 +19,21 @@ import org.bson.Document; import org.opencb.cellbase.core.api.*; import org.opencb.cellbase.core.config.CellBaseConfiguration; -import org.opencb.cellbase.core.config.DatabaseCredentials; -import org.opencb.cellbase.core.config.Species; -import org.opencb.commons.datastore.core.DataStoreServerAddress; -import org.opencb.commons.datastore.mongodb.MongoDBConfiguration; -import org.opencb.commons.datastore.mongodb.MongoDataStore; -import org.opencb.commons.datastore.mongodb.MongoDataStoreManager; - -import java.security.InvalidParameterException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; + public class MongoDBAdaptorFactory extends DBAdaptorFactory { - private static final String CELLBASE_DB_MONGODB_REPLICASET = "CELLBASE.DB.MONGODB.REPLICASET"; - private static final String SERVER_ADDRESS = "serverAddress"; + /** * MongoDataStoreManager acts as singleton by keeping a reference to all databases connections created. */ - private MongoDataStoreManager mongoDataStoreManager; + + private CellBaseConfiguration cellBaseConfiguration; // private static Map mongoDatastoreFactory; public MongoDBAdaptorFactory(CellBaseConfiguration cellBaseConfiguration) { super(cellBaseConfiguration); - - init(); - } - - private void init() { - if (mongoDataStoreManager == null) { -// String[] hosts = cellBaseConfiguration.getDatabases().get("mongodb").getHost().split(","); - String[] hosts = cellBaseConfiguration.getDatabases().getMongodb().getHost().split(","); - List dataStoreServerAddresses = new ArrayList<>(hosts.length); - for (String host : hosts) { - String[] hostPort = host.split(":"); - if (hostPort.length == 1) { - dataStoreServerAddresses.add(new DataStoreServerAddress(hostPort[0], 27017)); - } else { - dataStoreServerAddresses.add(new DataStoreServerAddress(hostPort[0], Integer.parseInt(hostPort[1]))); - } - } - mongoDataStoreManager = new MongoDataStoreManager(dataStoreServerAddresses); - logger.debug("MongoDBAdaptorFactory constructor, this should be only be printed once"); - } - -// logger = LoggerFactory.getLogger(this.getClass()); - } - - private MongoDataStore createMongoDBDatastore(String species, String assembly) { - /** - Database name has the following pattern in lower case and with no '.' in the name: - cellbase_speciesId_assembly_cellbaseVersion - Example: - cellbase_hsapiens_grch37_v3 - **/ - - DatabaseCredentials mongodbCredentials = cellBaseConfiguration.getDatabases().getMongodb(); - - // We need to look for the species object in the configuration - Species speciesObject = getSpecies(species); - if (speciesObject != null) { - species = speciesObject.getId(); - String cellbaseAssembly = getAssembly(speciesObject, assembly); - - if (species != null && !species.isEmpty() && cellbaseAssembly != null && !cellbaseAssembly.isEmpty()) { - cellbaseAssembly = cellbaseAssembly.toLowerCase(); - // Database name is built following the above pattern - String database = "cellbase" + "_" + species + "_" + cellbaseAssembly.replaceAll("\\.", "").replaceAll("-", "") - .replaceAll("_", "") + "_" + cellBaseConfiguration.getVersion(); - logger.debug("Database for the species is '{}'", database); - - MongoDBConfiguration mongoDBConfiguration; - MongoDBConfiguration.Builder builder = MongoDBConfiguration.builder(); - - // For authenticated databases - if (!mongodbCredentials.getUser().isEmpty() && !mongodbCredentials.getPassword().isEmpty()) { - // MongoDB could authenticate against different databases - builder.setUserPassword(mongodbCredentials.getUser(), mongodbCredentials.getPassword()); - if (mongodbCredentials.getOptions().containsKey(MongoDBConfiguration.AUTHENTICATION_DATABASE)) { - builder.setAuthenticationDatabase(mongodbCredentials.getOptions() - .get(MongoDBConfiguration.AUTHENTICATION_DATABASE)); - } - } - - if (mongodbCredentials.getOptions().get(MongoDBConfiguration.READ_PREFERENCE) != null - && !mongodbCredentials.getOptions().get(MongoDBConfiguration.READ_PREFERENCE).isEmpty()) { - builder.add(MongoDBConfiguration.READ_PREFERENCE, - mongodbCredentials.getOptions().get(MongoDBConfiguration.READ_PREFERENCE)); - } - - String replicaSet = mongodbCredentials.getOptions().get(MongoDBConfiguration.REPLICA_SET); - if (replicaSet != null && !replicaSet.isEmpty() && !replicaSet.contains(CELLBASE_DB_MONGODB_REPLICASET)) { - builder.setReplicaSet(mongodbCredentials.getOptions().get(MongoDBConfiguration.REPLICA_SET)); - } - - String connectionsPerHost = mongodbCredentials.getOptions().get(MongoDBConfiguration.CONNECTIONS_PER_HOST); - if (connectionsPerHost != null && !connectionsPerHost.isEmpty()) { - builder.setConnectionsPerHost(Integer.valueOf(mongodbCredentials.getOptions() - .get(MongoDBConfiguration.CONNECTIONS_PER_HOST))); - } - - mongoDBConfiguration = builder.build(); - - logger.debug("*************************************************************************************"); - logger.debug("MongoDataStore configuration parameters: "); - logger.debug("{} = {}", MongoDBConfiguration.AUTHENTICATION_DATABASE, - mongoDBConfiguration.get(MongoDBConfiguration.AUTHENTICATION_DATABASE)); - logger.debug("{} = {}", MongoDBConfiguration.READ_PREFERENCE, - mongoDBConfiguration.get(MongoDBConfiguration.READ_PREFERENCE)); - logger.debug("{} = {}", MongoDBConfiguration.REPLICA_SET, - mongoDBConfiguration.get(MongoDBConfiguration.REPLICA_SET)); - logger.debug("{} = {}", MongoDBConfiguration.CONNECTIONS_PER_HOST, - mongoDBConfiguration.get(MongoDBConfiguration.CONNECTIONS_PER_HOST)); - logger.debug("*************************************************************************************"); -// } else { -// mongoDBConfiguration = MongoDBConfiguration.builder().init().build(); -// } - - // A MongoDataStore to this host and database is returned - MongoDataStore mongoDatastore = mongoDataStoreManager.get(database, mongoDBConfiguration); - - // we return the MongoDataStore object - return mongoDatastore; - } else { - logger.error("Assembly is not valid, assembly '{}'. Valid assemblies: {}", assembly, - String.join(",", speciesObject.getAssemblies().stream().map((assemblyObject) - -> assemblyObject.getName()).collect(Collectors.toList()))); - throw new InvalidParameterException("Assembly is not valid, assembly '" + assembly - + "'. Please provide one of supported assemblies: {" - + String.join(",", speciesObject.getAssemblies().stream().map((assemblyObject) - -> assemblyObject.getName()).collect(Collectors.toList())) + "}"); - } - } else { - logger.error("Species name is not valid: '{}'. Valid species: {}", species, - String.join(",", cellBaseConfiguration.getAllSpecies().stream().map((tmpSpeciesObject) - -> (tmpSpeciesObject.getCommonName() + "|" + tmpSpeciesObject.getScientificName())) - .collect(Collectors.toList()))); - throw new InvalidParameterException("Species name is not valid: '" + species + "'. Please provide one" - + " of supported species: {" - + String.join(",", cellBaseConfiguration.getAllSpecies().stream().map((tmpSpeciesObject) - -> (tmpSpeciesObject.getCommonName() + "|" + tmpSpeciesObject.getScientificName())) - .collect(Collectors.toList())) + "}"); - } + this.cellBaseConfiguration = cellBaseConfiguration; } @@ -183,8 +55,7 @@ public GenomeDBAdaptor getGenomeDBAdaptor(String species) { @Override public GenomeDBAdaptor getGenomeDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new GenomeMongoDBAdaptor(species, assembly, mongoDatastore); + return new GenomeMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @Override @@ -194,8 +65,7 @@ public CellBaseDBAdaptor getMetaDBAdaptor(String species) { @Override public CellBaseDBAdaptor getMetaDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new MetaMongoDBAdaptor(species, assembly, mongoDatastore); + return new MetaMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @Override @@ -205,9 +75,8 @@ public GeneDBAdaptor getGeneDBAdaptor(String species) { @Override public GeneDBAdaptor getGeneDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - GeneMongoDBAdaptor geneMongoDBAdaptor = new GeneMongoDBAdaptor(species, assembly, mongoDatastore); -// geneMongoDBAdaptor.setClinicalDBAdaptor(getClinicalLegacyDBAdaptor(species, assembly)); + GeneMongoDBAdaptor geneMongoDBAdaptor = new GeneMongoDBAdaptor(species, assembly, cellBaseConfiguration); +// geneMongoDBAdaptor.setClinicalDBAdaptor(getClinicalDBAdaptor(species, assembly)); return geneMongoDBAdaptor; } @@ -219,8 +88,7 @@ public TranscriptDBAdaptor getTranscriptDBAdaptor(String species) { @Override public TranscriptDBAdaptor getTranscriptDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new TranscriptMongoDBAdaptor(species, assembly, mongoDatastore); + return new TranscriptMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @@ -231,8 +99,7 @@ public ConservationDBAdaptor getConservationDBAdaptor(String species) { @Override public ConservationDBAdaptor getConservationDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ConservationMongoDBAdaptor(species, assembly, mongoDatastore); + return new ConservationMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @@ -243,8 +110,7 @@ public XRefDBAdaptor getXRefDBAdaptor(String species) { @Override public XRefDBAdaptor getXRefDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new XRefMongoDBAdaptor(species, assembly, mongoDatastore); + return new XRefMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @@ -255,8 +121,7 @@ public VariantDBAdaptor getVariationDBAdaptor(String species) { @Override public VariantDBAdaptor getVariationDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new VariantMongoDBAdaptor(species, assembly, mongoDatastore); + return new VariantMongoDBAdaptor(species, assembly, cellBaseConfiguration); } // @Override @@ -290,8 +155,7 @@ public ClinicalDBAdaptor getClinicalLegacyDBAdaptor(String species) { @Override public ClinicalDBAdaptor getClinicalLegacyDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ClinicalLegacyMongoDBAdaptor(species, assembly, mongoDatastore); + return new ClinicalLegacyMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @Override @@ -301,14 +165,12 @@ public ClinicalDBAdaptor getClinicalDBAdaptor(String species) { @Override public ClinicalDBAdaptor getClinicalDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ClinicalMongoDBAdaptor(species, assembly, mongoDatastore); + return new ClinicalMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @Override public RepeatsDBAdaptor getRepeatsDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new RepeatsMongoDBAdaptor(species, assembly, mongoDatastore); + return new RepeatsMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @@ -345,8 +207,7 @@ public ProteinDBAdaptor getProteinDBAdaptor(String species) { @Override public ProteinDBAdaptor getProteinDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ProteinMongoDBAdaptor(species, assembly, mongoDatastore); + return new ProteinMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @@ -357,8 +218,7 @@ public ProteinProteinInteractionDBAdaptor getProteinProteinInteractionDBAdaptor( @Override public ProteinProteinInteractionDBAdaptor getProteinProteinInteractionDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new ProteinProteinInteractionMongoDBAdaptor(species, assembly, mongoDatastore); + return new ProteinProteinInteractionMongoDBAdaptor(species, assembly, cellBaseConfiguration); } @@ -369,8 +229,7 @@ public RegulationDBAdaptor getRegulationDBAdaptor(String species) { @Override public RegulationDBAdaptor getRegulationDBAdaptor(String species, String assembly) { - MongoDataStore mongoDatastore = createMongoDBDatastore(species, assembly); - return new RegulationMongoDBAdaptor(species, assembly, mongoDatastore); + return new RegulationMongoDBAdaptor(species, assembly, cellBaseConfiguration); } // // @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java index bbda1344b6..91d07dfe7b 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinMongoDBAdaptor.java @@ -26,12 +26,12 @@ import org.opencb.biodata.models.variant.avro.ProteinVariantAnnotation; import org.opencb.biodata.models.variant.avro.Score; import org.opencb.cellbase.core.api.ProteinDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.core.variant.annotation.VariantAnnotationUtils; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; import org.opencb.commons.datastore.mongodb.MongoDBCollection; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -71,8 +71,9 @@ public class ProteinMongoDBAdaptor extends MongoDBAdaptor implements ProteinDBAd } - public ProteinMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public ProteinMongoDBAdaptor(String species, String assembly, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("protein"); proteinSubstitutionMongoDBCollection = mongoDataStore.getCollection("protein_functional_prediction"); @@ -307,13 +308,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson document = parseQuery(query); - return mongoDBCollection.count(document); + return count(document, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson document = parseQuery(query); - return mongoDBCollection.distinct(field, document); + return distinct(field, document, mongoDBCollection); } @Override @@ -324,13 +325,13 @@ public QueryResult stats(Query query) { @Override public QueryResult get(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return mongoDBCollection.find(bson, null, Entry.class, options); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Entry.class); } @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return mongoDBCollection.find(bson, options); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java index e28fc60b8d..4d607544d1 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/ProteinProteinInteractionMongoDBAdaptor.java @@ -21,10 +21,10 @@ import org.bson.conversions.Bson; import org.opencb.biodata.models.protein.Interaction; import org.opencb.cellbase.core.api.ProteinProteinInteractionDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.ArrayList; import java.util.Iterator; @@ -37,8 +37,9 @@ public class ProteinProteinInteractionMongoDBAdaptor extends MongoDBAdaptor implements ProteinProteinInteractionDBAdaptor { - public ProteinProteinInteractionMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public ProteinProteinInteractionMongoDBAdaptor(String species, String assembly, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("variation"); logger.debug("ProteinProteinInteractionMongoDBAdaptor: in 'constructor'"); @@ -66,12 +67,14 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { - return mongoDBCollection.count(parseQuery(query)); + Bson document = parseQuery(query); + return count(document, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { - return mongoDBCollection.distinct(field, parseQuery(query)); + Bson document = parseQuery(query); + return distinct(field, document, mongoDBCollection); } @Override @@ -86,7 +89,8 @@ public QueryResult get(Query query, QueryOptions options) { @Override public QueryResult nativeGet(Query query, QueryOptions options) { - return mongoDBCollection.find(parseQuery(query), options); + Bson bson = parseQuery(query); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java index 49976af5ad..4727586106 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RegulationMongoDBAdaptor.java @@ -16,18 +16,17 @@ package org.opencb.cellbase.lib.impl; -import com.mongodb.MongoClient; import com.mongodb.client.model.Filters; import org.bson.Document; import org.bson.conversions.Bson; import org.opencb.biodata.models.core.Region; import org.opencb.biodata.models.core.RegulatoryFeature; import org.opencb.cellbase.core.api.RegulationDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.ArrayList; import java.util.Iterator; @@ -39,8 +38,9 @@ */ public class RegulationMongoDBAdaptor extends MongoDBAdaptor implements RegulationDBAdaptor { - public RegulationMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public RegulationMongoDBAdaptor(String species, String assembly, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("regulatory_region"); logger.debug("RegulationMongoDBAdaptor: in 'constructor'"); @@ -92,13 +92,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson bsonDocument = parseQuery(query); - return mongoDBCollection.count(bsonDocument); + return count(bsonDocument, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson bsonDocument = parseQuery(query); - return mongoDBCollection.distinct(field, bsonDocument); + return distinct(field, bsonDocument, mongoDBCollection); } @Override @@ -110,14 +110,13 @@ public QueryResult stats(Query query) { public QueryResult get(Query query, QueryOptions options) { Bson bson = parseQuery(query); options = addPrivateExcludeOptions(options); - logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()) .toJson()); - return mongoDBCollection.find(bson, null, RegulatoryFeature.class, options); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, RegulatoryFeature.class); } @Override public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); - return mongoDBCollection.find(bson, options); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RepeatsMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RepeatsMongoDBAdaptor.java index 27012b66e2..32e6abcd96 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RepeatsMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/RepeatsMongoDBAdaptor.java @@ -6,11 +6,11 @@ import org.bson.conversions.Bson; import org.opencb.biodata.models.variant.avro.Repeat; import org.opencb.cellbase.core.api.RepeatsDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.ArrayList; import java.util.Iterator; @@ -23,8 +23,8 @@ public class RepeatsMongoDBAdaptor extends MongoDBAdaptor implements RepeatsDBAdaptor { private static final String REPEAT_COLLECTION = "repeats"; - public RepeatsMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDatastore) { - super(species, assembly, mongoDatastore); + public RepeatsMongoDBAdaptor(String species, String assembly, CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection(REPEAT_COLLECTION); logger.debug("RepeatsMongoDBAdaptor: in 'constructor'"); diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java index 02869c32db..e39596f58a 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/TranscriptMongoDBAdaptor.java @@ -10,11 +10,11 @@ import org.opencb.biodata.models.core.Region; import org.opencb.biodata.models.core.Transcript; import org.opencb.cellbase.core.api.TranscriptDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -24,8 +24,9 @@ */ public class TranscriptMongoDBAdaptor extends MongoDBAdaptor implements TranscriptDBAdaptor { - public TranscriptMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public TranscriptMongoDBAdaptor(String species, String assembly, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("gene"); logger.debug("TranscriptMongoDBAdaptor: in 'constructor'"); @@ -91,8 +92,7 @@ public QueryResult count(Query query) { @Override public QueryResult distinct(Query query, String field) { Bson bsonDocument = parseQuery(query); - return mongoDBCollection.distinct(field, bsonDocument); - + return distinct(field, bsonDocument, mongoDBCollection); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java index 01d70f0554..ea7e87f101 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/VariantMongoDBAdaptor.java @@ -31,6 +31,7 @@ import org.opencb.biodata.models.variant.avro.StructuralVariantType; import org.opencb.biodata.models.variant.avro.VariantType; import org.opencb.cellbase.core.api.VariantDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.cellbase.core.variant.annotation.VariantAnnotationUtils; import org.opencb.cellbase.lib.MongoDBCollectionConfiguration; import org.opencb.cellbase.lib.VariantMongoIterator; @@ -38,7 +39,6 @@ import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; import org.opencb.commons.datastore.mongodb.MongoDBCollection; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.*; import java.util.function.Consumer; @@ -58,8 +58,9 @@ public class VariantMongoDBAdaptor extends MongoDBAdaptor implements VariantDBAd private MongoDBCollection caddDBCollection; - public VariantMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public VariantMongoDBAdaptor(String species, String assembly, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("variation"); caddDBCollection = mongoDataStore.getCollection("variation_functional_score"); @@ -125,13 +126,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson document = parseQuery(query); - return mongoDBCollection.count(document); + return count(document, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson document = parseQuery(query); - return mongoDBCollection.distinct(field, document); + return distinct(field, document, mongoDBCollection); } @Override @@ -151,7 +152,7 @@ public QueryResult get(Query query, QueryOptions options) { // options = addPrivateExcludeOptions(options); logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()) .toJson()); - return mongoDBCollection.find(bson, null, Variant.class, options); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Variant.class); } // FIXME: patch to exclude annotation.additionalAttributes from the results - to remove as soon as the variation @@ -175,7 +176,7 @@ public QueryResult nativeGet(Query query, QueryOptions options) { Bson bson = parseQuery(query); // options.put(MongoDBCollection.SKIP_COUNT, true); logger.debug("query: {}", bson.toBsonDocument(Document.class, MongoClient.getDefaultCodecRegistry()) .toJson()); - return mongoDBCollection.find(bson, options); + return executeBsonQuery(bson, null, query, options, mongoDBCollection, Document.class); } @Override diff --git a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java index 8cbd281853..2ad2acc72d 100644 --- a/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java +++ b/cellbase-lib/src/main/java/org/opencb/cellbase/lib/impl/XRefMongoDBAdaptor.java @@ -23,10 +23,10 @@ import org.bson.conversions.Bson; import org.opencb.biodata.models.core.Xref; import org.opencb.cellbase.core.api.XRefDBAdaptor; +import org.opencb.cellbase.core.config.CellBaseConfiguration; import org.opencb.commons.datastore.core.Query; import org.opencb.commons.datastore.core.QueryOptions; import org.opencb.commons.datastore.core.QueryResult; -import org.opencb.commons.datastore.mongodb.MongoDataStore; import java.util.ArrayList; import java.util.Arrays; @@ -40,8 +40,9 @@ */ public class XRefMongoDBAdaptor extends MongoDBAdaptor implements XRefDBAdaptor { - public XRefMongoDBAdaptor(String species, String assembly, MongoDataStore mongoDataStore) { - super(species, assembly, mongoDataStore); + public XRefMongoDBAdaptor(String species, String assembly, + CellBaseConfiguration cellBaseConfiguration) { + super(species, assembly, cellBaseConfiguration); mongoDBCollection = mongoDataStore.getCollection("gene"); logger.debug("XRefMongoDBAdaptor: in 'constructor'"); @@ -70,13 +71,13 @@ public QueryResult update(List objectList, String field, String[] innerFie @Override public QueryResult count(Query query) { Bson bson = parseQuery(query); - return mongoDBCollection.count(bson); + return count(bson, mongoDBCollection); } @Override public QueryResult distinct(Query query, String field) { Bson bson = parseQuery(query); - return mongoDBCollection.distinct(field, bson); + return distinct(field, bson, mongoDBCollection); } @Override diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/GenericRestWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/GenericRestWSServer.java index 6848978ad6..97e4d48197 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/GenericRestWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/GenericRestWSServer.java @@ -271,6 +271,10 @@ public void parseQueryParams() { queryOptions.put(SKIP, (skip >= 0) ? skip : -1); queryOptions.put(SKIP_COUNT, StringUtils.isNotBlank(skipCount) && Boolean.parseBoolean(skipCount)); queryOptions.put(COUNT, StringUtils.isNotBlank(count) && Boolean.parseBoolean(count)); + queryOptions.put("cache", (multivaluedMap.get("cache") != null) + ? Boolean.parseBoolean(multivaluedMap.get("cache").get(0)) + : false); + // outputFormat = (outputFormat != null && !outputFormat.equals("")) ? outputFormat : "json"; // Add all the others QueryParams from the URL diff --git a/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/feature/GeneWSServer.java b/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/feature/GeneWSServer.java index 45c402bce6..de44193d8f 100755 --- a/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/feature/GeneWSServer.java +++ b/cellbase-server/src/main/java/org/opencb/cellbase/server/ws/feature/GeneWSServer.java @@ -474,7 +474,12 @@ public Response getAllIDs() { value = "Comma separated list of gene names for which drug data is available, " + "e.g.: BRCA2,TTN." + " Exact text matches will be returned", - required = false, dataType = "list of strings", paramType = "query") + required = false, dataType = "list of strings", paramType = "query"), + @ApiImplicitParam(name = "cache", + value = "Comma separated list of gene names for which drug data is available, " + + "e.g.: BRCA2,TTN." + + " Exact text matches will be returned", + required = false, defaultValue = "false", dataType = "boolean", paramType = "query") }) public Response getByEnsemblId(@PathParam("geneId") @ApiParam(name = "geneId", diff --git a/pom.xml b/pom.xml index a023c8feb0..a0cc365607 100644 --- a/pom.xml +++ b/pom.xml @@ -167,7 +167,6 @@ org.objenesis objenesis 2.1 - test junit