diff --git a/iis-3rdparty-madis/src/main/resources/eu/dnetlib/iis/3rdparty/scripts/madis/functions/row/text.py b/iis-3rdparty-madis/src/main/resources/eu/dnetlib/iis/3rdparty/scripts/madis/functions/row/text.py index ce2b732ef..75686a390 100755 --- a/iis-3rdparty-madis/src/main/resources/eu/dnetlib/iis/3rdparty/scripts/madis/functions/row/text.py +++ b/iis-3rdparty-madis/src/main/resources/eu/dnetlib/iis/3rdparty/scripts/madis/functions/row/text.py @@ -213,6 +213,17 @@ def cqlfields(*args): cqlfields.registered=True +def string_split_value(s, a): + """ + Returns the substring of `s` starting from the last occurrence of `a` till the end. + If `a` is not found in `s`, returns an empty string. + """ + index = s.rfind(a) # Find the last occurrence of `a` + if index == -1: + return "" # `a` not found + return s[index+len(a):] # Return from last occurrence to the end +string_split_value.registered = True + def comprspaces(*args): """ .. function:: comprspaces(text1, [text2,...]) -> text diff --git a/iis-common/src/main/java/eu/dnetlib/iis/common/InfoSpaceConstants.java b/iis-common/src/main/java/eu/dnetlib/iis/common/InfoSpaceConstants.java index 54537a5af..3358a1ce2 100644 --- a/iis-common/src/main/java/eu/dnetlib/iis/common/InfoSpaceConstants.java +++ b/iis-common/src/main/java/eu/dnetlib/iis/common/InfoSpaceConstants.java @@ -25,6 +25,7 @@ public final class InfoSpaceConstants { public static final String ROW_PREFIX_DATASOURCE = "10|"; public static final String OPENAIRE_ENTITY_ID_PREFIX = "openaire____"; + public static final String LENS_ENTITY_ID_PREFIX = "lens.org____"; public static final String SEMANTIC_CLASS_MAIN_TITLE = "main title"; public static final String SEMANTIC_CLASS_ALTERNATIVE_TITLE = "alternative title"; diff --git a/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/DocumentToPatent.avdl b/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/DocumentToPatent.avdl index 3708e600d..8b0872ec1 100644 --- a/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/DocumentToPatent.avdl +++ b/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/DocumentToPatent.avdl @@ -4,9 +4,8 @@ protocol IIS{ record DocumentToPatent { // document identifier, foreign key: DocumentWithBasicMetadata.id ("document basic metadata" data store) string documentId; -// identifier of patent referenced in this document, -// foreign key: Patent.appln_nr - string appln_nr; +// identifier of patent (lens identifier) referenced in this document, + string lensId; // Find more details on `confidenceLevel` constraints in eu/dnetlib/iis/README.markdown file. float confidenceLevel; // text snippet surrounding the matched reference, required mostly for internal debugging and analytics diff --git a/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/ImportedPatent.avdl b/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/ImportedPatent.avdl deleted file mode 100644 index 792d86f54..000000000 --- a/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/ImportedPatent.avdl +++ /dev/null @@ -1,11 +0,0 @@ -@namespace("eu.dnetlib.iis.referenceextraction.patent.schemas") -protocol IIS{ - - record ImportedPatent { - string appln_auth; - string appln_nr; - string publn_auth; - string publn_nr; - string publn_kind; - } -} diff --git a/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/Patent.avdl b/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/Patent.avdl deleted file mode 100644 index c128120f9..000000000 --- a/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/Patent.avdl +++ /dev/null @@ -1,17 +0,0 @@ -@namespace("eu.dnetlib.iis.referenceextraction.patent.schemas") -protocol IIS{ - - record Patent { - string appln_auth; - string appln_nr; - union {null, string} appln_filing_date = null; - union {null, string} appln_nr_epodoc = null; - union {null, string} earliest_publn_date = null; - union {null, string} appln_abstract = null; - union {null, string} appln_title = null; - union {null, array} ipc_class_symbol = null; - union {null, array} applicant_names = null; - // applicant country codes related to names, may contain duplicates or nulls - union {null, array} applicant_country_codes = null; - } -} diff --git a/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/PatentReferenceExtractionInput.avdl b/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/PatentReferenceExtractionInput.avdl index c0f7c369b..08f9d46fb 100644 --- a/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/PatentReferenceExtractionInput.avdl +++ b/iis-schemas/src/main/avro/eu/dnetlib/iis/referenceextraction/patent/PatentReferenceExtractionInput.avdl @@ -2,6 +2,10 @@ protocol IIS{ record PatentReferenceExtractionInput { - string appln_nr; + string c1; + string c2; + union{null, string} normal = null; + union{null, string} c4 = null; + union{null, string} c5 = null; } } diff --git a/iis-wf/iis-wf-export-actionmanager/src/main/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExportCounterReporter.java b/iis-wf/iis-wf-export-actionmanager/src/main/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExportCounterReporter.java deleted file mode 100644 index e88d4a046..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/main/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExportCounterReporter.java +++ /dev/null @@ -1,77 +0,0 @@ -package eu.dnetlib.iis.wf.export.actionmanager.entity.patent; - -import com.google.common.base.Preconditions; -import eu.dnetlib.iis.common.report.ReportEntryFactory; -import eu.dnetlib.iis.common.schemas.ReportEntry; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import pl.edu.icm.sparkutils.avro.SparkAvroSaver; -import scala.Tuple2; - -import java.util.Arrays; - -/** - * Reporter of patent entity and relation exporter job counters.
- * It calculates entities and relation related counters and saves them as {@link ReportEntry} datastore. - * - * @author mhorst - */ -public class PatentExportCounterReporter { - - public static final String EXPORTED_PATENT_ENTITIES_COUNTER = "export.entities.patent"; - - public static final String PATENT_REFERENCES_COUNTER = "processing.referenceExtraction.patent.references"; - - public static final String DISTINCT_PUBLICATIONS_WITH_PATENT_REFERENCES_COUNTER = "processing.referenceExtraction.patent.docs"; - - private SparkAvroSaver avroSaver = new SparkAvroSaver(); - - //------------------------ LOGIC -------------------------- - - /** - * Calculates entities and relations related counters based on RDDs and saves them under outputReportPath. - * - * @param sc SparkContext instance. - * @param rdd RDD of exported document to patents with ids. - * @param outputReportPath Path to report saving location. - */ - public void report(JavaSparkContext sc, JavaRDD rdd, String outputReportPath) { - Preconditions.checkNotNull(sc, "sparkContext has not been set"); - Preconditions.checkNotNull(outputReportPath, "reportPath has not been set"); - - ReportEntry totalEntitiesCounter = ReportEntryFactory.createCounterReportEntry( - EXPORTED_PATENT_ENTITIES_COUNTER, totalEntitiesCount(rdd)); - ReportEntry totalRelationsCounter = ReportEntryFactory.createCounterReportEntry( - PATENT_REFERENCES_COUNTER, totalRelationsCount(rdd)); - ReportEntry distinctPublicationsCounter = ReportEntryFactory.createCounterReportEntry( - DISTINCT_PUBLICATIONS_WITH_PATENT_REFERENCES_COUNTER, distinctPublicationsCount(rdd)); - - JavaRDD report = sc.parallelize(Arrays.asList( - totalRelationsCounter, totalEntitiesCounter, distinctPublicationsCounter), 1); - - avroSaver.saveJavaRDD(report, ReportEntry.SCHEMA$, outputReportPath); - } - - //------------------------ PRIVATE -------------------------- - - private long totalEntitiesCount(JavaRDD documentToPatentsToExportWithIds) { - return documentToPatentsToExportWithIds - .map(PatentExportMetadata::getPatentId) - .distinct() - .count(); - } - - private long totalRelationsCount(JavaRDD documentToPatentsToExportWithIds) { - return documentToPatentsToExportWithIds - .mapToPair(x -> new Tuple2<>(x.getDocumentId(), x.getPatentId())) - .distinct() - .count(); - } - - private long distinctPublicationsCount(JavaRDD documentToPatentsToExportWithIds) { - return documentToPatentsToExportWithIds - .map(PatentExportMetadata::getDocumentId) - .distinct() - .count(); - } -} diff --git a/iis-wf/iis-wf-export-actionmanager/src/main/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExportMetadata.java b/iis-wf/iis-wf-export-actionmanager/src/main/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExportMetadata.java deleted file mode 100644 index 578314e35..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/main/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExportMetadata.java +++ /dev/null @@ -1,34 +0,0 @@ -package eu.dnetlib.iis.wf.export.actionmanager.entity.patent; - -import eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent; -import eu.dnetlib.iis.referenceextraction.patent.schemas.Patent; - -public class PatentExportMetadata { - private final DocumentToPatent documentToPatent; - private final Patent patent; - private final String documentId; - private final String patentId; - - public PatentExportMetadata(DocumentToPatent documentToPatent, Patent patent, String documentId, String patentId) { - this.documentToPatent = documentToPatent; - this.patent = patent; - this.documentId = documentId; - this.patentId = patentId; - } - - public DocumentToPatent getDocumentToPatent() { - return documentToPatent; - } - - public Patent getPatent() { - return patent; - } - - public String getDocumentId() { - return documentId; - } - - public String getPatentId() { - return patentId; - } -} diff --git a/iis-wf/iis-wf-export-actionmanager/src/main/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExporterJob.java b/iis-wf/iis-wf-export-actionmanager/src/main/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExporterJob.java deleted file mode 100644 index 89206f56e..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/main/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExporterJob.java +++ /dev/null @@ -1,486 +0,0 @@ -package eu.dnetlib.iis.wf.export.actionmanager.entity.patent; - -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.collect.Lists; -import eu.dnetlib.dhp.schema.action.AtomicAction; -import eu.dnetlib.dhp.schema.oaf.*; -import eu.dnetlib.iis.common.InfoSpaceConstants; -import eu.dnetlib.iis.common.java.io.HdfsUtils; -import eu.dnetlib.iis.common.spark.JavaSparkContextFactory; -import eu.dnetlib.iis.common.utils.DateTimeUtils; -import eu.dnetlib.iis.common.utils.IteratorUtils; -import eu.dnetlib.iis.common.utils.ListUtils; -import eu.dnetlib.iis.common.utils.RDDUtils; -import eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent; -import eu.dnetlib.iis.referenceextraction.patent.schemas.Patent; -import eu.dnetlib.iis.wf.export.actionmanager.AtomicActionSerializationUtils; -import eu.dnetlib.iis.wf.export.actionmanager.IdentifierFactory; -import eu.dnetlib.iis.wf.export.actionmanager.OafConstants; -import eu.dnetlib.iis.wf.export.actionmanager.cfg.StaticConfigurationProvider; -import eu.dnetlib.iis.wf.export.actionmanager.entity.ConfidenceLevelUtils; -import eu.dnetlib.iis.wf.export.actionmanager.module.AlgorithmName; -import eu.dnetlib.iis.wf.export.actionmanager.module.BuilderModuleHelper; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.io.SequenceFile; -import org.apache.hadoop.io.Text; -import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import pl.edu.icm.sparkutils.avro.SparkAvroLoader; -import scala.Tuple2; - -import java.io.IOException; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.*; -import java.util.stream.Collectors; - -/** - * Patent entity and relations exporter reading {@link DocumentToPatent} avro records and exporting them as entity and relation actions. - * - * @author pjacewicz - */ -public class PatentExporterJob { - - private static final String EPO = "EPO"; - private static final String EUROPEAN_PATENT_OFFICE__PATSTAT = "European Patent Office/PATSTAT"; - private static final String CLASS_EPO_ID = "epo_id"; - private static final String CLASS_EPO_NR_EPODOC = "epo_nr_epodoc"; - private static final String IPC = "IPC"; - private static final String INTERNATIONAL_PATENT_CLASSIFICATION = "International Patent Classification"; - private static final String PATENT_ENTITY_ID_PREFIX = "epopatstat__"; - private static final String IIS_ENTITIES_PATENT = "iis-entities-patent"; - private static final String INFERENCE_PROVENANCE = buildInferenceProvenance(); - private static final String PATENT_DATASOURCE_OPENAIRE_ID_PREFIX = InfoSpaceConstants.ROW_PREFIX_DATASOURCE + InfoSpaceConstants.OPENAIRE_ENTITY_ID_PREFIX + InfoSpaceConstants.ID_NAMESPACE_SEPARATOR; - private static final String PATENT_RESULT_OPENAIRE_ID_PREFIX = InfoSpaceConstants.ROW_PREFIX_RESULT + PATENT_ENTITY_ID_PREFIX + InfoSpaceConstants.ID_NAMESPACE_SEPARATOR; - private static final String PATENT_ID_PREFIX_EPO = buildRowPrefixDatasourceOpenaireEntityIdPrefixEpo(); - private static final KeyValue OAF_ENTITY_COLLECTEDFROM = buildOafEntityPatentKeyValue(); - private static final Qualifier OAF_ENTITY_RESULT_METADATA_RESULTTYPE = buildOafEntityResultMetadataResulttype(); - private static final Qualifier OAF_ENTITY_PID_QUALIFIER_CLASS_EPO_ID = buildOafEntityPidQualifierClassEpoId(); - private static final Qualifier OAF_ENTITY_PID_QUALIFIER_CLASS_EPO_NR_EPODOC = buildOafEntityPidQualifierClassEpoNrEpodoc(); - private static final Qualifier OAF_ENTITY_RESULT_METADATA_TITLE_QUALIFIER = buildOafEntityResultMetadataTitleQualifier(); - private static final Qualifier OAF_ENTITY_RESULT_METADATA_SUBJECT_QUALIFIER = buildOafEntityResultMetadataSubjectQualifier(); - private static final Qualifier OAF_ENTITY_RESULT_METADATA_RELEVANTDATE_QUALIFIER = buildOafEntityResultMetadataRelevantdateQualifier(); - private static final DataInfo OAF_ENTITY_DATAINFO = BuilderModuleHelper.buildInferenceForTrustLevel(false, - StaticConfigurationProvider.ACTION_TRUST_0_9, INFERENCE_PROVENANCE, IIS_ENTITIES_PATENT); - - private static final SparkAvroLoader avroLoader = new SparkAvroLoader(); - private static final int numberOfOutputFiles = 10; - private static final PatentExportCounterReporter counterReporter = new PatentExportCounterReporter(); - - private static final DateTimeFormatter PATENT_DATE_OF_COLLECTION_FORMATTER = - DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm"); - - private static String buildInferenceProvenance() { - return InfoSpaceConstants.SEMANTIC_CLASS_IIS + InfoSpaceConstants.INFERENCE_PROVENANCE_SEPARATOR + AlgorithmName.document_patent; - } - - private static String buildRowPrefixDatasourceOpenaireEntityIdPrefixEpo() { - return PATENT_DATASOURCE_OPENAIRE_ID_PREFIX + DigestUtils.md5Hex(EPO); - } - - private static Qualifier buildOafEntityResultMetadataResulttype() { - Qualifier qualifier = new Qualifier(); - qualifier.setClassid(InfoSpaceConstants.SEMANTIC_CLASS_PUBLICATION); - qualifier.setClassname(InfoSpaceConstants.SEMANTIC_CLASS_PUBLICATION); - qualifier.setSchemeid(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_RESULT_TYPOLOGIES); - qualifier.setSchemename(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_RESULT_TYPOLOGIES); - return qualifier; - } - - private static Instance buildOafEntityResultInstance(Patent patent, String patentEpoUrlRoot, List pid) { - Instance instance = new Instance(); - instance.setInstancetype(buildOafEntityResultInstanceInstancetype()); - instance.setHostedby(buildOafEntityPatentKeyValue()); - instance.setCollectedfrom(buildOafEntityPatentKeyValue()); - instance.setUrl(Collections.singletonList(buildOafEntityResultInstanceUrl(patent, patentEpoUrlRoot))); - instance.setAlternateIdentifier(pid); - return instance; - } - - private static String buildOafEntityResultInstanceUrl(Patent patent, String patentEpoUrlRoot) { - return String.format("%s%s%s", patentEpoUrlRoot, patent.getApplnAuth(), patent.getApplnNr()); - } - - private static Qualifier buildOafEntityResultInstanceInstancetype() { - Qualifier qualifier = new Qualifier(); - qualifier.setClassid(InfoSpaceConstants.SEMANTIC_CLASS_INSTANCE_TYPE_PATENT); - qualifier.setClassname(InfoSpaceConstants.SEMANTIC_CLASS_PATENT); - qualifier.setSchemeid(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_PUBLICATION_RESOURCE); - qualifier.setSchemename(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_PUBLICATION_RESOURCE); - return qualifier; - } - - private static KeyValue buildOafEntityPatentKeyValue() { - KeyValue keyValue = new KeyValue(); - keyValue.setKey(PATENT_ID_PREFIX_EPO); - keyValue.setValue(EUROPEAN_PATENT_OFFICE__PATSTAT); - return keyValue; - } - - private static Qualifier buildOafEntityPidQualifierClassEpoId() { - Qualifier qualifier = new Qualifier(); - qualifier.setClassid(CLASS_EPO_ID); - qualifier.setClassname(CLASS_EPO_ID); - qualifier.setSchemeid(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_PID_TYPES); - qualifier.setSchemename(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_PID_TYPES); - return qualifier; - } - - private static Qualifier buildOafEntityPidQualifierClassEpoNrEpodoc() { - Qualifier qualifier = new Qualifier(); - qualifier.setClassid(CLASS_EPO_NR_EPODOC); - qualifier.setClassname(CLASS_EPO_NR_EPODOC); - qualifier.setSchemeid(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_PID_TYPES); - qualifier.setSchemename(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_PID_TYPES); - return qualifier; - } - - private static Qualifier buildOafEntityResultMetadataTitleQualifier() { - Qualifier qualifier = new Qualifier(); - qualifier.setClassid(InfoSpaceConstants.SEMANTIC_CLASS_MAIN_TITLE); - qualifier.setClassname(InfoSpaceConstants.SEMANTIC_CLASS_MAIN_TITLE); - qualifier.setSchemeid(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_TITLE); - qualifier.setSchemename(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_TITLE); - return qualifier; - } - - private static Qualifier buildOafEntityResultMetadataSubjectQualifier() { - Qualifier qualifier = new Qualifier(); - qualifier.setClassid(IPC); - qualifier.setClassname(INTERNATIONAL_PATENT_CLASSIFICATION); - qualifier.setSchemeid(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_CLASSIFICATION_TAXONOMIES); - qualifier.setSchemename(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_CLASSIFICATION_TAXONOMIES); - return qualifier; - } - - private static Qualifier buildOafEntityResultMetadataRelevantdateQualifier() { - Qualifier qualifier = new Qualifier(); - qualifier.setClassid(InfoSpaceConstants.SEMANTIC_CLASS_NAME_SUBMITTED); - qualifier.setClassname(InfoSpaceConstants.SEMANTIC_CLASS_NAME_SUBMITTED); - qualifier.setSchemeid(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_DATACITE_DATE); - qualifier.setSchemename(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_DATACITE_DATE); - return qualifier; - } - - public static void main(String[] args) throws IOException { - JobParameters params = new JobParameters(); - JCommander jcommander = new JCommander(params); - jcommander.parse(args); - - try (JavaSparkContext sc = JavaSparkContextFactory.withConfAndKryo(new SparkConf())) { - HdfsUtils.remove(sc.hadoopConfiguration(), params.outputRelationPath); - HdfsUtils.remove(sc.hadoopConfiguration(), params.outputEntityPath); - HdfsUtils.remove(sc.hadoopConfiguration(), params.outputReportPath); - - final Float confidenceLevelThreshold = ConfidenceLevelUtils - .evaluateConfidenceLevelThreshold(params.trustLevelThreshold); - final String collectedFromKey = params.collectedFromKey; - - JavaRDD relMetaRDD = avroLoader - .loadJavaRDD(sc, params.inputDocumentToPatentPath, DocumentToPatent.class); - JavaRDD entMetaRDD = avroLoader - .loadJavaRDD(sc, params.inputPatentPath, Patent.class); - - JavaPairRDD validRelMetaByApplnNrRDD = relMetaRDD - .filter(x -> isRelationValidAndAboveThreshold(x, confidenceLevelThreshold)) - .mapToPair(x -> new Tuple2<>(x.getApplnNr(), x)); - - JavaPairRDD validEntMetaByApplnNrRDD = entMetaRDD - .filter(PatentExporterJob::isMetadataValid) - .mapToPair(x -> new Tuple2<>(x.getApplnNr(), x)); - - JavaRDD patentExportMetaRDD = validRelMetaByApplnNrRDD - .join(validEntMetaByApplnNrRDD) - .values() - .map(x -> { - DocumentToPatent relMeta = x._1(); - Patent entMeta = x._2(); - String documentId = generateDocumentId(relMeta); - String patentId = generatePatentId(entMeta); - return new PatentExportMetadata(relMeta, entMeta, documentId, patentId); - }); - - JavaRDD patentExportMetaDedupRDD = deduplicateByHigherConfidenceLevel( - patentExportMetaRDD).cache(); - - Configuration configuration = sc.hadoopConfiguration(); - configuration.set(FileOutputFormat.COMPRESS, Boolean.TRUE.toString()); - configuration.set(FileOutputFormat.COMPRESS_TYPE, SequenceFile.CompressionType.BLOCK.name()); - - JavaPairRDD relationsToExportRDD = relationsToExport(patentExportMetaDedupRDD, collectedFromKey); - RDDUtils.saveTextPairRDD(relationsToExportRDD, numberOfOutputFiles, params.outputRelationPath, configuration); - - String patentDateOfCollection = DateTimeUtils.format( - LocalDateTime.parse(params.patentDateOfCollection, PATENT_DATE_OF_COLLECTION_FORMATTER)); - JavaPairRDD entitiesToExportRDD = entitiesToExport(patentExportMetaDedupRDD, - patentDateOfCollection, params.patentEpoUrlRoot); - RDDUtils.saveTextPairRDD(entitiesToExportRDD, numberOfOutputFiles, params.outputEntityPath, configuration); - - counterReporter.report(sc, patentExportMetaDedupRDD, params.outputReportPath); - } - } - - private static boolean isRelationValidAndAboveThreshold(DocumentToPatent relation, Float confidenceLevelThreshold) { - return ConfidenceLevelUtils.isValidConfidenceLevel(relation.getConfidenceLevel(), confidenceLevelThreshold); - } - - private static boolean isMetadataValid(Patent meta) { - return StringUtils.isNotBlank(meta.getApplnTitle()); - } - - private static String generateDocumentId(DocumentToPatent relation) { - return relation.getDocumentId().toString(); - } - - private static String generatePatentId(Patent meta) { - return PATENT_RESULT_OPENAIRE_ID_PREFIX + - DigestUtils.md5Hex(meta.getApplnAuth().toString() + meta.getApplnNr().toString()); - } - - private static JavaRDD deduplicateByHigherConfidenceLevel(JavaRDD rdd) { - return rdd - .groupBy(x -> new Tuple2<>(x.getDocumentId(), x.getPatentId())) - .mapValues(xs -> IteratorUtils.toStream(xs.iterator()).reduce(PatentExporterJob::reduceByHigherConfidenceLevel)) - .filter(x -> x._2.isPresent()) - .mapValues(Optional::get) - .values(); - } - - private static PatentExportMetadata reduceByHigherConfidenceLevel(PatentExportMetadata x, PatentExportMetadata y) { - return x.getDocumentToPatent().getConfidenceLevel() > y.getDocumentToPatent().getConfidenceLevel() ? x : y; - } - - private static JavaPairRDD relationsToExport(JavaRDD rdd, String collectedFromKey) { - return AtomicActionSerializationUtils - .mapActionToText(rdd - .flatMap(x -> { - String documentId = x.getDocumentId(); - String patentId = x.getPatentId(); - Float confidenceLevel = x.getDocumentToPatent().getConfidenceLevel(); - return buildRelationActions(documentId, patentId, confidenceLevel, collectedFromKey).iterator(); - }) - ); - } - - private static List> buildRelationActions(String documentId, String patentId, Float confidenceLevel, String collectedFromKey) { - AtomicAction forwardAction = new AtomicAction<>(); - forwardAction.setClazz(Relation.class); - forwardAction.setPayload(buildRelation(documentId, patentId, confidenceLevel, collectedFromKey)); - - AtomicAction reverseAction = new AtomicAction<>(); - reverseAction.setClazz(Relation.class); - reverseAction.setPayload(buildRelation(patentId, documentId, confidenceLevel, collectedFromKey)); - - return Arrays.asList(forwardAction, reverseAction); - } - - private static Relation buildRelation(String source, String target, Float confidenceLevel, String collectedFromKey) { - return BuilderModuleHelper.createRelation(source, target, OafConstants.REL_TYPE_RESULT_RESULT, - OafConstants.SUBREL_TYPE_RELATIONSHIP, OafConstants.REL_CLASS_ISRELATEDTO, - BuilderModuleHelper.buildInferenceForConfidenceLevel(confidenceLevel, INFERENCE_PROVENANCE), - collectedFromKey); - } - - private static JavaPairRDD entitiesToExport(JavaRDD rdd, - String patentDateOfCollection, - String patentEpoUrlRoot) { - JavaRDD> atomicActions = rdd - .mapToPair(x -> new Tuple2<>(x.getPatentId(), x.getPatent())) - .distinct() - .map(x -> { - String patentId = x._1(); - Patent patent = x._2(); - return buildEntityAction(patent, patentId, patentDateOfCollection, patentEpoUrlRoot); - }); - - return AtomicActionSerializationUtils.mapActionToText(atomicActions); - } - - private static AtomicAction buildEntityAction(Patent patent, String patentId, - String patentDateOfCollection, String patentEpoUrlRoot) { - AtomicAction action = new AtomicAction<>(); - action.setClazz(Publication.class); - action.setPayload(buildOafInstance(patent, patentId, patentDateOfCollection, patentEpoUrlRoot)); - return action; - } - - private static StructuredProperty buildOafEntityPid(String value, Qualifier qualifier) { - StructuredProperty pid = new StructuredProperty(); - pid.setValue(value); - pid.setQualifier(qualifier); - return pid; - } - - private static Publication buildOafInstance(Patent patent, - String patentId, - String patentDateOfCollection, - String patentEpoUrlRoot) { - Publication result = new Publication(); - - result.setLastupdatetimestamp(System.currentTimeMillis()); - result.setCollectedfrom(Collections.singletonList(OAF_ENTITY_COLLECTEDFROM)); - - result.setDateofcollection(patentDateOfCollection); - result.setDateoftransformation(patentDateOfCollection); - result.setResulttype(OAF_ENTITY_RESULT_METADATA_RESULTTYPE); - result.setDataInfo(OAF_ENTITY_DATAINFO); - - if (StringUtils.isNotBlank(patent.getApplnTitle())) { - result.setTitle(Collections.singletonList(buildOafEntityResultMetadataTitle(patent.getApplnTitle()))); - } - - if (StringUtils.isNotBlank(patent.getApplnAbstract())) { - result.setDescription(Collections.singletonList(buildOafEntityResultMetadataDescription(patent.getApplnAbstract()))); - } - - if (StringUtils.isNotBlank(patent.getEarliestPublnDate())) { - result.setDateofacceptance(buildOafEntityResultMetadataDateofacceptance(patent.getEarliestPublnDate())); - } - - if (StringUtils.isNotBlank(patent.getApplnFilingDate())) { - result.setRelevantdate((Collections.singletonList(buildOafEntityResultMetadataRelevantdate(patent.getApplnFilingDate())))); - } - - if (Objects.nonNull(patent.getIpcClassSymbol())) { - result.setSubject(buildOafEntityResultMetadataSubjects(patent.getIpcClassSymbol())); - } - - if (Objects.nonNull(patent.getApplicantNames())) { - result.setAuthor(buildOafEntityResultMetadataAuthors(patent.getApplicantNames())); - } - if (Objects.nonNull(patent.getApplicantCountryCodes())) { - result.setCountry(buildOafEntityResultMetadataCountries(patent.getApplicantCountryCodes())); - } - - List pid = Lists.newArrayList(); - pid.add(buildOafEntityPid(String.format("%s%s", patent.getApplnAuth(), patent.getApplnNr()), OAF_ENTITY_PID_QUALIFIER_CLASS_EPO_ID)); - if (StringUtils.isNotBlank(patent.getApplnNrEpodoc())) { - pid.add(buildOafEntityPid(patent.getApplnNrEpodoc().toString(), OAF_ENTITY_PID_QUALIFIER_CLASS_EPO_NR_EPODOC)); - } - result.setInstance(Collections.singletonList(buildOafEntityResultInstance(patent, patentEpoUrlRoot, pid))); - - setId(result, patentId); - - return result; - } - - private static StructuredProperty buildOafEntityResultMetadataTitle(CharSequence applnTitle) { - StructuredProperty subject = new StructuredProperty(); - subject.setValue(applnTitle.toString()); - subject.setQualifier(OAF_ENTITY_RESULT_METADATA_TITLE_QUALIFIER); - return subject; - } - - private static Field buildOafEntityResultMetadataDescription(CharSequence applnAbstract) { - Field result = new Field<>(); - result.setValue(applnAbstract.toString()); - return result; - } - - private static List buildOafEntityResultMetadataSubjects(List ipcClassSymbols) { - return ipcClassSymbols.stream() - .filter(StringUtils::isNotBlank) - .map(PatentExporterJob::buildOafEntityResultMetadataSubject) - .collect(Collectors.toList()); - } - - private static Subject buildOafEntityResultMetadataSubject(CharSequence ipcClassSymbol) { - Subject subject = new Subject(); - subject.setValue(ipcClassSymbol.toString()); - subject.setQualifier(OAF_ENTITY_RESULT_METADATA_SUBJECT_QUALIFIER); - return subject; - } - - private static Field buildOafEntityResultMetadataDateofacceptance(CharSequence earliestPublnDate) { - Field result = new Field<>(); - result.setValue(earliestPublnDate.toString()); - return result; - } - - private static StructuredProperty buildOafEntityResultMetadataRelevantdate(CharSequence applnFilingDate) { - StructuredProperty relevantDate = new StructuredProperty(); - relevantDate.setValue(applnFilingDate.toString()); - relevantDate.setQualifier(OAF_ENTITY_RESULT_METADATA_RELEVANTDATE_QUALIFIER); - return relevantDate; - } - - private static List buildOafEntityResultMetadataAuthors(List applicantNames) { - List personNames = applicantNames.stream() - .filter(StringUtils::isNotBlank) - .collect(Collectors.toList()); - return ListUtils.zipWithIndex(personNames).stream() - .map(pair -> { - Integer rank = pair.getLeft() + 1; - CharSequence personName = pair.getRight(); - return buildOafEntityResultMetadataAuthor(personName, rank); - }) - .collect(Collectors.toList()); - } - - private static Author buildOafEntityResultMetadataAuthor(CharSequence personName, Integer rank) { - Author author = new Author(); - author.setFullname(personName.toString()); - author.setRank(rank); - return author; - } - - private static List buildOafEntityResultMetadataCountries(List applicantCountryCodes) { - return applicantCountryCodes.stream() - .filter(StringUtils::isNotBlank) - .distinct() - .sorted() - .map(PatentExporterJob::buildOafEntityResultMetadataCountry) - .collect(Collectors.toList()); - } - - private static Country buildOafEntityResultMetadataCountry(CharSequence sourceCountry) { - Country country = new Country(); - country.setClassid(sourceCountry.toString()); - country.setClassname(sourceCountry.toString()); - country.setSchemeid(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_COUNTRIES); - country.setSchemename(InfoSpaceConstants.SEMANTIC_SCHEME_DNET_COUNTRIES); - return country; - } - - private static void setId(Publication publication, String fallbackId) { - publication.setId(fallbackId); - publication.setId(IdentifierFactory.createIdentifier(publication)); - } - - @Parameters(separators = "=") - private static class JobParameters { - @Parameter(names = "-inputDocumentToPatentPath", required = true) - private String inputDocumentToPatentPath; - - @Parameter(names = "-inputPatentPath", required = true) - private String inputPatentPath; - - @Parameter(names = "-outputEntityPath", required = true) - private String outputEntityPath; - - @Parameter(names = "-outputRelationPath", required = true) - private String outputRelationPath; - - @Parameter(names = "-outputReportPath", required = true) - private String outputReportPath; - - @Parameter(names = "-trustLevelThreshold") - private String trustLevelThreshold; - - @Parameter(names = "-collectedFromKey", required = true) - private String collectedFromKey; - - @Parameter(names = "-patentDateOfCollection", required = true) - private String patentDateOfCollection; - - @Parameter(names = "-patentEpoUrlRoot", required = true) - private String patentEpoUrlRoot; - } -} diff --git a/iis-wf/iis-wf-export-actionmanager/src/main/java/eu/dnetlib/iis/wf/export/actionmanager/module/DocumentToPatentActionBuilderModuleFactory.java b/iis-wf/iis-wf-export-actionmanager/src/main/java/eu/dnetlib/iis/wf/export/actionmanager/module/DocumentToPatentActionBuilderModuleFactory.java new file mode 100644 index 000000000..f3a0b47fa --- /dev/null +++ b/iis-wf/iis-wf-export-actionmanager/src/main/java/eu/dnetlib/iis/wf/export/actionmanager/module/DocumentToPatentActionBuilderModuleFactory.java @@ -0,0 +1,81 @@ +package eu.dnetlib.iis.wf.export.actionmanager.module; + +import static eu.dnetlib.iis.wf.export.actionmanager.ExportWorkflowRuntimeParameters.EXPORT_RELATION_COLLECTEDFROM_KEY; + +import java.util.Arrays; +import java.util.List; + +import org.apache.hadoop.conf.Configuration; + +import eu.dnetlib.dhp.schema.action.AtomicAction; +import eu.dnetlib.dhp.schema.oaf.Relation; +import eu.dnetlib.iis.common.InfoSpaceConstants; +import eu.dnetlib.iis.common.WorkflowRuntimeParameters; +import eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent; +import eu.dnetlib.iis.wf.export.actionmanager.IdentifierFactory; +import eu.dnetlib.iis.wf.export.actionmanager.OafConstants; + +/** + * {@link DocumentToPatent} based action builder module. + * + * @author mhorst + * + */ +public class DocumentToPatentActionBuilderModuleFactory extends AbstractActionBuilderFactory { + + + // ------------------------ CONSTRUCTORS -------------------------- + + public DocumentToPatentActionBuilderModuleFactory() { + super(AlgorithmName.document_patent); + } + + // ------------------------ LOGIC --------------------------------- + + @Override + public ActionBuilderModule instantiate(Configuration config) { + return new DocumentToPatentActionBuilderModule(provideTrustLevelThreshold(config), + WorkflowRuntimeParameters.getParamValue(EXPORT_RELATION_COLLECTEDFROM_KEY, config)); + } + + // ------------------------ INNER CLASS -------------------------- + + class DocumentToPatentActionBuilderModule extends AbstractRelationBuilderModule { + + + // ------------------------ CONSTRUCTORS -------------------------- + + /** + * @param trustLevelThreshold trust level threshold or null when all records should be exported + * @param collectedFromKey collectedFrom key to be set for relation + */ + public DocumentToPatentActionBuilderModule(Float trustLevelThreshold, String collectedFromKey) { + super(trustLevelThreshold, buildInferenceProvenance(), collectedFromKey); + } + + // ------------------------ LOGIC -------------------------- + + @Override + public List> build(DocumentToPatent object) throws TrustLevelThresholdExceededException { + return Arrays.asList( + createAction(object.getDocumentId().toString(), generatePatentId(object.getLensId().toString()), + object.getConfidenceLevel(), OafConstants.REL_CLASS_ISRELATEDTO)); + } + + // ------------------------ PRIVATE -------------------------- + + /** + * Creates similarity related actions. + */ + private AtomicAction createAction(String source, String target, float confidenceLevel, + String relClass) throws TrustLevelThresholdExceededException { + return createAtomicActionWithRelation(source, target, OafConstants.REL_TYPE_RESULT_RESULT, + OafConstants.SUBREL_TYPE_RELATIONSHIP, relClass, confidenceLevel); + } + + private String generatePatentId(String lensId) { + return InfoSpaceConstants.ROW_PREFIX_RESULT + InfoSpaceConstants.LENS_ENTITY_ID_PREFIX + + InfoSpaceConstants.ID_NAMESPACE_SEPARATOR + IdentifierFactory.md5(lensId); + } + } +} diff --git a/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/oozie_app/workflow.xml b/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/oozie_app/workflow.xml index e6357d3d6..d65fbd14c 100644 --- a/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/oozie_app/workflow.xml @@ -9,6 +9,10 @@ input_document_to_dataset $UNDEFINED$ + + input_document_to_patent + $UNDEFINED$ + input_document_to_service $UNDEFINED$ @@ -79,6 +83,11 @@ $UNDEFINED$ document_referencedDatasets action-set identifier of exported data + + action_set_id_document_patent + $UNDEFINED$ + document_patent action-set identifier of exported data + action_set_id_document_eoscServices $UNDEFINED$ @@ -136,6 +145,11 @@ $UNDEFINED$ document_referencedDatasets trust level threshold + + trust_level_threshold_document_patent + $UNDEFINED$ + document_patent trust level threshold + trust_level_threshold_document_referencedDocuments $UNDEFINED$ @@ -266,6 +280,10 @@ export.trust.level.threshold.document_referencedDatasets ${trust_level_threshold_document_referencedDatasets} + + export.trust.level.threshold.document_patent + ${trust_level_threshold_document_patent} + export.trust.level.threshold.document_referencedDocuments ${trust_level_threshold_document_referencedDocuments} @@ -298,6 +316,7 @@ + @@ -324,6 +343,13 @@ + + + ${input_document_to_patent eq "$UNDEFINED$"} + + + + ${input_document_to_service eq "$UNDEFINED$"} @@ -446,6 +472,35 @@ + + + + + + + + export.action.builder.factory.classname + eu.dnetlib.iis.wf.export.actionmanager.module.DocumentToPatentActionBuilderModuleFactory + + + mapreduce.input.fileinputformat.inputdir + ${input_document_to_patent} + + + mapreduce.output.fileoutputformat.outputdir + ${output}/document_patent/${action_set_id_document_patent} + + + + mapreduce.job.reduces + ${reduce_tasks} + + + + + + + diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExportCounterReporterTest.java b/iis-wf/iis-wf-export-actionmanager/src/test/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExportCounterReporterTest.java deleted file mode 100644 index 2be666d62..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExportCounterReporterTest.java +++ /dev/null @@ -1,82 +0,0 @@ -package eu.dnetlib.iis.wf.export.actionmanager.entity.patent; - -import eu.dnetlib.iis.common.report.ReportEntryFactory; -import eu.dnetlib.iis.common.schemas.ReportEntry; -import eu.dnetlib.iis.common.spark.TestWithSharedSparkContext; -import eu.dnetlib.iis.common.utils.ListTestUtils; -import org.apache.avro.specific.SpecificRecordBase; -import org.apache.spark.api.java.JavaRDD; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.Captor; -import org.mockito.InjectMocks; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import pl.edu.icm.sparkutils.avro.SparkAvroSaver; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.*; - -@ExtendWith(MockitoExtension.class) -public class PatentExportCounterReporterTest extends TestWithSharedSparkContext { - private static final String outputReportPath = "/path/to/report"; - - @Mock - private SparkAvroSaver avroSaver; - - @Captor - private ArgumentCaptor> reportEntriesCaptor; - - @InjectMocks - private final PatentExportCounterReporter counterReporter = new PatentExportCounterReporter(); - - @Test - public void reportShouldThrowExceptionWhenSparkContextIsNull() { - //when - assertThrows(NullPointerException.class, () -> - counterReporter.report(null, jsc().emptyRDD(), outputReportPath)); - } - - @Test - public void reportShouldThrowExceptionWhenOutputReportPathIsNull() { - //when - assertThrows(NullPointerException.class, () -> - counterReporter.report(jsc(), jsc().emptyRDD(), null)); - } - - @Test - public void reportShouldCreateAndSaveReportAsAvroDatastoreOfReportEntries() { - //given - JavaRDD rdd = jsc().parallelize(Arrays.asList( - new PatentExportMetadata(null, null, "d1", "p1"), - new PatentExportMetadata(null, null, "d1", "p2"), - new PatentExportMetadata(null, null, "d2", "p2") - ) - ); - - //when - counterReporter.report(jsc(), rdd, outputReportPath); - - //then - verify(avroSaver, times(1)) - .saveJavaRDD(reportEntriesCaptor.capture(), eq(ReportEntry.SCHEMA$), eq(outputReportPath)); - List actualReportEntriesAsJson = reportEntriesCaptor.getValue() - .map(SpecificRecordBase::toString).collect() - .stream().sorted().collect(Collectors.toList()); - List expectedReportEntriesAsJson = Stream.of( - ReportEntryFactory.createCounterReportEntry(PatentExportCounterReporter.EXPORTED_PATENT_ENTITIES_COUNTER, 2), - ReportEntryFactory.createCounterReportEntry(PatentExportCounterReporter.PATENT_REFERENCES_COUNTER, 3), - ReportEntryFactory.createCounterReportEntry(PatentExportCounterReporter.DISTINCT_PUBLICATIONS_WITH_PATENT_REFERENCES_COUNTER, 2) - ) - .map(SpecificRecordBase::toString) - .sorted() - .collect(Collectors.toList()); - ListTestUtils.compareLists(actualReportEntriesAsJson, expectedReportEntriesAsJson); - } -} diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExporterJobTest.java b/iis-wf/iis-wf-export-actionmanager/src/test/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExporterJobTest.java deleted file mode 100644 index 099a5b94c..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExporterJobTest.java +++ /dev/null @@ -1,194 +0,0 @@ -package eu.dnetlib.iis.wf.export.actionmanager.entity.patent; - -import eu.dnetlib.dhp.schema.action.AtomicAction; -import eu.dnetlib.dhp.schema.oaf.Publication; -import eu.dnetlib.dhp.schema.oaf.Relation; -import eu.dnetlib.iis.common.ClassPathResourceProvider; -import eu.dnetlib.iis.common.SlowTest; -import eu.dnetlib.iis.common.java.io.SequenceFileTextValueReader; -import eu.dnetlib.iis.common.schemas.ReportEntry; -import eu.dnetlib.iis.common.schemas.ReportEntryType; -import eu.dnetlib.iis.common.utils.AvroTestUtils; -import eu.dnetlib.iis.common.utils.IteratorUtils; -import eu.dnetlib.iis.common.utils.JsonAvroTestUtils; -import eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent; -import eu.dnetlib.iis.referenceextraction.patent.schemas.Patent; -import eu.dnetlib.iis.wf.export.actionmanager.AtomicActionDeserializationUtils; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import pl.edu.icm.sparkutils.test.SparkJob; -import pl.edu.icm.sparkutils.test.SparkJobBuilder; -import pl.edu.icm.sparkutils.test.SparkJobExecutor; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -@SlowTest -public class PatentExporterJobTest { - - private final SparkJobExecutor executor = new SparkJobExecutor(); - - @TempDir - public Path workingDir; - - private String inputDocumentToPatentPath; - private String inputPatentPath; - private String outputRelationPath; - private String outputEntityPath; - private String outputReportPath; - - private static final String INPUT_DOCUMENT_TO_PATENT_PATH = - "eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/document_to_patent.json"; - private static final String INPUT_PATENT_PATH = - "eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/patent.json"; - - private static final String INPUT_DOCUMENT_TO_PATENT_NULLCHECK_PATH = - "eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/nullcheck/document_to_patent.json"; - private static final String INPUT_PATENT_NULLCHECK_PATH = - "eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/nullcheck/patent.json"; - - private static final String PATENT_DATE_OF_COLLECTION = "2019-11-20T23:59"; - private static final String PATENT_EPO_URL_ROOT = "https://register.epo.org/application?number="; - - private static final String RELATION_COLLECTED_FROM_KEY = "someRepo"; - - @BeforeEach - public void before() { - inputDocumentToPatentPath = workingDir.resolve("patent_exporter").resolve("input_document_to_patent").toString(); - inputPatentPath = workingDir.resolve("patent_exporter").resolve("input_patent").toString(); - outputRelationPath = workingDir.resolve("patent_exporter").resolve("output_relation").toString(); - outputEntityPath = workingDir.resolve("patent_exporter").resolve("output_entity").toString(); - outputReportPath = workingDir.resolve("patent_exporter").resolve("output_report").toString(); - } - - @Test - public void shouldNotExportEntitiesWhenConfidenceLevelIsBelowThreshold() throws IOException { - //given - AvroTestUtils.createLocalAvroDataStore( - JsonAvroTestUtils.readJsonDataStore( - ClassPathResourceProvider.getResourcePath(INPUT_DOCUMENT_TO_PATENT_PATH), DocumentToPatent.class), - inputDocumentToPatentPath); - AvroTestUtils.createLocalAvroDataStore( - JsonAvroTestUtils.readJsonDataStore(ClassPathResourceProvider.getResourcePath(INPUT_PATENT_PATH), Patent.class), - inputPatentPath); - SparkJob sparkJob = buildSparkJob("0.99"); - - //when - executor.execute(sparkJob); - - //then - List> actualRelationActions = IteratorUtils.toList(SequenceFileTextValueReader.fromFile(outputRelationPath), - x -> AtomicActionDeserializationUtils.deserializeAction(x.toString())); - assertEquals(0, actualRelationActions.size()); - - List> actualEntityActions = IteratorUtils.toList(SequenceFileTextValueReader.fromFile(outputEntityPath), - x -> AtomicActionDeserializationUtils.deserializeAction(x.toString())); - assertEquals(0, actualEntityActions.size()); - - assertCountersInReport(0, 0, 0); - } - - @Test - public void shouldExportEntitiesWhenConfidenceLevelIsAboveThreshold() throws IOException { - //given - AvroTestUtils.createLocalAvroDataStore( - JsonAvroTestUtils.readJsonDataStore( - ClassPathResourceProvider.getResourcePath(INPUT_DOCUMENT_TO_PATENT_PATH), DocumentToPatent.class), - inputDocumentToPatentPath); - AvroTestUtils.createLocalAvroDataStore( - JsonAvroTestUtils.readJsonDataStore( - ClassPathResourceProvider.getResourcePath(INPUT_PATENT_PATH), Patent.class), - inputPatentPath); - SparkJob sparkJob = buildSparkJob("0.5"); - - //when - executor.execute(sparkJob); - - //then - //relations - List> actualRelationActions = IteratorUtils.toList(SequenceFileTextValueReader.fromFile(outputRelationPath), - x -> AtomicActionDeserializationUtils.deserializeAction(x.toString())); - assertEquals(6, actualRelationActions.size()); - - actualRelationActions.forEach(action -> verifyAction(action, Relation.class)); - - // entities - List> actualEntityActions = IteratorUtils.toList(SequenceFileTextValueReader.fromFile(outputEntityPath), - x -> AtomicActionDeserializationUtils.deserializeAction(x.toString())); - assertEquals(2, actualEntityActions.size()); - - actualEntityActions.forEach(action -> verifyAction(action, Publication.class)); - - //report - assertCountersInReport(3, 2, 2); - } - - @Test - public void shouldNotExportEntitiesNorRelationsWhenEntityTitleIsNull() throws IOException { - //given - AvroTestUtils.createLocalAvroDataStore( - JsonAvroTestUtils.readJsonDataStore( - ClassPathResourceProvider.getResourcePath(INPUT_DOCUMENT_TO_PATENT_NULLCHECK_PATH), DocumentToPatent.class), - inputDocumentToPatentPath); - AvroTestUtils.createLocalAvroDataStore( - JsonAvroTestUtils.readJsonDataStore( - ClassPathResourceProvider.getResourcePath(INPUT_PATENT_NULLCHECK_PATH), Patent.class), - inputPatentPath); - SparkJob sparkJob = buildSparkJob("0.5"); - - //when - executor.execute(sparkJob); - - //then - checking only if no exception is thrown - assertCountersInReport(0, 0, 0); - } - - private SparkJob buildSparkJob(String trustLevelThreshold) { - return SparkJobBuilder.create() - .setAppName(getClass().getName()) - .setMainClass(PatentExporterJob.class) - .addArg("-inputDocumentToPatentPath", inputDocumentToPatentPath) - .addArg("-inputPatentPath", inputPatentPath) - .addArg("-trustLevelThreshold", trustLevelThreshold) - .addArg("-collectedFromKey", RELATION_COLLECTED_FROM_KEY) - .addArg("-patentDateOfCollection", PATENT_DATE_OF_COLLECTION) - .addArg("-patentEpoUrlRoot", PATENT_EPO_URL_ROOT) - .addArg("-outputRelationPath", outputRelationPath) - .addArg("-outputEntityPath", outputEntityPath) - .addArg("-outputReportPath", outputReportPath) - .addJobProperty("spark.driver.host", "localhost") - .build(); - } - - private void verifyAction(AtomicAction action, Class clazz) { - assertEquals(clazz, action.getClazz()); - assertNotNull(action.getPayload()); - assertEquals(clazz, action.getPayload().getClass()); - // comparing action payload is out of the scope of this test - } - - private void assertCountersInReport(Integer expectedReferencesCount, - Integer expectedEntitiesCount, - Integer expectedDistinctPubsReferencesCount) throws IOException { - List reportEntries = AvroTestUtils.readLocalAvroDataStore(outputReportPath); - assertEquals(3, reportEntries.size()); - - assertEquals(ReportEntryType.COUNTER, reportEntries.get(0).getType()); - assertEquals(PatentExportCounterReporter.PATENT_REFERENCES_COUNTER, reportEntries.get(0).getKey().toString()); - assertEquals(expectedReferencesCount, Integer.valueOf(reportEntries.get(0).getValue().toString())); - - assertEquals(ReportEntryType.COUNTER, reportEntries.get(1).getType()); - assertEquals(PatentExportCounterReporter.EXPORTED_PATENT_ENTITIES_COUNTER, reportEntries.get(1).getKey().toString()); - assertEquals(expectedEntitiesCount, Integer.valueOf(reportEntries.get(1).getValue().toString())); - - assertEquals(ReportEntryType.COUNTER, reportEntries.get(2).getType()); - assertEquals(PatentExportCounterReporter.DISTINCT_PUBLICATIONS_WITH_PATENT_REFERENCES_COUNTER, reportEntries.get(2).getKey().toString()); - assertEquals(expectedDistinctPubsReferencesCount, Integer.valueOf(reportEntries.get(2).getValue().toString())); - } - -} diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExporterWorkflowTest.java b/iis-wf/iis-wf-export-actionmanager/src/test/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExporterWorkflowTest.java deleted file mode 100644 index ca62cbd6c..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/java/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/PatentExporterWorkflowTest.java +++ /dev/null @@ -1,13 +0,0 @@ -package eu.dnetlib.iis.wf.export.actionmanager.entity.patent; - -import eu.dnetlib.iis.common.AbstractOozieWorkflowTestCase; -import org.junit.jupiter.api.Test; - -public class PatentExporterWorkflowTest extends AbstractOozieWorkflowTestCase { - - @Test - public void testExportPatent() { - testWorkflow("eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default"); - } - -} diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/java/eu/dnetlib/iis/wf/export/actionmanager/module/DocumentToPatentActionBuilderModuleFactoryTest.java b/iis-wf/iis-wf-export-actionmanager/src/test/java/eu/dnetlib/iis/wf/export/actionmanager/module/DocumentToPatentActionBuilderModuleFactoryTest.java new file mode 100644 index 000000000..fe89a58b3 --- /dev/null +++ b/iis-wf/iis-wf-export-actionmanager/src/test/java/eu/dnetlib/iis/wf/export/actionmanager/module/DocumentToPatentActionBuilderModuleFactoryTest.java @@ -0,0 +1,79 @@ +package eu.dnetlib.iis.wf.export.actionmanager.module; + +import eu.dnetlib.dhp.schema.action.AtomicAction; +import eu.dnetlib.dhp.schema.oaf.Relation; +import eu.dnetlib.iis.common.InfoSpaceConstants; +import eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent; +import eu.dnetlib.iis.wf.export.actionmanager.IdentifierFactory; +import eu.dnetlib.iis.wf.export.actionmanager.OafConstants; +import eu.dnetlib.iis.wf.export.actionmanager.module.VerificationUtils.Expectations; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static eu.dnetlib.iis.wf.export.actionmanager.module.VerificationUtils.assertOafRel; +import static org.junit.jupiter.api.Assertions.*; + +/** + * @author mhorst + * + */ +public class DocumentToPatentActionBuilderModuleFactoryTest extends AbstractActionBuilderModuleFactoryTest { + + + // ----------------------- CONSTRUCTORS ------------------- + + public DocumentToPatentActionBuilderModuleFactoryTest() throws Exception { + super(DocumentToPatentActionBuilderModuleFactory.class, AlgorithmName.document_patent); + } + + // ----------------------- TESTS -------------------------- + + + @Test + public void testBuildBelowThreshold() { + // given + DocumentToPatent documentToPatent = buildDocumentToPatent("documentId", "patentId", 0.4f); + ActionBuilderModule module = factory.instantiate(config); + + // execute + assertThrows(TrustLevelThresholdExceededException.class, () -> module.build(documentToPatent)); + } + + @Test + public void testBuild() throws Exception { + // given + String docId = "documentId"; + String lensId = "lensId"; + float matchStrength = 0.9f; + ActionBuilderModule module = factory.instantiate(config); + + // execute + List> actions = module.build(buildDocumentToPatent(docId, lensId, matchStrength)); + + // assert + assertNotNull(actions); + assertEquals(1, actions.size()); + + AtomicAction action = actions.get(0); + assertNotNull(action); + assertEquals(Relation.class, action.getClazz()); + String patentId = "50|lens.org____::" + IdentifierFactory.md5(lensId); + Expectations expectations = new Expectations(docId, patentId, matchStrength, + OafConstants.REL_TYPE_RESULT_RESULT, OafConstants.SUBREL_TYPE_RELATIONSHIP, + OafConstants.REL_CLASS_ISRELATEDTO); + assertOafRel(action.getPayload(), expectations); + } + + // ----------------------- PRIVATE -------------------------- + + private static DocumentToPatent buildDocumentToPatent(String docId, String lensId, + float confidenceLevel) { + DocumentToPatent.Builder builder = DocumentToPatent.newBuilder(); + builder.setDocumentId(docId); + builder.setLensId(lensId); + builder.setConfidenceLevel(confidenceLevel); + return builder.build(); + } + +} diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/document_to_patent.json b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/document_to_patent.json deleted file mode 100644 index 7c10da05e..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/document_to_patent.json +++ /dev/null @@ -1,7 +0,0 @@ -{"documentId":"document1","appln_nr":"patent1","confidenceLevel":0.9,"textsnippet":null} -{"documentId":"document2","appln_nr":"patent1","confidenceLevel":0.8,"textsnippet":null} -{"documentId":"document2","appln_nr":"patent2","confidenceLevel":0.7,"textsnippet":null} -{"documentId":"document2","appln_nr":"patent2","confidenceLevel":0.6,"textsnippet":null} -{"documentId":"document3","appln_nr":"patent3","confidenceLevel":0.1,"textsnippet":null} -{"documentId":"document1","appln_nr":"non-existing","confidenceLevel":0.9,"textsnippet":null} -{"documentId":"document1","appln_nr":"patent1-notitle","confidenceLevel":0.9,"textsnippet":null} diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/nullcheck/document_to_patent.json b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/nullcheck/document_to_patent.json deleted file mode 100644 index 011718394..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/nullcheck/document_to_patent.json +++ /dev/null @@ -1,2 +0,0 @@ -{"documentId":"document1","appln_nr":"patent1","confidenceLevel":0.9,"textsnippet":null} -{"documentId":"document2","appln_nr":"patent2","confidenceLevel":0.9,"textsnippet":null} diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/nullcheck/patent.json b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/nullcheck/patent.json deleted file mode 100644 index d04cc257c..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/nullcheck/patent.json +++ /dev/null @@ -1,2 +0,0 @@ -{"appln_auth":"EP","appln_nr":"patent1","appln_filing_date":null,"appln_nr_epodoc":null,"earliest_publn_date":null,"appln_abstract":null,"appln_title":null,"ipc_class_symbol":null,"applicant_names":null,"applicant_country_codes":null} -{"appln_auth":"EP","appln_nr":"patent2","appln_filing_date":null,"appln_nr_epodoc":null,"earliest_publn_date":null,"appln_abstract":null,"appln_title":null,"ipc_class_symbol":null,"applicant_names":null,"applicant_country_codes":null} diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/patent.json b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/patent.json deleted file mode 100644 index abc3902f7..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/patent.json +++ /dev/null @@ -1,4 +0,0 @@ -{"appln_auth":"EP","appln_nr":"patent1","appln_filing_date":"2000-02-15","appln_nr_epodoc":"EP20000103094","earliest_publn_date":"2000-09-20","appln_abstract":"This is an abstract for patent 1","appln_title":"Method and means for using additional cards in a mobile station","ipc_class_symbol":["G06K 7/00","G06K 17/00"],"applicant_names":["Nokia Corporation","Lipponen, Markku","Laitinen, Timo"],"applicant_country_codes":["FI","FI","DE"]} -{"appln_auth":"EP","appln_nr":"patent1-notitle","appln_filing_date":"2000-02-15","appln_nr_epodoc":"EP20000103094","earliest_publn_date":"2000-09-20","appln_abstract":"This is an abstract for patent 1","appln_title":null,"ipc_class_symbol":["G06K 7/00","G06K 17/00"],"applicant_names":["Nokia Corporation","Lipponen, Markku","Laitinen, Timo"],"applicant_country_codes":["FI","FI","DE"]} -{"appln_auth":"EP","appln_nr":"patent2","appln_filing_date":"2001-08-29","appln_nr_epodoc":"EP20010119799","earliest_publn_date":"2002-03-13","appln_abstract":"This is an abstract for patent 2","appln_title":"Diphosphines","ipc_class_symbol":["B01J 23/46","B01J 31/24"],"applicant_names":["Saltigo GmbH","Driessen-Hlscher, Birgit, Dr.","Kralik, Joachim"],"applicant_country_codes":["DE","DE","FI"]} -{"appln_auth":"EP","appln_nr":"patent3","appln_filing_date":"2006-10-09","appln_nr_epodoc":"EP20060021113","earliest_publn_date":"2007-04-18","appln_abstract":"This is an abstract for patent 3","appln_title":"Textile material finished with cationic compound and its use","ipc_class_symbol":["C11D 3/37","C11D 17/04","D06C 11/00","D06M 15/61","D06M 15/643"],"applicant_names":["Kornbusch & Starting GmbH & Co. KG","Becker, Christoph"],"applicant_country_codes":["DE","DE"]} diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/oozie_app/import.txt b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/oozie_app/import.txt deleted file mode 100644 index 97a10a464..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/oozie_app/import.txt +++ /dev/null @@ -1,2 +0,0 @@ -## This is a classpath-based import file (this header is required) -export_patent classpath eu/dnetlib/iis/wf/export/actionmanager/entity/patent/oozie_app diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/oozie_app/workflow.xml b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/oozie_app/workflow.xml deleted file mode 100644 index 0b4448757..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/oozie_app/workflow.xml +++ /dev/null @@ -1,155 +0,0 @@ - - - - ${jobTracker} - ${nameNode} - - - mapreduce.job.queuename - ${queueName} - - - oozie.launcher.mapred.job.queue.name - ${oozieLauncherQueueName} - - - - - - - - - - - - - eu.dnetlib.iis.common.java.ProcessWrapper - eu.dnetlib.iis.common.java.jsonworkflownodes.Producer - -C{document_to_patent, - eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent, - eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/document_to_patent.json} - - -C{patent, - eu.dnetlib.iis.referenceextraction.patent.schemas.Patent, - eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/input/patent.json} - - -Odocument_to_patent=${workingDir}/producer/document_to_patent - -Opatent=${workingDir}/producer/patent - - - - - - - - ${wf:appPath()}/export_patent - - - - input_document_to_patent - ${workingDir}/producer/document_to_patent - - - input_patent - ${workingDir}/producer/patent - - - action_set_id_document_patent - patent-actionset-id - - - action_set_id_entity_patent - patent-entity-actionset-id - - - trust_level_threshold_document_patent - 0.5 - - - collectedfrom_key - repo-id-1 - - - patent_date_of_collection - 2019-11-20T23:59 - - - patent_epo_url_root - https://register.epo.org/application?number= - - - output_root_relations - ${workingDir}/output/document_patent - - - output_root_entities - ${workingDir}/output/entities_patent - - - output_report_relative_path - export_patent - - - output_report_root_path - ${workingDir}/report - - - - - - - - - - eu.dnetlib.iis.common.java.ProcessWrapper - eu.dnetlib.iis.wf.export.actionmanager.sequencefile.TestingConsumer - -Iseqfile=${workingDir}/output/document_patent/patent-actionset-id - - -Pexpectation_file_paths=/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document2_to_patent2.properties, - /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent2_to_document2.properties, - /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document1_to_patent1.properties, - /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1_to_document1.properties, - /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document2_to_patent1.properties, - /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1_to_document2.properties - - - - - - - - - eu.dnetlib.iis.common.java.ProcessWrapper - eu.dnetlib.iis.wf.export.actionmanager.sequencefile.TestingConsumer - -Iseqfile=${workingDir}/output/entities_patent/patent-entity-actionset-id - - -Pexpectation_file_paths=/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1.expectations, - /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent2.expectations - - - - - - - - - eu.dnetlib.iis.common.java.ProcessWrapper - eu.dnetlib.iis.common.java.jsonworkflownodes.TestingConsumer - -C{report, - eu.dnetlib.iis.common.schemas.ReportEntry, - eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/report.json} - - -Ireport=${workingDir}/report/export_patent - - - - - - - Unfortunately, the process failed -- error message: - [${wf:errorMessage(wf:lastErrorNode())}] - - - - - diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document2_to_patent1.properties b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document2_to_patent1.properties deleted file mode 100644 index 4398667d0..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document2_to_patent1.properties +++ /dev/null @@ -1,18 +0,0 @@ -clazz.canonicalName=eu.dnetlib.dhp.schema.oaf.Relation - -payload.relType=resultResult -payload.subRelType=relationship -payload.relClass=IsRelatedTo -payload.source=document2 -payload.target=50|epopatstat__::a7c39f7f55a8b97defd39b9bb4f87e36 - -payload.collectedfrom[0].key=repo-id-1 -payload.collectedfrom[0].value=OpenAIRE - -payload.dataInfo.inferred=true -payload.dataInfo.trust=0.72 -payload.dataInfo.inferenceprovenance=iis::document_patent -payload.dataInfo.provenanceaction.classid=iis -payload.dataInfo.provenanceaction.classname=iis -payload.dataInfo.provenanceaction.schemeid=dnet:provenanceActions -payload.dataInfo.provenanceaction.schemename=dnet:provenanceActions \ No newline at end of file diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document2_to_patent2.properties b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document2_to_patent2.properties deleted file mode 100644 index f69e41dba..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document2_to_patent2.properties +++ /dev/null @@ -1,18 +0,0 @@ -clazz.canonicalName=eu.dnetlib.dhp.schema.oaf.Relation - -payload.relType=resultResult -payload.subRelType=relationship -payload.relClass=IsRelatedTo -payload.source=document2 -payload.target=50|epopatstat__::6124072c17594dec919a7465b1697bf1 - -payload.collectedfrom[0].key=repo-id-1 -payload.collectedfrom[0].value=OpenAIRE - -payload.dataInfo.inferred=true -payload.dataInfo.trust=0.63 -payload.dataInfo.inferenceprovenance=iis::document_patent -payload.dataInfo.provenanceaction.classid=iis -payload.dataInfo.provenanceaction.classname=iis -payload.dataInfo.provenanceaction.schemeid=dnet:provenanceActions -payload.dataInfo.provenanceaction.schemename=dnet:provenanceActions \ No newline at end of file diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1.expectations b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1.expectations deleted file mode 100644 index 25e887bec..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1.expectations +++ /dev/null @@ -1,87 +0,0 @@ -clazz.canonicalName=eu.dnetlib.dhp.schema.oaf.Publication - -payload.id=50|epopatstat__::a7c39f7f55a8b97defd39b9bb4f87e36 - -payload.dateofcollection=2019-11-20T23:59:00.000Z -payload.dateoftransformation=2019-11-20T23:59:00.000Z - -payload.title[0].value=Method and means for using additional cards in a mobile station -payload.title[0].qualifier.classid=main title -payload.title[0].qualifier.classname=main title -payload.title[0].qualifier.schemeid=dnet:dataCite_title -payload.title[0].qualifier.schemename=dnet:dataCite_title - -payload.description[0].value=This is an abstract for patent 1 - -payload.subject[0].value=G06K 7/00 -payload.subject[0].qualifier.classid=IPC -payload.subject[0].qualifier.classname=International Patent Classification -payload.subject[0].qualifier.schemeid=dnet:subject_classification_typologies -payload.subject[0].qualifier.schemename=dnet:subject_classification_typologies - -payload.subject[1].value=G06K 17/00 -payload.subject[1].qualifier.classid=IPC -payload.subject[1].qualifier.classname=International Patent Classification -payload.subject[1].qualifier.schemeid=dnet:subject_classification_typologies -payload.subject[1].qualifier.schemename=dnet:subject_classification_typologies - -payload.dateofacceptance.value=2000-09-20 -payload.relevantdate[0].value=2000-02-15 -payload.relevantdate[0].qualifier.classid=submitted -payload.relevantdate[0].qualifier.classname=submitted -payload.relevantdate[0].qualifier.schemeid=dnet:dataCite_date -payload.relevantdate[0].qualifier.schemename=dnet:dataCite_date - -payload.author[0].fullname=Nokia Corporation -payload.author[0].rank=1 -payload.author[1].fullname=Lipponen, Markku -payload.author[1].rank=2 -payload.author[2].fullname=Laitinen, Timo -payload.author[2].rank=3 - -payload.country[0].classid=DE -payload.country[0].classname=DE -payload.country[0].schemeid=dnet:countries -payload.country[0].schemename=dnet:countries - -payload.country[1].classid=FI -payload.country[1].classname=FI -payload.country[1].schemeid=dnet:countries -payload.country[1].schemename=dnet:countries - -payload.resulttype.classid=publication -payload.resulttype.classname=publication -payload.resulttype.schemeid=dnet:result_typologies -payload.resulttype.schemename=dnet:result_typologies - -payload.instance[0].instancetype.classid=0019 -payload.instance[0].instancetype.classname=Patent -payload.instance[0].instancetype.schemeid=dnet:publication_resource -payload.instance[0].instancetype.schemename=dnet:publication_resource - -payload.instance[0].url[0]=https://register.epo.org/application?number=EPpatent1 -payload.instance[0].hostedby.key=10|openaire____::79e8e56d43f34eb7698bfb7535d1b37a -payload.instance[0].hostedby.value=European Patent Office/PATSTAT -payload.instance[0].collectedfrom.key=10|openaire____::79e8e56d43f34eb7698bfb7535d1b37a -payload.instance[0].collectedfrom.value=European Patent Office/PATSTAT -payload.instance[0].alternateIdentifier[0].value=EPpatent1 -payload.instance[0].alternateIdentifier[0].qualifier.classid=epo_id -payload.instance[0].alternateIdentifier[0].qualifier.classname=epo_id -payload.instance[0].alternateIdentifier[0].qualifier.schemeid=dnet:pid_types -payload.instance[0].alternateIdentifier[0].qualifier.schemename=dnet:pid_types -payload.instance[0].alternateIdentifier[1].value=EP20000103094 -payload.instance[0].alternateIdentifier[1].qualifier.classid=epo_nr_epodoc -payload.instance[0].alternateIdentifier[1].qualifier.classname=epo_nr_epodoc -payload.instance[0].alternateIdentifier[1].qualifier.schemeid=dnet:pid_types -payload.instance[0].alternateIdentifier[1].qualifier.schemename=dnet:pid_types - -payload.collectedfrom[0].key=10|openaire____::79e8e56d43f34eb7698bfb7535d1b37a -payload.collectedfrom[0].value=European Patent Office/PATSTAT - -payload.dataInfo.inferred=false -payload.dataInfo.trust=0.9 -payload.dataInfo.inferenceprovenance=iis::document_patent -payload.dataInfo.provenanceaction.classid=iis-entities-patent -payload.dataInfo.provenanceaction.classname=iis-entities-patent -payload.dataInfo.provenanceaction.schemeid=dnet:provenanceActions -payload.dataInfo.provenanceaction.schemename=dnet:provenanceActions diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1_to_document1.properties b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1_to_document1.properties deleted file mode 100644 index 890724b5c..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1_to_document1.properties +++ /dev/null @@ -1,18 +0,0 @@ -clazz.canonicalName=eu.dnetlib.dhp.schema.oaf.Relation - -payload.relType=resultResult -payload.subRelType=relationship -payload.relClass=IsRelatedTo -payload.source=50|epopatstat__::a7c39f7f55a8b97defd39b9bb4f87e36 -payload.target=document1 - -payload.collectedfrom[0].key=repo-id-1 -payload.collectedfrom[0].value=OpenAIRE - -payload.dataInfo.inferred=true -payload.dataInfo.trust=0.81 -payload.dataInfo.inferenceprovenance=iis::document_patent -payload.dataInfo.provenanceaction.classid=iis -payload.dataInfo.provenanceaction.classname=iis -payload.dataInfo.provenanceaction.schemeid=dnet:provenanceActions -payload.dataInfo.provenanceaction.schemename=dnet:provenanceActions \ No newline at end of file diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1_to_document2.properties b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1_to_document2.properties deleted file mode 100644 index ea4f7244d..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1_to_document2.properties +++ /dev/null @@ -1,18 +0,0 @@ -clazz.canonicalName=eu.dnetlib.dhp.schema.oaf.Relation - -payload.relType=resultResult -payload.subRelType=relationship -payload.relClass=IsRelatedTo -payload.source=50|epopatstat__::a7c39f7f55a8b97defd39b9bb4f87e36 -payload.target=document2 - -payload.collectedfrom[0].key=repo-id-1 -payload.collectedfrom[0].value=OpenAIRE - -payload.dataInfo.inferred=true -payload.dataInfo.trust=0.72 -payload.dataInfo.inferenceprovenance=iis::document_patent -payload.dataInfo.provenanceaction.classid=iis -payload.dataInfo.provenanceaction.classname=iis -payload.dataInfo.provenanceaction.schemeid=dnet:provenanceActions -payload.dataInfo.provenanceaction.schemename=dnet:provenanceActions \ No newline at end of file diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent2.expectations b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent2.expectations deleted file mode 100644 index 556f3509f..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent2.expectations +++ /dev/null @@ -1,87 +0,0 @@ -clazz.canonicalName=eu.dnetlib.dhp.schema.oaf.Publication - -payload.id=50|epopatstat__::6124072c17594dec919a7465b1697bf1 - -payload.dateofcollection=2019-11-20T23:59:00.000Z -payload.dateoftransformation=2019-11-20T23:59:00.000Z - -payload.title[0].value=Diphosphines -payload.title[0].qualifier.classid=main title -payload.title[0].qualifier.classname=main title -payload.title[0].qualifier.schemeid=dnet:dataCite_title -payload.title[0].qualifier.schemename=dnet:dataCite_title - -payload.description[0].value=This is an abstract for patent 2 - -payload.subject[0].value=B01J 23/46 -payload.subject[0].qualifier.classid=IPC -payload.subject[0].qualifier.classname=International Patent Classification -payload.subject[0].qualifier.schemeid=dnet:subject_classification_typologies -payload.subject[0].qualifier.schemename=dnet:subject_classification_typologies - -payload.subject[1].value=B01J 31/24 -payload.subject[1].qualifier.classid=IPC -payload.subject[1].qualifier.classname=International Patent Classification -payload.subject[1].qualifier.schemeid=dnet:subject_classification_typologies -payload.subject[1].qualifier.schemename=dnet:subject_classification_typologies - -payload.dateofacceptance.value=2002-03-13 -payload.relevantdate[0].value=2001-08-29 -payload.relevantdate[0].qualifier.classid=submitted -payload.relevantdate[0].qualifier.classname=submitted -payload.relevantdate[0].qualifier.schemeid=dnet:dataCite_date -payload.relevantdate[0].qualifier.schemename=dnet:dataCite_date - -payload.author[0].fullname=Saltigo GmbH -payload.author[0].rank=1 -payload.author[1].fullname=Driessen-Hlscher, Birgit, Dr. -payload.author[1].rank=2 -payload.author[2].fullname=Kralik, Joachim -payload.author[2].rank=3 - -payload.country[0].classid=DE -payload.country[0].classname=DE -payload.country[0].schemeid=dnet:countries -payload.country[0].schemename=dnet:countries - -payload.country[1].classid=FI -payload.country[1].classname=FI -payload.country[1].schemeid=dnet:countries -payload.country[1].schemename=dnet:countries - -payload.resulttype.classid=publication -payload.resulttype.classname=publication -payload.resulttype.schemeid=dnet:result_typologies -payload.resulttype.schemename=dnet:result_typologies - -payload.instance[0].instancetype.classid=0019 -payload.instance[0].instancetype.classname=Patent -payload.instance[0].instancetype.schemeid=dnet:publication_resource -payload.instance[0].instancetype.schemename=dnet:publication_resource - -payload.instance[0].url[0]=https://register.epo.org/application?number=EPpatent2 -payload.instance[0].hostedby.key=10|openaire____::79e8e56d43f34eb7698bfb7535d1b37a -payload.instance[0].hostedby.value=European Patent Office/PATSTAT -payload.instance[0].collectedfrom.key=10|openaire____::79e8e56d43f34eb7698bfb7535d1b37a -payload.instance[0].collectedfrom.value=European Patent Office/PATSTAT -payload.instance[0].alternateIdentifier[0].value=EPpatent2 -payload.instance[0].alternateIdentifier[0].qualifier.classid=epo_id -payload.instance[0].alternateIdentifier[0].qualifier.classname=epo_id -payload.instance[0].alternateIdentifier[0].qualifier.schemeid=dnet:pid_types -payload.instance[0].alternateIdentifier[0].qualifier.schemename=dnet:pid_types -payload.instance[0].alternateIdentifier[1].value=EP20010119799 -payload.instance[0].alternateIdentifier[1].qualifier.classid=epo_nr_epodoc -payload.instance[0].alternateIdentifier[1].qualifier.classname=epo_nr_epodoc -payload.instance[0].alternateIdentifier[1].qualifier.schemeid=dnet:pid_types -payload.instance[0].alternateIdentifier[1].qualifier.schemename=dnet:pid_types - -payload.collectedfrom[0].key=10|openaire____::79e8e56d43f34eb7698bfb7535d1b37a -payload.collectedfrom[0].value=European Patent Office/PATSTAT - -payload.dataInfo.inferred=false -payload.dataInfo.trust=0.9 -payload.dataInfo.inferenceprovenance=iis::document_patent -payload.dataInfo.provenanceaction.classid=iis-entities-patent -payload.dataInfo.provenanceaction.classname=iis-entities-patent -payload.dataInfo.provenanceaction.schemeid=dnet:provenanceActions -payload.dataInfo.provenanceaction.schemename=dnet:provenanceActions diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent2_to_document2.properties b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent2_to_document2.properties deleted file mode 100644 index 2bbde13d1..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent2_to_document2.properties +++ /dev/null @@ -1,18 +0,0 @@ -clazz.canonicalName=eu.dnetlib.dhp.schema.oaf.Relation - -payload.relType=resultResult -payload.subRelType=relationship -payload.relClass=IsRelatedTo -payload.source=50|epopatstat__::6124072c17594dec919a7465b1697bf1 -payload.target=document2 - -payload.collectedfrom[0].key=repo-id-1 -payload.collectedfrom[0].value=OpenAIRE - -payload.dataInfo.inferred=true -payload.dataInfo.trust=0.63 -payload.dataInfo.inferenceprovenance=iis::document_patent -payload.dataInfo.provenanceaction.classid=iis -payload.dataInfo.provenanceaction.classname=iis -payload.dataInfo.provenanceaction.schemeid=dnet:provenanceActions -payload.dataInfo.provenanceaction.schemename=dnet:provenanceActions \ No newline at end of file diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/report.json b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/report.json deleted file mode 100644 index ae8482e2f..000000000 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/report.json +++ /dev/null @@ -1,3 +0,0 @@ -{"key":"processing.referenceExtraction.patent.references", "type":"COUNTER", "value": "3"} -{"key":"export.entities.patent", "type":"COUNTER", "value": "2"} -{"key":"processing.referenceExtraction.patent.docs", "type":"COUNTER", "value": "2"} diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/input/document_to_patent.json b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/input/document_to_patent.json new file mode 100644 index 000000000..f0ee22d60 --- /dev/null +++ b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/input/document_to_patent.json @@ -0,0 +1 @@ +{"documentId":"document1","lensId":"patent1","confidenceLevel":0.9,"textsnippet":null} diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/oozie_app/workflow.xml b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/oozie_app/workflow.xml index ab04dc50a..7208f72ed 100644 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/oozie_app/workflow.xml @@ -37,6 +37,11 @@ eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/input/document_to_dataset.json} -Odocument_to_dataset=${workingDir}/producer/document_to_dataset + -C{document_to_patent, + eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent, + eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/input/document_to_patent.json} + + -Odocument_to_patent=${workingDir}/producer/document_to_patent -C{document_to_service, eu.dnetlib.iis.referenceextraction.service.schemas.DocumentToService, eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/input/document_to_service.json} @@ -104,6 +109,10 @@ input_document_to_dataset ${workingDir}/producer/document_to_dataset + + input_document_to_patent + ${workingDir}/producer/document_to_patent + input_document_to_service ${workingDir}/producer/document_to_service @@ -161,6 +170,10 @@ action_set_id_document_referencedDatasets actionset-id + + action_set_id_document_patent + actionset-id + action_set_id_document_eoscServices actionset-id @@ -219,6 +232,7 @@ + @@ -293,6 +307,17 @@ + + + eu.dnetlib.iis.common.java.ProcessWrapper + eu.dnetlib.iis.wf.export.actionmanager.sequencefile.TestingConsumer + -Iseqfile=${workingDir}/output/document_patent/actionset-id + -Pexpectation_file_paths=/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/output/document_to_patent_1.properties + + + + + eu.dnetlib.iis.common.java.ProcessWrapper diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document1_to_patent1.properties b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/output/document_to_patent_1.properties similarity index 90% rename from iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document1_to_patent1.properties rename to iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/output/document_to_patent_1.properties index c7fb2c1d2..21d86a00e 100644 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document1_to_patent1.properties +++ b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/sequencefile/sampledataproducer/output/document_to_patent_1.properties @@ -4,7 +4,7 @@ payload.relType=resultResult payload.subRelType=relationship payload.relClass=IsRelatedTo payload.source=document1 -payload.target=50|epopatstat__::a7c39f7f55a8b97defd39b9bb4f87e36 +payload.target=50|lens.org____::58220ce949de07493fab9315a7799aeb payload.collectedfrom[0].key=repo-id-1 payload.collectedfrom[0].value=OpenAIRE diff --git a/iis-wf/iis-wf-import/src/main/java/eu/dnetlib/iis/wf/importer/patent/PatentReaderJob.java b/iis-wf/iis-wf-import/src/main/java/eu/dnetlib/iis/wf/importer/patent/PatentReaderJob.java deleted file mode 100644 index 188606d3b..000000000 --- a/iis-wf/iis-wf-import/src/main/java/eu/dnetlib/iis/wf/importer/patent/PatentReaderJob.java +++ /dev/null @@ -1,95 +0,0 @@ -package eu.dnetlib.iis.wf.importer.patent; - -import org.apache.log4j.Logger; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.SparkSession; - -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.collect.Lists; - -import eu.dnetlib.iis.common.java.io.HdfsUtils; -import eu.dnetlib.iis.common.report.ReportEntryFactory; -import eu.dnetlib.iis.common.schemas.ReportEntry; -import eu.dnetlib.iis.common.spark.SparkSessionFactory; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import pl.edu.icm.sparkutils.avro.SparkAvroSaver; - -/** - * Spark job responsible for reading {@link ImportedPatent} records from TSV file. - * - * @author mhorst - */ -public class PatentReaderJob { - - private static SparkAvroSaver avroSaver = new SparkAvroSaver(); - - public static final Logger log = Logger.getLogger(PatentReaderJob.class); - - private static final String COUNTER_READ_TOTAL = "import.patents"; - - - //------------------------ LOGIC -------------------------- - - public static void main(String[] args) throws Exception { - - PatentReaderJobParameters params = new PatentReaderJobParameters(); - JCommander jcommander = new JCommander(params); - jcommander.parse(args); - - try (SparkSession session = SparkSessionFactory.withConfAndKryo(new SparkConf())) { - JavaSparkContext sc = JavaSparkContext.fromSparkContext(session.sparkContext()); - HdfsUtils.remove(sc.hadoopConfiguration(), params.outputPath); - HdfsUtils.remove(sc.hadoopConfiguration(), params.outputReportPath); - - JavaRDD results = session.read() - .option("sep", "\t") - .option("header","true") - .csv(params.inputTsvLocation) - .toJavaRDD() - .map(PatentReaderJob::buildEntry); - - storeInOutput(results, generateReportEntries(sc, results), params.outputPath, params.outputReportPath); - } - } - - //------------------------ PRIVATE -------------------------- - - private static ImportedPatent buildEntry(Row row) { - return ImportedPatent.newBuilder() - .setApplnAuth("EP") - .setApplnNr(row.getAs("appln_nr")) - .setPublnAuth(row.getAs("publn_auth")) - .setPublnNr(row.getAs("publn_nr")) - .setPublnKind(row.getAs("publn_kind")) - .build(); - } - - private static JavaRDD generateReportEntries(JavaSparkContext sparkContext, - JavaRDD entries) { - ReportEntry fromCacheEntitiesCounter = ReportEntryFactory.createCounterReportEntry(COUNTER_READ_TOTAL, entries.count()); - return sparkContext.parallelize(Lists.newArrayList(fromCacheEntitiesCounter), 1); - } - - private static void storeInOutput(JavaRDD results, JavaRDD reports, - String resultOutputPath, String reportOutputPath) { - avroSaver.saveJavaRDD(results, ImportedPatent.SCHEMA$, resultOutputPath); - avroSaver.saveJavaRDD(reports, ReportEntry.SCHEMA$, reportOutputPath); - } - - @Parameters(separators = "=") - private static class PatentReaderJobParameters { - @Parameter(names = "-inputTsvLocation", required = true) - private String inputTsvLocation; - - @Parameter(names = "-outputPath", required = true) - private String outputPath; - - @Parameter(names = "-outputReportPath", required = true) - private String outputReportPath; - } -} diff --git a/iis-wf/iis-wf-import/src/test/java/eu/dnetlib/iis/wf/importer/patent/PatentReaderJobTest.java b/iis-wf/iis-wf-import/src/test/java/eu/dnetlib/iis/wf/importer/patent/PatentReaderJobTest.java deleted file mode 100644 index 8d3d241c9..000000000 --- a/iis-wf/iis-wf-import/src/test/java/eu/dnetlib/iis/wf/importer/patent/PatentReaderJobTest.java +++ /dev/null @@ -1,71 +0,0 @@ -package eu.dnetlib.iis.wf.importer.patent; - -import eu.dnetlib.iis.common.ClassPathResourceProvider; -import eu.dnetlib.iis.common.SlowTest; -import eu.dnetlib.iis.common.java.io.DataStore; -import eu.dnetlib.iis.common.java.io.HdfsTestUtils; -import eu.dnetlib.iis.common.schemas.ReportEntry; -import eu.dnetlib.iis.common.utils.AvroAssertTestUtil; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import org.apache.hadoop.conf.Configuration; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import pl.edu.icm.sparkutils.test.SparkJob; -import pl.edu.icm.sparkutils.test.SparkJobBuilder; -import pl.edu.icm.sparkutils.test.SparkJobExecutor; - -import java.io.IOException; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -@SlowTest -public class PatentReaderJobTest { - private SparkJobExecutor executor = new SparkJobExecutor(); - - @TempDir - public Path workingDir; - - private Path outputDir; - private Path outputReportDir; - - @BeforeEach - public void before() { - outputDir = workingDir.resolve("output"); - outputReportDir = workingDir.resolve("report"); - } - - @Test - public void shouldReadPatentEPOFileAndStorePatentsAsAvroDatastores() throws IOException { - // given - String patentsEpoPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/importer/patent/sampletest/oozie_app/input/patents_epo.tsv"); - String patentsEpoMappedPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/importer/patent/data/output/patents_epo_output.json"); - String reportPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/importer/patent/data/output/report.json"); - SparkJob sparkJob = buildSparkJob(patentsEpoPath); - - // when - executor.execute(sparkJob); - - // then - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputDir.toString(), patentsEpoMappedPath, ImportedPatent.class); - - assertEquals(1, - HdfsTestUtils.countFiles(new Configuration(), outputReportDir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputReportDir.toString(), reportPath, ReportEntry.class); - } - - private SparkJob buildSparkJob(String patentsEpoPath) { - return SparkJobBuilder.create() - .setAppName(getClass().getName()) - .setMainClass(PatentReaderJob.class) - .addArg("-inputTsvLocation", patentsEpoPath) - .addArg("-outputPath", outputDir.toString()) - .addArg("-outputReportPath", outputReportDir.toString()) - .addJobProperty("spark.driver.host", "localhost") - .build(); - } -} diff --git a/iis-wf/iis-wf-import/src/test/java/eu/dnetlib/iis/wf/importer/patent/WorkflowTest.java b/iis-wf/iis-wf-import/src/test/java/eu/dnetlib/iis/wf/importer/patent/WorkflowTest.java deleted file mode 100644 index 22d4b2e7a..000000000 --- a/iis-wf/iis-wf-import/src/test/java/eu/dnetlib/iis/wf/importer/patent/WorkflowTest.java +++ /dev/null @@ -1,12 +0,0 @@ -package eu.dnetlib.iis.wf.importer.patent; - -import eu.dnetlib.iis.common.AbstractOozieWorkflowTestCase; -import org.junit.jupiter.api.Test; - -public class WorkflowTest extends AbstractOozieWorkflowTestCase { - - @Test - public void testImportPatentWorkflow() { - testWorkflow("eu/dnetlib/iis/wf/importer/patent/sampletest"); - } -} diff --git a/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/data/output/patents_epo_output.json b/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/data/output/patents_epo_output.json deleted file mode 100644 index 8e95ad7d5..000000000 --- a/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/data/output/patents_epo_output.json +++ /dev/null @@ -1,5 +0,0 @@ -{"appln_auth":"EP","appln_nr":"00000001","publn_auth":"EP","publn_nr":"1118543","publn_kind":"B1"} -{"appln_auth":"EP","appln_nr":"00000002","publn_auth":"EP","publn_nr":"1217010","publn_kind":"B1"} -{"appln_auth":"EP","appln_nr":"0000003","publn_auth":"WO","publn_nr":"0042078","publn_kind":"A1"} -{"appln_auth":"EP","appln_nr":"0000004","publn_auth":"WO","publn_nr":"0060207","publn_kind":"A1"} -{"appln_auth":"EP","appln_nr":"0000005","publn_auth":"WO","publn_nr":"0040431","publn_kind":"A1"} diff --git a/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/data/output/report.json b/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/data/output/report.json deleted file mode 100644 index fcbeaefd1..000000000 --- a/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/data/output/report.json +++ /dev/null @@ -1 +0,0 @@ -{"key": "import.patents","type":"COUNTER","value":"5"} \ No newline at end of file diff --git a/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/sampletest/oozie_app/import.txt b/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/sampletest/oozie_app/import.txt deleted file mode 100644 index 6ecb05e48..000000000 --- a/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/sampletest/oozie_app/import.txt +++ /dev/null @@ -1,2 +0,0 @@ -## This is a classpath-based import file (this header is required) -import_patent classpath eu/dnetlib/iis/wf/importer/patent/oozie_app \ No newline at end of file diff --git a/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/sampletest/oozie_app/input/patents_epo.tsv b/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/sampletest/oozie_app/input/patents_epo.tsv deleted file mode 100644 index 62b7ffe93..000000000 --- a/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/sampletest/oozie_app/input/patents_epo.tsv +++ /dev/null @@ -1,6 +0,0 @@ -appln_nr publn_auth publn_nr publn_kind -00000001 EP 1118543 B1 -00000002 EP 1217010 B1 -0000003 WO 0042078 A1 -0000004 WO 0060207 A1 -0000005 WO 0040431 A1 \ No newline at end of file diff --git a/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/sampletest/oozie_app/workflow.xml b/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/sampletest/oozie_app/workflow.xml deleted file mode 100644 index ef23c45de..000000000 --- a/iis-wf/iis-wf-import/src/test/resources/eu/dnetlib/iis/wf/importer/patent/sampletest/oozie_app/workflow.xml +++ /dev/null @@ -1,74 +0,0 @@ - - - - - ${jobTracker} - ${nameNode} - - - mapreduce.job.queuename - ${queueName} - - - oozie.launcher.mapred.job.queue.name - ${oozieLauncherQueueName} - - - - - - - - - ${wf:appPath()}/import_patent - - - - input_tsv - ${wf:appPath()}/input/patents_epo.tsv - - - output - ${workingDir}/output - - - output_report_root_path - ${workingDir}/report - - - output_report_relative_path - patent - - - - - - - - - - eu.dnetlib.iis.common.java.ProcessWrapper - eu.dnetlib.iis.common.java.jsonworkflownodes.TestingConsumer - -C{patent, - eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent, - eu/dnetlib/iis/wf/importer/patent/data/output/patents_epo_output.json} - - -C{report, - eu.dnetlib.iis.common.schemas.ReportEntry, - eu/dnetlib/iis/wf/importer/patent/data/output/report.json} - - -Ipatent=${workingDir}/output - -Ireport=${workingDir}/report/patent - - - - - - - Unfortunately, the process failed -- error message: - [${wf:errorMessage(wf:lastErrorNode())}] - - - - - diff --git a/iis-wf/iis-wf-primary/src/main/resources/eu/dnetlib/iis/wf/primary/main/oozie_app/workflow.xml b/iis-wf/iis-wf-primary/src/main/resources/eu/dnetlib/iis/wf/primary/main/oozie_app/workflow.xml index 9776d3f3c..a42dac59f 100644 --- a/iis-wf/iis-wf-primary/src/main/resources/eu/dnetlib/iis/wf/primary/main/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-primary/src/main/resources/eu/dnetlib/iis/wf/primary/main/oozie_app/workflow.xml @@ -167,8 +167,9 @@ WoS types - import_patent_tsv - input TSV file location with patents basic metadata + import_patent_db + /user/dnet.production/iis/cache/lens.org/lens.db + input database file location for patents basic metadata, currently fixed, to be dynamically imported in future @@ -219,10 +220,6 @@ software_webcrawl_cache_location software reference extraction HDFS cache location - - patent_cache_location - patent HDFS cache location - referenceextraction_project_fundingclass_blacklist_regex @@ -313,11 +310,6 @@ $UNDEFINED$ document to patent action-set identifier of exported data - - export_action_set_id_entity_patent - $UNDEFINED$ - action-set identifier of exported data containing patent entities - export_action_set_id_citation_relations $UNDEFINED$ @@ -389,15 +381,6 @@ http://www.rcsb.org/pdb/explore/explore.do?structureId= protein databank URL root part to be concatenated with pdb identifier when forming final URL - - export_patent_date_of_collection - date of collection of patent file formatted as yyyy-MM-dd'T'HH:mm - - - export_patent_epo_url_root - https://register.epo.org/application?number= - EPO patent web archive URL root part to be concatenated with patent auth and nr when forming final URL - output_remote_location optional remote cluster output location where inference output dump should be distcped as sequence files. When not specified results will be exported straight to the ActionManager. @@ -582,11 +565,6 @@ mimetypes_wos ${import_content_mimetypes_wos} - - - input_patent_tsv - ${import_patent_tsv} - resultset_client_read_timeout @@ -731,8 +709,8 @@ ${workingDir}/primary_import/concept - input_patent - ${workingDir}/primary_import/patent + input_patent_db + ${import_patent_db} output_merged_metadata @@ -791,10 +769,6 @@ output_document_to_patent ${workingDir}/exported/document_to_patent - - output_patent_metadata - ${workingDir}/exported/patent_metadata - output_report_root_path ${workingDir}/report @@ -872,10 +846,6 @@ input_document_to_patent ${workingDir}/exported/document_to_patent - - input_patent - ${workingDir}/exported/patent_metadata - active_export_software @@ -942,10 +912,6 @@ action_set_id_document_patent ${export_action_set_id_document_patent} - - action_set_id_entity_patent - ${export_action_set_id_entity_patent} - action_set_id_citation_relations ${export_action_set_id_citation_relations} @@ -1003,14 +969,6 @@ referenceextraction_pdb_url_root ${export_referenceextraction_pdb_url_root} - - patent_date_of_collection - ${export_patent_date_of_collection} - - - patent_epo_url_root - ${export_patent_epo_url_root} - output_report_root_path ${workingDir}/report diff --git a/iis-wf/iis-wf-primary/src/main/resources/eu/dnetlib/iis/wf/primary/processing/oozie_app/workflow.xml b/iis-wf/iis-wf-primary/src/main/resources/eu/dnetlib/iis/wf/primary/processing/oozie_app/workflow.xml index 1de50f657..f63a81b5f 100644 --- a/iis-wf/iis-wf-primary/src/main/resources/eu/dnetlib/iis/wf/primary/processing/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-primary/src/main/resources/eu/dnetlib/iis/wf/primary/processing/oozie_app/workflow.xml @@ -121,8 +121,8 @@ input concept directory - input_patent - input patent + input_patent_db + input patent database file @@ -347,35 +347,6 @@ DocumentText datastore block size affecting number of tasks issued by each text mining module. Introduced to prevent long lasting tasks from occupying cluster resources for too long. - - - patentServiceAuthnConsumerKey - remote EPO endpoint API authentication consumer key, required by OpenPatentWebServiceFacadeFactory - - - patentServiceAuthnConsumerSecret - remote EPO endpoint API authentication consumer secret, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointAuthHost - remote EPO endpoint authentication host, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointAuthUriRoot - remote EPO endpoint authentication URI root, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointOpsHost - remote EPO endpoint OPS host, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointOpsUriRoot - remote EPO endpoint OPS URI root, required by OpenPatentWebServiceFacadeFactory - - - patent_cache_location - patent HDFS cache location - output_merged_metadata @@ -433,10 +404,6 @@ output_document_to_patent patent reference extraction output directory - - output_patent_metadata - patent metadata retrived from EPO endpoint output directory - output_report_root_path @@ -1003,10 +970,6 @@ input_document_text ${workingDir}/transformers_common_union_document_text/out - - cacheRootDir - ${patent_cache_location} - @@ -1017,9 +980,7 @@ - - eu.dnetlib.iis.common.java.ProcessWrapper eu.dnetlib.iis.common.java.jsonworkflownodes.Producer @@ -1027,12 +988,7 @@ eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent, eu/dnetlib/iis/common/data/empty.json} - -C{patent_metadata, - eu.dnetlib.iis.referenceextraction.patent.schemas.Patent, - eu/dnetlib/iis/common/data/empty.json} - -Oreferenceextraction_patent=${output_document_to_patent} - -Opatent_metadata=${output_patent_metadata} diff --git a/iis-wf/iis-wf-primary/src/test/java/eu/dnetlib/iis/wf/primary/processing/FileContentReturningPatentServiceFacadeFactory.java b/iis-wf/iis-wf-primary/src/test/java/eu/dnetlib/iis/wf/primary/processing/FileContentReturningPatentServiceFacadeFactory.java deleted file mode 100644 index 5b6a5f3c8..000000000 --- a/iis-wf/iis-wf-primary/src/test/java/eu/dnetlib/iis/wf/primary/processing/FileContentReturningPatentServiceFacadeFactory.java +++ /dev/null @@ -1,84 +0,0 @@ -package eu.dnetlib.iis.wf.primary.processing; - -import com.google.common.base.Preconditions; -import eu.dnetlib.iis.common.ClassPathResourceProvider; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import eu.dnetlib.iis.wf.importer.facade.ServiceFacadeFactory; -import eu.dnetlib.iis.wf.referenceextraction.FacadeContentRetriever; -import eu.dnetlib.iis.wf.referenceextraction.FacadeContentRetrieverResponse; -import eu.dnetlib.iis.wf.referenceextraction.patent.OpenPatentWebServiceFacadeFactory; -import eu.dnetlib.iis.wf.referenceextraction.patent.PatentWebServiceFacadeException; - -import java.io.Serializable; -import java.util.Map; - -import static eu.dnetlib.iis.wf.referenceextraction.patent.OpenPatentWebServiceFacadeFactory.*; - -/** - * Simple factory producing {@link FacadeContentRetriever} for patent metadata. - *

- * Validates all the parameters required by {@link OpenPatentWebServiceFacadeFactory}. - */ -public class FileContentReturningPatentServiceFacadeFactory implements ServiceFacadeFactory>, Serializable { - - /** - * Simple mock retrieving XML contents as files from classpath. Relies on - * publn_auth, publn_nr, publn_kind fields defined in {@link ImportedPatent} - * while generating filename: - *

- * publn_auth + '.' + publn_nr + '.' + publn_kind + ".xml" - * - * @author mhorst - */ - @Override - public FacadeContentRetriever instantiate(Map parameters) { - validateMandatoryParameters(parameters); - return new FacadeContentRetriever() { - private static final String classPathRoot = "/eu/dnetlib/iis/wf/primary/processing/data/patent/mock_facade_storage/"; - - @Override - protected String buildUrl(ImportedPatent objToBuildUrl) { - StringBuilder strBuilder = new StringBuilder(); - strBuilder.append(objToBuildUrl.getPublnAuth()); - strBuilder.append('.'); - strBuilder.append(objToBuildUrl.getPublnNr()); - strBuilder.append('.'); - strBuilder.append(objToBuildUrl.getPublnKind()); - strBuilder.append(".xml"); - return strBuilder.toString(); - } - - @Override - protected FacadeContentRetrieverResponse retrieveContentOrThrow(String url, int retryCount) throws Exception { - try { - return FacadeContentRetrieverResponse.success(ClassPathResourceProvider.getResourceContent( - classPathRoot + url)); - } catch (Exception e) { - throw new PatentWebServiceFacadeException(e.getMessage()); - } - } - }; - } - - private static void validateMandatoryParameters(Map parameters) { - checkParameter(parameters, PARAM_CONSUMER_KEY, "expectedConsumerKey"); - checkParameter(parameters, PARAM_CONSUMER_SECRET, "expectedConsumerSecret"); - - checkParameter(parameters, PARAM_SERVICE_ENDPOINT_AUTH_HOST, "expectedAuthHost"); - checkParameter(parameters, PARAM_SERVICE_ENDPOINT_AUTH_PORT, "expectedAuthPort"); - checkParameter(parameters, PARAM_SERVICE_ENDPOINT_AUTH_SCHEME, "expectedAuthScheme"); - checkParameter(parameters, PARAM_SERVICE_ENDPOINT_AUTH_URI_ROOT, "expectedAuthUriRoot"); - - checkParameter(parameters, PARAM_SERVICE_ENDPOINT_OPS_HOST, "expectedOpsHost"); - checkParameter(parameters, PARAM_SERVICE_ENDPOINT_OPS_PORT, "expectedOpsPort"); - checkParameter(parameters, PARAM_SERVICE_ENDPOINT_OPS_SCHEME, "expectedOpsScheme"); - checkParameter(parameters, PARAM_SERVICE_ENDPOINT_OPS_URI_ROOT, "expectedOpsUriRoot"); - } - - private static void checkParameter(Map parameters, String paramName, String expectedValue) { - String paramValue = parameters.get(paramName); - Preconditions.checkArgument(expectedValue.equals(paramValue), - "'%s' parameter value: '%s' is different than the expected one: '%s'", paramName, paramValue, expectedValue); - } - -} diff --git a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/meta/patent.json b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/meta/patent.json deleted file mode 100644 index bf02b34ca..000000000 --- a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/meta/patent.json +++ /dev/null @@ -1,2 +0,0 @@ -{"appln_auth": "EP", "appln_nr":"00103094", "publn_auth": "WO", "publn_nr": "123456789", "publn_kind": "A1"} -{"appln_auth": "EP", "appln_nr":"93430009", "publn_auth": "EP", "publn_nr": "0587515", "publn_kind": "A1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/meta/raw_patent.json b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/meta/raw_patent.json new file mode 100644 index 000000000..613ec3247 --- /dev/null +++ b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/meta/raw_patent.json @@ -0,0 +1 @@ +{"c1":"024-040-013-839-479","c2":"2013058681", "normal":"2013058681", "c4":"WO", "c5":"A3"} \ No newline at end of file diff --git a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/text/id-50.txt b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/text/id-50.txt index 265de7e1d..166aef47b 100644 --- a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/text/id-50.txt +++ b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/text/id-50.txt @@ -1,3 +1,3 @@ This is a simple text containing patent reference which should be picked up by reference extraction algorithm. -Process for salting-drying and smoking cold meat products and a device for carrying this out. Patent No. 93430009.6, filing date: July 10. Dieuzeide, R. & Novella, M. (1951) Essai sur la technique des salaisons de poissons. \ No newline at end of file +When should thiopurine therapy during sustained remission in inflammatory bowel disease be stopped? Adv Res Gastroentero Hepatol 2017 4 555647 Ivachtchenko AV, Mitkin OD, Kadieva MG, Okun IM (2013) Substituted phenoxyacetic acids and esters and amides thereof, comprising a 2,6-dioxo-2,3,6,7-tetrahydro-1H-purine-8-yl fragment and constituting A 2A adenosine receptor antagonists, and the use thereof. European Patent WO2013058681 Koles L Furst S Illes P Purine ionotropic (P2X) receptors Curr Pharm Des 2007 13 2368 2384 \ No newline at end of file diff --git a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/oozie_app/import.txt b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/oozie_app/import.txt index 55c8da40e..0cfe6dcff 100644 --- a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/oozie_app/import.txt +++ b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/oozie_app/import.txt @@ -1,2 +1,3 @@ ## This is a classpath-based import file (this header is required) primary_processing classpath eu/dnetlib/iis/wf/primary/processing/oozie_app +sqlite_builder classpath eu/dnetlib/iis/wf/referenceextraction/patent/sqlite_builder/oozie_app \ No newline at end of file diff --git a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/oozie_app/workflow.xml b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/oozie_app/workflow.xml index 993a867c5..efb8def93 100644 --- a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/oozie_app/workflow.xml @@ -20,10 +20,18 @@ org.apache.avro.Schema.Type.NULL - + + + + + + + + + @@ -82,9 +90,9 @@ eu.dnetlib.iis.referenceextraction.softwareurl.schemas.SoftwareHeritageOrigin, eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/meta/sh_origins.json} - -C{patent, - eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent, - eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/meta/patent.json} + -C{raw_patent, + eu.dnetlib.iis.referenceextraction.patent.schemas.PatentReferenceExtractionInput, + eu/dnetlib/iis/wf/primary/processing/sampledataproducer/input/meta/raw_patent.json} -Odocument_metadata=${workingDir}/producer/document_metadata -Odocument_to_project=${workingDir}/producer/document_to_project @@ -98,7 +106,7 @@ -Odocument_text_wos=${workingDir}/producer/document_text_wos -Odocument_text_classpath=${workingDir}/producer/document_text_classpath -Osoftwareheritage_origins=${workingDir}/producer/softwareheritage_origins - -Opatent=${workingDir}/producer/patent + -Oraw_patent=${workingDir}/producer/raw_patent @@ -201,10 +209,35 @@ - + + + + + ${wf:appPath()}/sqlite_builder + + + + workingDir + ${workingDir}/patent_sqlite_builder/working_dir + + + input_patent + ${workingDir}/producer/raw_patent + + + output_patent_db + ${workingDir}/patent_sqlite_builder/lens.db + + + + + + + + ${wf:appPath()}/primary_processing @@ -311,8 +344,8 @@ ${workingDir}/producer/softwareheritage_origins - input_patent - ${workingDir}/producer/patent + input_patent_db + ${workingDir}/patent_sqlite_builder/lens.db output_merged_metadata @@ -342,10 +375,6 @@ output_document_to_patent ${workingDir}/exported/document_to_patent - - output_patent_metadata - ${workingDir}/exported/patent_metadata - output_document_to_pdb ${workingDir}/exported/document_to_pdb @@ -419,17 +448,9 @@ webcrawlLockManagerFactoryClassName eu.dnetlib.iis.common.lock.LockManagerFactoryMock - - patentLockManagerFactoryClassName - eu.dnetlib.iis.common.lock.LockManagerFactoryMock - software_webcrawl_cache_location ${workingDir}/cache/software_webcrawl - - - patent_cache_location - ${workingDir}/cache/patent metric_pusher_creator_class_name @@ -439,58 +460,10 @@ metric_pusher_address ${workingDir}/report/pushgateway - - patentMetadataRetrieverFacadeFactoryClassname - eu.dnetlib.iis.wf.primary.processing.FileContentReturningPatentServiceFacadeFactory - - - patentServiceAuthnConsumerKey - expectedConsumerKey - - - patentServiceAuthnConsumerSecret - expectedConsumerSecret - - - patentServiceEndpointAuthHost - expectedAuthHost - - - patentServiceEndpointAuthPort - expectedAuthPort - - - patentServiceEndpointAuthScheme - expectedAuthScheme - - - patentServiceEndpointAuthUriRoot - expectedAuthUriRoot - - - patentServiceEndpointOpsHost - expectedOpsHost - - - patentServiceEndpointOpsPort - expectedOpsPort - - - patentServiceEndpointOpsScheme - expectedOpsScheme - - - patentServiceEndpointOpsUriRoot - expectedOpsUriRoot - affiliation_matching_number_of_emitted_files 1 - - referenceextraction_patent_number_of_emitted_filesa - 1 - webcrawlNumberOfEmittedFiles 1 @@ -516,6 +489,11 @@ eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/document_to_dataset.json} -Idocument_to_dataset=${workingDir}/exported/document_to_dataset + -C{document_to_patent, + eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent, + eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/document_to_patent.json} + + -Idocument_to_patent=${workingDir}/exported/document_to_patent -C{document_to_project, eu.dnetlib.iis.referenceextraction.project.schemas.DocumentToProject, eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/document_to_project.json} @@ -551,11 +529,6 @@ eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/document_to_patent.json} -Idocument_to_patent=${workingDir}/exported/document_to_patent - -C{patent_metadata, - eu.dnetlib.iis.referenceextraction.patent.schemas.Patent, - eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/extracted_patent_metadata.json} - - -Ipatent_metadata=${workingDir}/exported/patent_metadata -C{citation, eu.dnetlib.iis.export.schemas.Citations, eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/citations.json} @@ -629,18 +602,15 @@ -C{documentToDatasetTotalReport,eu.dnetlib.iis.common.schemas.ReportEntry,eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/document_to_dataset_total.json} -IdocumentToDatasetTotalReport=${workingDir}/report/document_to_dataset_total - -C{documentToProjectFunderReport,eu.dnetlib.iis.common.schemas.ReportEntry,eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/document_to_project_by_funder.json} + -C{documentToPatentReport,eu.dnetlib.iis.common.schemas.ReportEntry,eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/document_to_patent.json} + -IdocumentToPatentReport=${workingDir}/report/document_to_patent + + -C{documentToProjectFunderReport,eu.dnetlib.iis.common.schemas.ReportEntry,eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/document_to_project_by_funder.json} -IdocumentToProjectFunderReport=${workingDir}/report/document_to_project_by_funder -C{docSoftwareUrlWebcrawlReport,eu.dnetlib.iis.common.schemas.ReportEntry,eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/doc_softwareurl_webcrawl.json} -IdocSoftwareUrlWebcrawlReport=${workingDir}/report/referenceextraction_software - -C{patentMetadataRetrieval,eu.dnetlib.iis.common.schemas.ReportEntry,eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/doc_patent_metadata_retrieval.json} - -IpatentMetadataRetrieval=${workingDir}/report/patent_metadata_retrieval - - -C{patentMetadataExtraction,eu.dnetlib.iis.common.schemas.ReportEntry,eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/doc_patent_metadata_extraction.json} - -IpatentMetadataExtraction=${workingDir}/report/patent_metadata_extraction - -C{execTimeReport,eu.dnetlib.iis.common.schemas.ReportEntry,eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/primary-processing-execution-times.json} -IexecTimeReport=${workingDir}/report/primary-processing-execution-times diff --git a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/document_to_patent.json b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/document_to_patent.json index 08fb05aaa..c1ed0d185 100644 --- a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/document_to_patent.json +++ b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/document_to_patent.json @@ -1,6 +1,6 @@ { "documentId": "id-50", - "appln_nr": "93430009", + "lensId": "024-040-013-839-479", "confidenceLevel": 0.8, - "textsnippet": "be picked up by reference extraction algorithm Process for salting drying and smoking cold meat products and a device for carrying this out Patent No 93430009.6 filing date July 10 Dieuzeide R Novella M 1951 Essai" + "textsnippet": "antagonists and the use thereof European Patent WO2013058681 Koles L Furst" } \ No newline at end of file diff --git a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/extracted_patent_metadata.json b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/extracted_patent_metadata.json deleted file mode 100644 index 5713c5209..000000000 --- a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/extracted_patent_metadata.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "appln_auth": "EP", - "appln_nr": "93430009", - "appln_filing_date": "1993-07-01", - "appln_nr_epodoc": "EP19930430009", - "earliest_publn_date": "1994-03-16", - "appln_abstract": "The subject of the present invention is a method and an apparatus for salting/drying and smoking meat products which have been prepared and cut into fillets beforehand, according to which method the said products are exposed to a stream of smoke in a smoking chamber (21), characterised in that the said fillets of products (1) are firstly salted and dried simultaneously by means of a drying/impregnating operation, by placing them in contact with a sodium chloride and sugar mixture in a processing tank (6), and the said fillets of products are then smoked at a low temperature, while the colloidal particles of the smoke are bound by electrostatic means (22/38/40). Applicable to smoking foodstuffs such as fish and meat. \u003cIMAGE\u003e", - "appln_title": "Process and apparatus for salting-drying and cold smoking of meat food products.", - "ipc_class_symbol": [ - "A23B4/056", - "A23B4/044", - "A23B4/052", - "A23B4/28", - "A23B4/30" - ], - "applicant_names": [ - "COOPERATION INTERNATIONAL EN R", - "IFREMER", - "CIRAD COOP INT RECH AGRO DEV" - ], - "applicant_country_codes": [ - "FR", - "FR", - "FR" - ] -} \ No newline at end of file diff --git a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/doc_patent_metadata_extraction.json b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/doc_patent_metadata_extraction.json deleted file mode 100644 index 8f9e9d166..000000000 --- a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/doc_patent_metadata_extraction.json +++ /dev/null @@ -1,2 +0,0 @@ -{"key": "processing.referenceExtraction.patent.metadataextraction.processed.fault", "type": "COUNTER", "value": "0"} -{"key": "processing.referenceExtraction.patent.metadataextraction.processed.total", "type": "COUNTER", "value": "1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/doc_patent_metadata_retrieval.json b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/doc_patent_metadata_retrieval.json deleted file mode 100644 index d4a382550..000000000 --- a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/doc_patent_metadata_retrieval.json +++ /dev/null @@ -1,3 +0,0 @@ -{"key": "processing.referenceExtraction.patent.retrieval.fromCache.total", "type": "COUNTER", "value": "0"} -{"key": "processing.referenceExtraction.patent.retrieval.processed.fault", "type": "COUNTER", "value": "0"} -{"key": "processing.referenceExtraction.patent.retrieval.processed.total", "type": "COUNTER", "value": "1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/document_to_patent.json b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/document_to_patent.json new file mode 100644 index 000000000..80fbe4e8f --- /dev/null +++ b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/document_to_patent.json @@ -0,0 +1 @@ +{"key":"processing.referenceExtraction.patent.references","type":"COUNTER","value":"1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/pushgateway.json b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/pushgateway.json index 707b5605d..8d858cfaf 100644 --- a/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/pushgateway.json +++ b/iis-wf/iis-wf-primary/src/test/resources/eu/dnetlib/iis/wf/primary/processing/sampledataproducer/output/report/pushgateway.json @@ -41,11 +41,7 @@ {"key":"processing.referenceExtraction.dataset.deduped.references","type":"COUNTER","value":"1"} {"key":"processing.referenceExtraction.service.references", "type":"COUNTER", "value": "1"} {"key":"processing.referenceExtraction.service.duration", "type": "DURATION", "value": "$(LONG_RANGE: [1000, 500000])"} -{"key":"processing.referenceExtraction.patent.retrieval.fromCache.total", "type": "COUNTER", "value": "0"} -{"key":"processing.referenceExtraction.patent.retrieval.processed.total","type":"COUNTER","value":"1"} -{"key":"processing.referenceExtraction.patent.retrieval.processed.fault","type":"COUNTER","value": "0"} -{"key":"processing.referenceExtraction.patent.metadataextraction.processed.total","type":"COUNTER","value":"1"} -{"key":"processing.referenceExtraction.patent.metadataextraction.processed.fault","type":"COUNTER","value":"0"} +{"key":"processing.referenceExtraction.patent.references","type":"COUNTER","value":"1"} {"key":"processing.referenceExtraction.patent.duration","type":"DURATION","value":"$(LONG_RANGE: [1000, 700000])"} {"key":"processing.referenceExtraction.project.duration","type":"DURATION","value":"$(LONG_RANGE: [1000, 1000000])"} {"key":"processing.referenceExtraction.project.references.ec","type":"COUNTER","value":"1"} diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/ConnectionDetails.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/ConnectionDetails.java deleted file mode 100644 index bdec686cf..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/ConnectionDetails.java +++ /dev/null @@ -1,99 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent; - -/** - * Open patent web service connection details. - * - * @author mhorst - * - */ -public class ConnectionDetails { - - private final int connectionTimeout; - private final int readTimeout; - private final String authHostName; - private final int authPort; - private final String authScheme; - private final String authUriRoot; - private final String opsHostName; - private final int opsPort; - private final String opsScheme; - private final String opsUriRoot; - private final String consumerCredential; - private final long throttleSleepTime; - private final int maxRetriesCount; - - ConnectionDetails(int connectionTimeout, int readTimeout, - String authHostName, int authPort, String authScheme, String authUriRoot, - String opsHostName, int opsPort, String opsScheme, String opsUriRoot, - String consumerCredential, long throttleSleepTime, int maxRetriesCount) { - this.connectionTimeout = connectionTimeout; - this.readTimeout = readTimeout; - - this.authHostName = authHostName; - this.authPort = authPort; - this.authScheme = authScheme; - this.authUriRoot = authUriRoot; - - this.opsHostName = opsHostName; - this.opsPort = opsPort; - this.opsScheme = opsScheme; - this.opsUriRoot = opsUriRoot; - - this.consumerCredential = consumerCredential; - this.throttleSleepTime = throttleSleepTime; - this.maxRetriesCount = maxRetriesCount; - } - - public int getConnectionTimeout() { - return connectionTimeout; - } - - public int getReadTimeout() { - return readTimeout; - } - - public String getAuthHostName() { - return authHostName; - } - - public int getAuthPort() { - return authPort; - } - - public String getAuthScheme() { - return authScheme; - } - - public String getAuthUriRoot() { - return authUriRoot; - } - - public String getOpsHostName() { - return opsHostName; - } - - public int getOpsPort() { - return opsPort; - } - - public String getOpsScheme() { - return opsScheme; - } - - public String getOpsUriRoot() { - return opsUriRoot; - } - - public String getConsumerCredential() { - return consumerCredential; - } - - public long getThrottleSleepTime() { - return throttleSleepTime; - } - - public int getMaxRetriesCount() { - return maxRetriesCount; - } - -} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/ConnectionDetailsBuilder.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/ConnectionDetailsBuilder.java deleted file mode 100644 index fb88f0d30..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/ConnectionDetailsBuilder.java +++ /dev/null @@ -1,101 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent; - -/** - * Open patent web service connection details builder. - * - * @author mhorst - * - */ -public class ConnectionDetailsBuilder { - - private int connectionTimeout; - private int readTimeout; - private String authHostName; - private int authPort; - private String authScheme; - private String authUriRoot; - private String opsHostName; - private int opsPort; - private String opsScheme; - private String opsUriRoot; - private String consumerCredential; - private long throttleSleepTime; - private int maxRetriesCount; - - public static ConnectionDetailsBuilder newBuilder() { - return new ConnectionDetailsBuilder(); - } - - public ConnectionDetailsBuilder withConnectionTimeout(int connectionTimeout) { - this.connectionTimeout = connectionTimeout; - return this; - } - - public ConnectionDetailsBuilder withReadTimeout(int readTimeout) { - this.readTimeout = readTimeout; - return this; - } - - public ConnectionDetailsBuilder withAuthHostName(String authHostName) { - this.authHostName = authHostName; - return this; - } - - public ConnectionDetailsBuilder withAuthPort(int authPort) { - this.authPort = authPort; - return this; - } - - public ConnectionDetailsBuilder withAuthScheme(String authScheme) { - this.authScheme = authScheme; - return this; - } - - public ConnectionDetailsBuilder withAuthUriRoot(String authUriRoot) { - this.authUriRoot = authUriRoot; - return this; - } - - public ConnectionDetailsBuilder withOpsHostName(String opsHostName) { - this.opsHostName = opsHostName; - return this; - } - - public ConnectionDetailsBuilder withOpsPort(int opsPort) { - this.opsPort = opsPort; - return this; - } - - public ConnectionDetailsBuilder withOpsScheme(String opsScheme) { - this.opsScheme = opsScheme; - return this; - } - - public ConnectionDetailsBuilder withOpsUriRoot(String opsUriRoot) { - this.opsUriRoot = opsUriRoot; - return this; - } - - public ConnectionDetailsBuilder withConsumerCredential(String consumerCredential) { - this.consumerCredential = consumerCredential; - return this; - } - - public ConnectionDetailsBuilder withThrottleSleepTime(long throttleSleepTime) { - this.throttleSleepTime = throttleSleepTime; - return this; - } - - public ConnectionDetailsBuilder withMaxRetriesCount(int maxRetriesCount) { - this.maxRetriesCount = maxRetriesCount; - return this; - } - - public ConnectionDetails build() { - return new ConnectionDetails(connectionTimeout, readTimeout, - authHostName, authPort, authScheme, authUriRoot, - opsHostName, opsPort, opsScheme, opsUriRoot, - consumerCredential, throttleSleepTime, maxRetriesCount); - } -} - diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/OpenPatentWebServiceFacade.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/OpenPatentWebServiceFacade.java deleted file mode 100644 index cafc3cb2f..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/OpenPatentWebServiceFacade.java +++ /dev/null @@ -1,331 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent; - -import static eu.dnetlib.iis.wf.referenceextraction.patent.OpenPatentWebServiceFacadeFactory.DEFAULT_CHARSET; - -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.ObjectOutputStream; -import java.io.Serializable; -import java.net.HttpURLConnection; -import java.util.Collections; -import java.util.Objects; - -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import eu.dnetlib.iis.wf.referenceextraction.FacadeContentRetriever; -import eu.dnetlib.iis.wf.referenceextraction.FacadeContentRetrieverResponse; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.HttpRequest; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; - -import eu.dnetlib.iis.wf.importer.HttpClientUtils; - -/** - * Remote EPO endpoint based patent service facade. - * - * @author mhorst - * - */ -public class OpenPatentWebServiceFacade extends FacadeContentRetriever { - - private static final long serialVersionUID = -9154710658560662015L; - - private static final Logger log = LoggerFactory.getLogger(OpenPatentWebServiceFacade.class); - - private String authUriRoot; - - private String opsUriRoot; - - // security related - private String currentSecurityToken; - - private String consumerCredential; - - private long throttleSleepTime; - - private int maxRetriesCount; - - // fields to be reinitialized after deserialization - private transient JsonParser jsonParser; - - private transient CloseableHttpClient httpClient; - - private transient HttpHost authHost; - - private transient HttpHost opsHost; - - /** - * Serialization / deserialization details. - */ - private SerDe serDe; - - - // ------------------- CONSTRUCTORS ------------------------ - - public OpenPatentWebServiceFacade(ConnectionDetails connDetails) { - - this(buildHttpClient(connDetails.getConnectionTimeout(), connDetails.getReadTimeout()), - new HttpHost(connDetails.getAuthHostName(), connDetails.getAuthPort(), connDetails.getAuthScheme()), - connDetails.getAuthUriRoot(), - new HttpHost(connDetails.getOpsHostName(), connDetails.getOpsPort(), connDetails.getOpsScheme()), - connDetails.getOpsUriRoot(), connDetails.getConsumerCredential(), connDetails.getThrottleSleepTime(), - connDetails.getMaxRetriesCount(), new JsonParser()); - - // persisting for further serialization and deserialization - this.serDe = new SerDe(connDetails.getConnectionTimeout(), connDetails.getReadTimeout(), - connDetails.getAuthHostName(), connDetails.getAuthPort(), connDetails.getAuthScheme(), - connDetails.getOpsHostName(), connDetails.getOpsPort(), connDetails.getOpsScheme()); - } - - /** - * Using this constructor simplifies object instantiation but also makes the whole object non-serializable. - */ - protected OpenPatentWebServiceFacade(CloseableHttpClient httpClient, HttpHost authHost, String authUriRoot, HttpHost opsHost, - String opsUriRoot, String consumerCredential, long throttleSleepTime, int maxRetriesCount, - JsonParser jsonParser) { - this.httpClient = httpClient; - this.authHost = authHost; - this.authUriRoot = authUriRoot; - this.opsHost = opsHost; - this.opsUriRoot = opsUriRoot; - this.consumerCredential = consumerCredential; - this.throttleSleepTime = throttleSleepTime; - this.maxRetriesCount = maxRetriesCount; - this.jsonParser = jsonParser; - } - - /** - * Builds url to query for a patent. - * - * @param objToBuildUrl Patent metadata for building query url. - * @return Url to query for specific patent. - */ - @Override - protected String buildUrl(ImportedPatent objToBuildUrl) { - StringBuilder strBuilder = new StringBuilder(opsUriRoot); - if (!opsUriRoot.endsWith("/")) { - strBuilder.append('/'); - } - strBuilder.append(objToBuildUrl.getPublnAuth()); - strBuilder.append('.'); - strBuilder.append(objToBuildUrl.getPublnNr()); - strBuilder.append('.'); - strBuilder.append(objToBuildUrl.getPublnKind()); - strBuilder.append("/biblio"); - return strBuilder.toString(); - } - - /** - * Retrieves patent metadata from EPO endpoint. - * - * This method is recursive and requires response entity to be consumed in order - * not to hit the ConnectionPoolTimeoutException when connecting the same host - * more than 2 times within recursion (e.g. when reattepmting). - */ - @Override - protected FacadeContentRetrieverResponse retrieveContentOrThrow(String url, int retryCount) throws Exception { - if (retryCount > maxRetriesCount) { - return failureWhenOverMaxRetries(url, maxRetriesCount); - } - - HttpRequest httpRequest = buildPatentMetaRequest(url, getSecurityToken()); - try (CloseableHttpResponse httpResponse = httpClient.execute(opsHost, httpRequest)) { - int statusCode = httpResponse.getStatusLine().getStatusCode(); - - switch (statusCode) { - case HttpURLConnection.HTTP_OK: { - HttpEntity entity = httpResponse.getEntity(); - if (!isResponseEntityValid(entity)) { - return FacadeContentRetrieverResponse.persistentFailure(new PatentWebServiceFacadeException( - "got empty entity in response, server response: " + httpResponse)); - } - return FacadeContentRetrieverResponse.success(EntityUtils.toString(entity)); - } - case HttpURLConnection.HTTP_BAD_REQUEST: { - log.info("got 400 HTTP code in response, potential reason: access token invalid or expired, server response: {}", - httpResponse); - reauthenticate(); - return retrieveContentOrThrow(url, ++retryCount); - } - case HttpURLConnection.HTTP_FORBIDDEN: { - log.warn("got 403 HTTP code in response, potential reason: endpoint rate limit reached. Delaying for {} ms, server response: {}", - throttleSleepTime, httpResponse); - Thread.sleep(throttleSleepTime); - return retrieveContentOrThrow(url, ++retryCount); - } - case HttpURLConnection.HTTP_NOT_FOUND: { - return FacadeContentRetrieverResponse.persistentFailure(new PatentWebServiceFacadeException( - "unable to find element at: " + httpRequest.getRequestLine())); - } - default: { - return FacadeContentRetrieverResponse.persistentFailure(new PatentWebServiceFacadeException(String.format( - "got unhandled HTTP status code when accessing endpoint: %d, full status: %s, server response: %s", - statusCode, httpResponse.getStatusLine(), httpResponse))); - } - } - } - } - - // -------------------------- PRIVATE ------------------------- - - private static Boolean isResponseEntityValid(HttpEntity entity){ - return Objects.nonNull(entity); - } - - private void reinitialize(SerDe serDe, String authUriRoot, String opsUriRoot, - String consumerCredential, long throttleSleepTime, int maxRetriesCount) { - - this.serDe = serDe; - - this.httpClient = buildHttpClient(serDe.connectionTimeout, serDe.readTimeout); - this.authHost = new HttpHost(serDe.authHostName, serDe.authPort, serDe.authScheme); - this.opsHost = new HttpHost(serDe.opsHostName, serDe.opsPort, serDe.opsScheme); - - this.authUriRoot = authUriRoot; - this.opsUriRoot = opsUriRoot; - - this.consumerCredential = consumerCredential; - this.throttleSleepTime = throttleSleepTime; - this.maxRetriesCount = maxRetriesCount; - - this.jsonParser = new JsonParser(); - } - - /** - * Builds HTTP client issuing requests to SH endpoint. - */ - protected static CloseableHttpClient buildHttpClient(int connectionTimeout, int readTimeout) { - return HttpClientUtils.buildHttpClient(connectionTimeout, readTimeout); - } - - protected String getSecurityToken() throws Exception { - if (StringUtils.isBlank(this.currentSecurityToken)) { - reauthenticate(); - } - return currentSecurityToken; - } - - protected void reauthenticate() throws Exception { - currentSecurityToken = authenticate(); - } - - /** - * Handles authentication operation resulting in a generation of security token. - * Never returns null. - * - * @throws IOException - */ - private String authenticate() throws Exception { - - try (CloseableHttpResponse httpResponse = httpClient.execute(authHost, buildAuthRequest(consumerCredential, authUriRoot))) { - int statusCode = httpResponse.getStatusLine().getStatusCode(); - - if (statusCode == 200) { - String jsonContent = IOUtils.toString(httpResponse.getEntity().getContent(), DEFAULT_CHARSET); - - JsonObject jsonObject = jsonParser.parse(jsonContent).getAsJsonObject(); - JsonElement accessToken = jsonObject.get("access_token"); - - if (accessToken == null) { - throw new PatentWebServiceFacadeException("access token is missing: " + jsonContent); - } else { - return accessToken.getAsString(); - } - } else { - throw new PatentWebServiceFacadeException(String.format( - "Authentication failed! HTTP status code when accessing endpoint: %d, full status: %s, server response: %s", - statusCode, httpResponse.getStatusLine(), EntityUtils.toString(httpResponse.getEntity()))); - } - } - } - - protected static HttpRequest buildAuthRequest(String consumerCredential, String uriRoot) { - HttpPost httpRequest = new HttpPost(uriRoot); - httpRequest.addHeader("Authorization", "Basic " + consumerCredential); - BasicNameValuePair grantType = new BasicNameValuePair("grant_type", "client_credentials"); - httpRequest.setEntity(new UrlEncodedFormEntity(Collections.singletonList(grantType), DEFAULT_CHARSET)); - return httpRequest; - } - - protected static HttpRequest buildPatentMetaRequest(String url, String bearerToken) { - HttpGet httpRequest = new HttpGet(url); - httpRequest.addHeader("Authorization", "Bearer " + bearerToken); - return httpRequest; - } - - // -------------------------- SerDe -------------------------------- - - private void writeObject(ObjectOutputStream oos) throws IOException { - if (this.serDe == null) { - throw new IOException("unable to serialize object: " - + "http connection related details are missing!"); - } - oos.defaultWriteObject(); - oos.writeObject(this.serDe); - oos.writeObject(this.authUriRoot); - oos.writeObject(this.opsUriRoot); - oos.writeObject(this.consumerCredential); - oos.writeObject(this.throttleSleepTime); - oos.writeObject(this.maxRetriesCount); - } - - private void readObject(ObjectInputStream ois) throws ClassNotFoundException, IOException { - ois.defaultReadObject(); - reinitialize((SerDe) ois.readObject(), - (String) ois.readObject(), (String) ois.readObject(), - (String) ois.readObject(), (Long) ois.readObject(), (Integer) ois.readObject()); - } - - // -------------------------- INNER CLASS -------------------------- - - static class SerDe implements Serializable { - - private static final long serialVersionUID = 1289144732356257009L; - - // http client related - private int connectionTimeout; - - private int readTimeout; - - // EPO endpoints - private String authHostName; - - private int authPort; - - private String authScheme; - - private String opsHostName; - - private int opsPort; - - private String opsScheme; - - public SerDe(int connectionTimeout, int readTimeout, - String authHostName, int authPort, String authScheme, - String opsHostName, int opsPort, String opsScheme) { - this.connectionTimeout = connectionTimeout; - this.readTimeout = readTimeout; - this.authHostName = authHostName; - this.authPort = authPort; - this.authScheme = authScheme; - this.opsHostName = opsHostName; - this.opsPort = opsPort; - this.opsScheme = opsScheme; - } - } - -} diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/OpenPatentWebServiceFacadeFactory.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/OpenPatentWebServiceFacadeFactory.java deleted file mode 100644 index 4f0fcd36f..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/OpenPatentWebServiceFacadeFactory.java +++ /dev/null @@ -1,86 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent; - -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.Map; - -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import eu.dnetlib.iis.wf.referenceextraction.FacadeContentRetriever; -import org.apache.commons.lang3.StringUtils; - -import com.google.common.base.Preconditions; - -import eu.dnetlib.iis.wf.importer.facade.ServiceFacadeFactory; - -/** - * RESTful Open Patent WebService based facade factory. - * - * @author mhorst - * - */ -public class OpenPatentWebServiceFacadeFactory implements ServiceFacadeFactory> { - - public static final Charset DEFAULT_CHARSET = StandardCharsets.UTF_8; - - public static final String PARAM_CONSUMER_KEY = "authnConsumerKey"; - public static final String PARAM_CONSUMER_SECRET = "authnConsumerSecret"; - - public static final String PARAM_SERVICE_ENDPOINT_AUTH_HOST = "endpointAuthHost"; - public static final String PARAM_SERVICE_ENDPOINT_AUTH_PORT = "endpointAuthPort"; - public static final String PARAM_SERVICE_ENDPOINT_AUTH_SCHEME = "endpointAuthScheme"; - public static final String PARAM_SERVICE_ENDPOINT_AUTH_URI_ROOT = "endpointAuthUriRoot"; - - public static final String PARAM_SERVICE_ENDPOINT_OPS_HOST = "endpointOpsHost"; - public static final String PARAM_SERVICE_ENDPOINT_OPS_PORT = "endpointOpsPort"; - public static final String PARAM_SERVICE_ENDPOINT_OPS_SCHEME = "endpointOpsScheme"; - public static final String PARAM_SERVICE_ENDPOINT_OPS_URI_ROOT = "endpointOpsUriRoot"; - - public static final String PARAM_SERVICE_ENDPOINT_READ_TIMEOUT = "endpointReadTimeout"; - public static final String PARAM_SERVICE_ENDPOINT_CONNECTION_TIMEOUT = "endpointConnectionTimeout"; - - public static final String PARAM_SERVICE_ENDPOINT_THROTTLE_SLEEP_TIME = "endpointThrottleSleepTime"; - public static final String PARAM_SERVICE_ENDPOINT_RETRIES_COUNT = "endpointRetriesCount"; - - @Override - public FacadeContentRetriever instantiate(Map conf) { - - Preconditions.checkArgument(StringUtils.isNotBlank(conf.get(PARAM_CONSUMER_KEY))); - Preconditions.checkArgument(StringUtils.isNotBlank(conf.get(PARAM_CONSUMER_SECRET))); - - Preconditions.checkArgument(StringUtils.isNotBlank(conf.get(PARAM_SERVICE_ENDPOINT_AUTH_HOST))); - Preconditions.checkArgument(StringUtils.isNotBlank(conf.get(PARAM_SERVICE_ENDPOINT_AUTH_PORT))); - Preconditions.checkArgument(StringUtils.isNotBlank(conf.get(PARAM_SERVICE_ENDPOINT_AUTH_SCHEME))); - Preconditions.checkArgument(StringUtils.isNotBlank(conf.get(PARAM_SERVICE_ENDPOINT_AUTH_URI_ROOT))); - - Preconditions.checkArgument(StringUtils.isNotBlank(conf.get(PARAM_SERVICE_ENDPOINT_OPS_HOST))); - Preconditions.checkArgument(StringUtils.isNotBlank(conf.get(PARAM_SERVICE_ENDPOINT_OPS_PORT))); - Preconditions.checkArgument(StringUtils.isNotBlank(conf.get(PARAM_SERVICE_ENDPOINT_OPS_SCHEME))); - Preconditions.checkArgument(StringUtils.isNotBlank(conf.get(PARAM_SERVICE_ENDPOINT_OPS_URI_ROOT))); - - String connectionTimeout = conf.getOrDefault(PARAM_SERVICE_ENDPOINT_CONNECTION_TIMEOUT, "60000"); - String readTimeout = conf.getOrDefault(PARAM_SERVICE_ENDPOINT_READ_TIMEOUT, "60000"); - String throttleSleepTime = conf.getOrDefault(PARAM_SERVICE_ENDPOINT_THROTTLE_SLEEP_TIME, "10000"); - String retriesCount = conf.getOrDefault(PARAM_SERVICE_ENDPOINT_RETRIES_COUNT, "10"); - - return new OpenPatentWebServiceFacade(ConnectionDetailsBuilder.newBuilder() - .withConnectionTimeout(Integer.parseInt(connectionTimeout)) - .withReadTimeout(Integer.parseInt(readTimeout)) - .withAuthHostName(conf.get(PARAM_SERVICE_ENDPOINT_AUTH_HOST)) - .withAuthPort(Integer.parseInt(conf.get(PARAM_SERVICE_ENDPOINT_AUTH_PORT))) - .withAuthScheme(conf.get(PARAM_SERVICE_ENDPOINT_AUTH_SCHEME)) - .withAuthUriRoot(conf.get(PARAM_SERVICE_ENDPOINT_AUTH_URI_ROOT)) - .withOpsHostName(conf.get(PARAM_SERVICE_ENDPOINT_OPS_HOST)) - .withOpsPort(Integer.parseInt(conf.get(PARAM_SERVICE_ENDPOINT_OPS_PORT))) - .withOpsScheme(conf.get(PARAM_SERVICE_ENDPOINT_OPS_SCHEME)) - .withOpsUriRoot(conf.get(PARAM_SERVICE_ENDPOINT_OPS_URI_ROOT)) - .withConsumerCredential(buildCredential(conf.get(PARAM_CONSUMER_KEY), conf.get(PARAM_CONSUMER_SECRET))) - .withThrottleSleepTime(Long.parseLong(throttleSleepTime)) - .withMaxRetriesCount(Integer.parseInt(retriesCount)).build()); - } - - protected static String buildCredential(String key, String secret) { - return Base64.getEncoder().encodeToString((key + ':' + secret).getBytes(DEFAULT_CHARSET)); - } - -} diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentDBBuilder.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentDBBuilder.java index 3fba70b62..1f6b1ced4 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentDBBuilder.java +++ b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentDBBuilder.java @@ -5,6 +5,8 @@ import java.util.Map; import eu.dnetlib.iis.referenceextraction.patent.schemas.PatentReferenceExtractionInput; + +import org.apache.commons.io.FileUtils; import org.apache.commons.lang.StringUtils; import com.google.common.base.Preconditions; @@ -36,6 +38,7 @@ public ProcessExecutionContext initializeProcess(Map parameters) String targetDbLocation = System.getProperty("java.io.tmpdir") + File.separatorChar + "patents.db"; File targetDbFile = new File(targetDbLocation); + FileUtils.copyFile(new File("scripts/base_lens.db"), targetDbFile); targetDbFile.setWritable(true); return new ProcessExecutionContext( diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentMetadataExtractorJob.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentMetadataExtractorJob.java deleted file mode 100644 index 028e48ac4..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentMetadataExtractorJob.java +++ /dev/null @@ -1,133 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent; - -import java.io.IOException; - -import org.apache.log4j.Logger; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; - -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.collect.Lists; - -import eu.dnetlib.iis.audit.schemas.Fault; -import eu.dnetlib.iis.common.fault.FaultUtils; -import eu.dnetlib.iis.common.java.io.HdfsUtils; -import eu.dnetlib.iis.common.report.ReportEntryFactory; -import eu.dnetlib.iis.common.schemas.ReportEntry; -import eu.dnetlib.iis.common.spark.JavaSparkContextFactory; -import eu.dnetlib.iis.metadataextraction.schemas.DocumentText; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import eu.dnetlib.iis.referenceextraction.patent.schemas.Patent; -import eu.dnetlib.iis.wf.referenceextraction.patent.parser.OpsPatentMetadataXPathBasedParser; -import eu.dnetlib.iis.wf.referenceextraction.patent.parser.PatentMetadataParser; -import eu.dnetlib.iis.wf.referenceextraction.patent.parser.PatentMetadataParserException; -import pl.edu.icm.sparkutils.avro.SparkAvroLoader; -import pl.edu.icm.sparkutils.avro.SparkAvroSaver; -import scala.Tuple2; - -/** - * Job responsible for extracting {@link Patent} metadata out of the XML file - * obtained from EPO endpoint. - * - * @author mhorst - * - */ -public class PatentMetadataExtractorJob { - - protected static final String COUNTER_PROCESSED_TOTAL = "processing.referenceExtraction.patent.metadataextraction.processed.total"; - - protected static final String COUNTER_PROCESSED_FAULT = "processing.referenceExtraction.patent.metadataextraction.processed.fault"; - - - private static final Logger log = Logger.getLogger(PatentMetadataExtractorJob.class); - - private static final SparkAvroLoader avroLoader = new SparkAvroLoader(); - private static final SparkAvroSaver avroSaver = new SparkAvroSaver(); - - // ------------------------ LOGIC -------------------------- - - public static void main(String[] args) throws IOException { - JobParameters params = new JobParameters(); - JCommander jcommander = new JCommander(params); - jcommander.parse(args); - - try (JavaSparkContext sc = JavaSparkContextFactory.withConfAndKryo(new SparkConf())) { - HdfsUtils.remove(sc.hadoopConfiguration(), params.outputPath); - HdfsUtils.remove(sc.hadoopConfiguration(), params.outputFaultPath); - HdfsUtils.remove(sc.hadoopConfiguration(), params.outputReportPath); - - PatentMetadataParser parser = new OpsPatentMetadataXPathBasedParser(); - - JavaRDD importedPatent = avroLoader.loadJavaRDD(sc, params.inputImportedPatentPath, ImportedPatent.class); - JavaRDD toBeProcessedContents = avroLoader.loadJavaRDD(sc, params.inputDocumentTextPath, DocumentText.class); - - JavaPairRDD> pairedInput = toBeProcessedContents - .mapToPair(x -> new Tuple2<>(x.getId(), x)) - .join(importedPatent.mapToPair(x -> new Tuple2<>(x.getApplnNr(), x))); - - JavaRDD> parsedPatentsWithFaults = pairedInput - .map(x -> parse(x._2._1, x._2._2, parser)); - - JavaRDD patents = parsedPatentsWithFaults.filter(e -> e._1 != null).map(e -> e._1); - patents.cache(); - JavaRDD faults = parsedPatentsWithFaults.filter(e -> e._2 != null).map(e -> e._2); - faults.cache(); - - avroSaver.saveJavaRDD(patents, Patent.SCHEMA$, params.outputPath); - avroSaver.saveJavaRDD(faults, Fault.SCHEMA$, params.outputFaultPath); - avroSaver.saveJavaRDD(generateReportEntries(sc, patents, faults), ReportEntry.SCHEMA$, - params.outputReportPath); - } - } - - // ------------------------ PRIVATE -------------------------- - - private static JavaRDD generateReportEntries(JavaSparkContext sparkContext, - JavaRDD patents, JavaRDD faults) { - - long faultCount = faults.count(); - ReportEntry processedPatentsCounter = ReportEntryFactory.createCounterReportEntry(COUNTER_PROCESSED_TOTAL, patents.count() + faultCount); - ReportEntry processedFaultsCounter = ReportEntryFactory.createCounterReportEntry(COUNTER_PROCESSED_FAULT, faultCount); - - return sparkContext.parallelize(Lists.newArrayList(processedPatentsCounter, processedFaultsCounter), 1); - } - - private static Tuple2 parse(DocumentText patent, ImportedPatent importedPatent, PatentMetadataParser parser) { - try { - Patent.Builder resultBuilder = fillDataFromImport(Patent.newBuilder(), importedPatent); - return new Tuple2<>(parser.parse(patent.getText(), resultBuilder).build(), null); - } catch (PatentMetadataParserException e) { - log.error("error while parsing xml contents of patent id " + patent.getId() + ", text content: " - + patent.getText(), e); - return new Tuple2<>(null, FaultUtils.exceptionToFault(importedPatent.getApplnNr(), e, null)); - } - } - - private static Patent.Builder fillDataFromImport(Patent.Builder patentBuilder, ImportedPatent importedPatent) { - patentBuilder.setApplnAuth(importedPatent.getApplnAuth()); - patentBuilder.setApplnNr(importedPatent.getApplnNr()); - return patentBuilder; - } - - @Parameters(separators = "=") - private static class JobParameters { - @Parameter(names = "-inputImportedPatentPath", required = true) - private String inputImportedPatentPath; - - @Parameter(names = "-inputDocumentTextPath", required = true) - private String inputDocumentTextPath; - - @Parameter(names = "-outputPath", required = true) - private String outputPath; - - @Parameter(names = "-outputFaultPath", required = true) - private String outputFaultPath; - - @Parameter(names = "-outputReportPath", required = true) - private String outputReportPath; - } -} diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentMetadataRetrieverJob.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentMetadataRetrieverJob.java deleted file mode 100644 index eac6826ec..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentMetadataRetrieverJob.java +++ /dev/null @@ -1,231 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent; - -import java.util.Map; - -import eu.dnetlib.iis.wf.referenceextraction.FacadeContentRetriever; -import eu.dnetlib.iis.wf.referenceextraction.FacadeContentRetrieverResponse; -import org.apache.commons.lang3.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.Optional; - -import com.beust.jcommander.DynamicParameter; -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; - -import eu.dnetlib.iis.audit.schemas.Fault; -import eu.dnetlib.iis.common.cache.CacheMetadataManagingProcess; -import eu.dnetlib.iis.common.cache.CacheStorageUtils; -import eu.dnetlib.iis.common.cache.CacheStorageUtils.CacheRecordType; -import eu.dnetlib.iis.common.cache.CacheStorageUtils.CachedStorageJobParameters; -import eu.dnetlib.iis.common.cache.CacheStorageUtils.OutputPaths; -import eu.dnetlib.iis.common.fault.FaultUtils; -import eu.dnetlib.iis.common.java.io.HdfsUtils; -import eu.dnetlib.iis.common.lock.LockManager; -import eu.dnetlib.iis.common.lock.LockManagerUtils; -import eu.dnetlib.iis.common.report.ReportEntryFactory; -import eu.dnetlib.iis.common.schemas.ReportEntry; -import eu.dnetlib.iis.common.spark.JavaSparkContextFactory; -import eu.dnetlib.iis.metadataextraction.schemas.DocumentText; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import eu.dnetlib.iis.wf.importer.facade.ServiceFacadeUtils; -import pl.edu.icm.sparkutils.avro.SparkAvroLoader; -import pl.edu.icm.sparkutils.avro.SparkAvroSaver; -import scala.Tuple2; - -/** - * Job responsible for retrieving full patent metadata via {@link OpenPatentWebServiceFacade} based on - * {@link ImportedPatent} input. Stores results in cache for further usage. - */ -public class PatentMetadataRetrieverJob { - - private static final String COUNTER_PROCESSED_TOTAL = "processing.referenceExtraction.patent.retrieval.processed.total"; - - private static final String COUNTER_PROCESSED_FAULT = "processing.referenceExtraction.patent.retrieval.processed.fault"; - - private static final String COUNTER_FROMCACHE_TOTAL = "processing.referenceExtraction.patent.retrieval.fromCache.total"; - - private static final SparkAvroLoader avroLoader = new SparkAvroLoader(); - private static final SparkAvroSaver avroSaver = new SparkAvroSaver(); - - //------------------------ LOGIC -------------------------- - - public static void main(String[] args) throws Exception { - JobParameters params = new JobParameters(); - JCommander jcommander = new JCommander(params); - jcommander.parse(args); - - try (JavaSparkContext sc = JavaSparkContextFactory.withConfAndKryo(new SparkConf())) { - - Configuration hadoopConf = sc.hadoopConfiguration(); - - HdfsUtils.remove(hadoopConf, params.getOutputPath()); - HdfsUtils.remove(hadoopConf, params.getOutputFaultPath()); - HdfsUtils.remove(hadoopConf, params.getOutputReportPath()); - - LockManager lockManager = LockManagerUtils.instantiateLockManager(params.getLockManagerFactoryClassName(), - hadoopConf); - - FacadeContentRetriever contentRetriever = ServiceFacadeUtils - .instantiate(params.patentServiceFacadeFactoryClassName, params.patentServiceFacadeParams); - - JavaRDD importedPatents = avroLoader.loadJavaRDD(sc, params.inputPath, ImportedPatent.class); - - final Path cacheRootDir = new Path(params.getCacheRootDir()); - CacheMetadataManagingProcess cacheManager = new CacheMetadataManagingProcess(); - - String existingCacheId = cacheManager.getExistingCacheId(hadoopConf, cacheRootDir); - - // skipping already extracted - - //TODO: https://github.com/openaire/iis/issues/1238 - JavaRDD cachedSources = CacheStorageUtils.getRddOrEmpty(sc, avroLoader, cacheRootDir, - existingCacheId, CacheRecordType.data, DocumentText.class) - .filter(x -> StringUtils.isNotBlank(x.getText())); - JavaRDD cachedFaults = CacheStorageUtils.getRddOrEmpty(sc, avroLoader, cacheRootDir, - existingCacheId, CacheRecordType.fault, Fault.class); - - JavaPairRDD> cacheById = cachedSources - .mapToPair(x -> new Tuple2<>(x.getId(), Optional.of(x))) - .union(cachedFaults.mapToPair(x -> new Tuple2<>(x.getInputObjectId(), Optional.empty()))); - JavaPairRDD inputById = importedPatents - .mapToPair(x -> new Tuple2<>(getId(x), x)); - JavaPairRDD>>> inputJoinedWithCache = - inputById.leftOuterJoin(cacheById); - - JavaRDD toBeProcessed = inputJoinedWithCache - .filter(x -> !x._2._2.isPresent()).values().map(x -> x._1); - JavaRDD entitiesReturnedFromCache = inputJoinedWithCache - .filter(x -> x._2._2.isPresent() && x._2._2.get().isPresent()) - .values().map(x -> x._2.get().get()); - - JavaPairRDD> returnedFromRemoteService = - retrieveFromRemoteService(toBeProcessed, contentRetriever); - returnedFromRemoteService.cache(); - - JavaRDD retrievedPatentMetaToBeCached = mapContentRetrieverResponsesToDocumentTextForCache( - returnedFromRemoteService); - JavaRDD faultsToBeCached = mapContentRetrieverResponsesToFaultForCache( - returnedFromRemoteService); - if (!retrievedPatentMetaToBeCached.isEmpty() || !faultsToBeCached.isEmpty()) { - // storing new cache entry - CacheStorageUtils.storeInCache(avroSaver, DocumentText.SCHEMA$, cachedSources.union(retrievedPatentMetaToBeCached), - cachedFaults.union(faultsToBeCached), cacheRootDir, lockManager, cacheManager, hadoopConf, - params.numberOfEmittedFiles); - } - - JavaRDD retrievedPatentMeta = mapContentRetrieverResponsesToDocumentTextForOutput( - returnedFromRemoteService); - JavaRDD faults = mapContentRetrieverResponsesToFaultForOutput( - returnedFromRemoteService); - - JavaRDD entitiesToBeWritten; - if (!retrievedPatentMeta.isEmpty()) { - // merging final results - entitiesToBeWritten = entitiesReturnedFromCache.union(retrievedPatentMeta); - } else { - entitiesToBeWritten = entitiesReturnedFromCache; - } - - // store final results - long entitiesReturnedFromCacheCount = entitiesReturnedFromCache.count(); - long faultsReturnedFromCacheCount = inputJoinedWithCache - .filter(x -> x._2._2.isPresent() && !x._2._2.get().isPresent()) - .count(); - long processedEntitiesCount = retrievedPatentMeta.count(); - long processedFaultsCount = faults.count(); - storeInOutput(entitiesToBeWritten, - //notice: we do not propagate faults from cache, only new faults are written - faults, generateReportEntries(sc, entitiesReturnedFromCacheCount, faultsReturnedFromCacheCount, - processedEntitiesCount, processedFaultsCount), - new OutputPaths(params), params.numberOfEmittedFiles); - } - } - - //------------------------ PRIVATE -------------------------- - - private static JavaPairRDD> retrieveFromRemoteService( - JavaRDD importedPatent, - FacadeContentRetriever contentRetriever) { - return importedPatent - .repartition(1)// limiting number of partitions to 1 in order to run EPO retrieval within a single task - .mapToPair(patent -> new Tuple2<>(getId(patent), contentRetriever.retrieveContent(patent))); - } - - private static JavaRDD mapContentRetrieverResponsesToDocumentTextForCache( - JavaPairRDD> returnedFromRemoteService) { - return returnedFromRemoteService - .filter(e -> FacadeContentRetrieverResponse.isSuccess(e._2)) - .map(e -> DocumentText.newBuilder().setId(e._1).setText(e._2.getContent()).build()); - } - - private static JavaRDD mapContentRetrieverResponsesToFaultForCache( - JavaPairRDD> returnedFromRemoteService) { - return returnedFromRemoteService - .filter(e -> e._2.getClass().equals(FacadeContentRetrieverResponse.PersistentFailure.class)) - .map(e -> FaultUtils.exceptionToFault(e._1, e._2.getException(), null)); - } - - private static JavaRDD mapContentRetrieverResponsesToDocumentTextForOutput( - JavaPairRDD> returnedFromRemoteService) { - return returnedFromRemoteService - .filter(e -> FacadeContentRetrieverResponse.isSuccess(e._2)) - .map(e -> DocumentText.newBuilder().setId(e._1).setText(e._2.getContent()).build()); - } - - private static JavaRDD mapContentRetrieverResponsesToFaultForOutput( - JavaPairRDD> returnedFromRemoteService) { - return returnedFromRemoteService - .filter(e -> FacadeContentRetrieverResponse.isFailure(e._2)) - .map(e -> FaultUtils.exceptionToFault(e._1, e._2.getException(), null)); - } - - private static JavaRDD generateReportEntries(JavaSparkContext sparkContext, - long fromCacheEntitiesCount, - long fromCacheFaultsCount, - long processedEntitiesCount, - long processedFaultsCount) { - ReportEntry fromCacheTotalCounter = ReportEntryFactory.createCounterReportEntry(COUNTER_FROMCACHE_TOTAL, - fromCacheEntitiesCount + fromCacheFaultsCount); - ReportEntry processedTotalCounter = ReportEntryFactory.createCounterReportEntry(COUNTER_PROCESSED_TOTAL, - processedEntitiesCount + processedFaultsCount); - ReportEntry processedFaultsCounter = ReportEntryFactory.createCounterReportEntry(COUNTER_PROCESSED_FAULT, - processedFaultsCount); - - return sparkContext.parallelize(Lists.newArrayList(fromCacheTotalCounter, processedTotalCounter, processedFaultsCounter)); - } - - private static void storeInOutput(JavaRDD retrievedPatentMeta, - JavaRDD faults, JavaRDD reports, OutputPaths outputPaths, int numberOfEmittedFiles) { - avroSaver.saveJavaRDD(retrievedPatentMeta.repartition(numberOfEmittedFiles), DocumentText.SCHEMA$, outputPaths.getResult()); - avroSaver.saveJavaRDD(faults.repartition(numberOfEmittedFiles), Fault.SCHEMA$, outputPaths.getFault()); - avroSaver.saveJavaRDD(reports.repartition(1), ReportEntry.SCHEMA$, outputPaths.getReport()); - } - - private static CharSequence getId(ImportedPatent patent) { - return patent.getApplnNr(); - } - - @Parameters(separators = "=") - private static class JobParameters extends CachedStorageJobParameters { - @Parameter(names = "-inputPath", required = true) - private String inputPath; - - @Parameter(names = "-numberOfEmittedFiles", required = true) - private int numberOfEmittedFiles; - - @Parameter(names = "-patentServiceFacadeFactoryClassName", required = true) - private String patentServiceFacadeFactoryClassName; - - @DynamicParameter(names = "-D", description = "dynamic parameters related to patent service facade", required = false) - private Map patentServiceFacadeParams = Maps.newHashMap(); - } - -} diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentWebServiceFacadeException.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentWebServiceFacadeException.java deleted file mode 100644 index ff162e5b3..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentWebServiceFacadeException.java +++ /dev/null @@ -1,10 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent; - -/** - * Patent web service facade exception. - */ -public class PatentWebServiceFacadeException extends Exception { - public PatentWebServiceFacadeException(String message) { - super(message); - } -} diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/input/PatentMetadataRetrieverInputTransformerJob.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/input/PatentMetadataRetrieverInputTransformerJob.java deleted file mode 100644 index 6c5406869..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/input/PatentMetadataRetrieverInputTransformerJob.java +++ /dev/null @@ -1,72 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent.input; - -import java.io.IOException; - -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaPairRDD; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; - -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -import eu.dnetlib.iis.common.java.io.HdfsUtils; -import eu.dnetlib.iis.common.spark.JavaSparkContextFactory; -import eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import pl.edu.icm.sparkutils.avro.SparkAvroLoader; -import pl.edu.icm.sparkutils.avro.SparkAvroSaver; -import scala.Tuple2; - -/** - * Job responsible for filtering the input to the PatentMetadataRetrieverJob by - * limiting the set of {@link ImportedPatent} records only to the ones matched - * with publications. - * - * @author mhorst - * - */ -public class PatentMetadataRetrieverInputTransformerJob { - - private static final SparkAvroLoader avroLoader = new SparkAvroLoader(); - private static final SparkAvroSaver avroSaver = new SparkAvroSaver(); - - //------------------------ LOGIC -------------------------- - - public static void main(String[] args) throws IOException { - JobParameters params = new JobParameters(); - JCommander jcommander = new JCommander(params); - jcommander.parse(args); - - try (JavaSparkContext sc = JavaSparkContextFactory.withConfAndKryo(new SparkConf())) { - HdfsUtils.remove(sc.hadoopConfiguration(), params.outputPath); - - JavaPairRDD importedPatentsById = avroLoader - .loadJavaRDD(sc, params.inputImportedPatentPath, ImportedPatent.class) - .mapToPair(x -> new Tuple2<>(x.getApplnNr(), x)); - - JavaPairRDD matchedDedupedPatentsById = avroLoader - .loadJavaRDD(sc, params.inputMatchedPatentPath, DocumentToPatent.class).map(DocumentToPatent::getApplnNr) - .distinct().mapToPair(x -> new Tuple2<>(x, true)); - - JavaRDD filteredPatents = matchedDedupedPatentsById.join(importedPatentsById).values().map(x -> x._2); - - avroSaver.saveJavaRDD(filteredPatents, ImportedPatent.SCHEMA$, params.outputPath); - } - } - - //------------------------ PRIVATE -------------------------- - - @Parameters(separators = "=") - private static class JobParameters { - @Parameter(names = "-inputImportedPatentPath", required = true) - private String inputImportedPatentPath; - - @Parameter(names = "-inputMatchedPatentPath", required = true) - private String inputMatchedPatentPath; - - @Parameter(names = "-outputPath", required = true) - private String outputPath; - } -} diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/input/PatentReferenceExtractionInputTransformerJob.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/input/PatentReferenceExtractionInputTransformerJob.java deleted file mode 100644 index c85f8c444..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/input/PatentReferenceExtractionInputTransformerJob.java +++ /dev/null @@ -1,58 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent.input; - -import java.io.IOException; - -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; - -import com.beust.jcommander.JCommander; -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -import eu.dnetlib.iis.common.java.io.HdfsUtils; -import eu.dnetlib.iis.common.spark.JavaSparkContextFactory; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import eu.dnetlib.iis.referenceextraction.patent.schemas.PatentReferenceExtractionInput; -import pl.edu.icm.sparkutils.avro.SparkAvroLoader; -import pl.edu.icm.sparkutils.avro.SparkAvroSaver; - -public class PatentReferenceExtractionInputTransformerJob { - private static final SparkAvroLoader sparkAvroLoader = new SparkAvroLoader(); - private static final SparkAvroSaver sparkAvroSaver = new SparkAvroSaver(); - - //------------------------ LOGIC -------------------------- - - public static void main(String[] args) throws IOException { - JobParameters params = new JobParameters(); - JCommander jcommander = new JCommander(params); - jcommander.parse(args); - - try (JavaSparkContext sc = JavaSparkContextFactory.withConfAndKryo(new SparkConf())) { - HdfsUtils.remove(sc.hadoopConfiguration(), params.outputPath); - - JavaRDD convertedRDD = sparkAvroLoader - .loadJavaRDD(sc, params.inputPath, ImportedPatent.class) - .map(PatentReferenceExtractionInputTransformerJob::convert); - - sparkAvroSaver.saveJavaRDD(convertedRDD, PatentReferenceExtractionInput.SCHEMA$, params.outputPath); - } - } - - //------------------------ PRIVATE -------------------------- - - private static PatentReferenceExtractionInput convert(ImportedPatent patent) { - return PatentReferenceExtractionInput.newBuilder() - .setApplnNr(patent.getApplnNr()) - .build(); - } - - @Parameters(separators = "=") - private static class JobParameters { - @Parameter(names = "-inputPath", required = true) - private String inputPath; - - @Parameter(names = "-outputPath", required = true) - private String outputPath; - } -} diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/parser/OpsPatentMetadataXPathBasedParser.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/parser/OpsPatentMetadataXPathBasedParser.java deleted file mode 100644 index 76e0608bf..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/parser/OpsPatentMetadataXPathBasedParser.java +++ /dev/null @@ -1,270 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent.parser; - -import java.io.IOException; -import java.io.ObjectInputStream; -import java.io.StringReader; -import java.text.MessageFormat; -import java.time.LocalDate; -import java.time.format.DateTimeFormatter; -import java.time.format.DateTimeParseException; -import java.util.List; -import java.util.stream.Collectors; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.xpath.XPath; -import javax.xml.xpath.XPathConstants; -import javax.xml.xpath.XPathFactory; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.log4j.Logger; -import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; - -import com.google.common.collect.Lists; - -import eu.dnetlib.iis.referenceextraction.patent.schemas.Patent; - -/** - * Pull parser processing XML records obtained from OPS EPO public endpoint. - * - * @author mhorst - * - */ -public class OpsPatentMetadataXPathBasedParser implements PatentMetadataParser { - - - private static final long serialVersionUID = 2157218881928411205L; - - - private static final Logger log = Logger.getLogger(OpsPatentMetadataXPathBasedParser.class); - - private static final String XPATH_EXPR_INVENTION_TITLE = "//bibliographic-data/invention-title"; - - private static final String XPATH_EXPR_ABSTRACT = "//abstract"; - - private static final String XPATH_EXPR_CLASS_IPC = "//bibliographic-data/classification-ipc/text"; - - private static final String XPATH_EXPR_TEMPLATE_APLN_REFERENCE_DOC_ID = "//bibliographic-data/application-reference/document-id[@document-id-type=\"{0}\"]/doc-number"; - - private static final String DOCUMENT_ID_TYPE_EPODOC = "epodoc"; - - private static final String XPATH_EXPR_APLN_DATE = "//bibliographic-data/application-reference/document-id/date"; - - private static final String XPATH_EXPR_PUBL_DATE = "//bibliographic-data/publication-reference/document-id/date"; - - private static final String XPATH_EXPR_APPLICANT_NAME = "//bibliographic-data/parties/applicants/applicant[@data-format=\"{0}\"]"; - - private static final String DATA_FORMAT_EPODOC = "epodoc"; - - private static final String ATTRIB_NAME_LANG = "lang"; - - private static final String ATTRIB_VALUE_LANG_EN = "en"; - - private static final String DATE_FORMAT_PATTERN_SOURCE = "yyyyMMdd"; - - private static final DateTimeFormatter DATE_FORMAT_SOURCE = DateTimeFormatter.ofPattern(DATE_FORMAT_PATTERN_SOURCE); - - private static final String DATE_FORMAT_PATTERN_TARGET = "yyyy-MM-dd"; - - private static final DateTimeFormatter DATE_FORMAT_TARGET = DateTimeFormatter.ofPattern(DATE_FORMAT_PATTERN_TARGET); - - private String xPathExpApplnDocIdEpodoc; - - private String xPathExpApplicantEpodocName; - - - // ------------------------------- CONSTRUCTOR ------------------------------- - - public OpsPatentMetadataXPathBasedParser() { - instantiateParser(); - } - - // ------------------------------- PUBLIC ------------------------------------ - - @Override - public Patent.Builder parse(CharSequence source, Patent.Builder patentBuilder) throws PatentMetadataParserException { - try { - XPath xPath = XPathFactory.newInstance().newXPath(); - - DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); - Document xmlDocument = builder.parse(new InputSource(new StringReader(source.toString()))); - - patentBuilder.setApplnTitle(extractTrimmedValueForPreferredEnglishLanguage( - (NodeList) xPath.compile(XPATH_EXPR_INVENTION_TITLE).evaluate(xmlDocument, XPathConstants.NODESET))); - - patentBuilder.setApplnAbstract(extractTrimmedValueForPreferredEnglishLanguage( - (NodeList) xPath.compile(XPATH_EXPR_ABSTRACT).evaluate(xmlDocument, XPathConstants.NODESET))); - - CharSequence epoDoc = extractFirstNonEmptyTrimmedTextContent( - (NodeList) xPath.compile(xPathExpApplnDocIdEpodoc).evaluate(xmlDocument, XPathConstants.NODESET)); - if (StringUtils.isNotBlank(epoDoc)) { - patentBuilder.setApplnNrEpodoc(epoDoc); - } - - patentBuilder.setApplnFilingDate(convertDate(extractEarliestDate( - (NodeList) xPath.compile(XPATH_EXPR_APLN_DATE).evaluate(xmlDocument, XPathConstants.NODESET)))); - - patentBuilder.setEarliestPublnDate(convertDate(extractEarliestDate( - (NodeList) xPath.compile(XPATH_EXPR_PUBL_DATE).evaluate(xmlDocument, XPathConstants.NODESET)))); - - List ipcClasses = extractNonEmptyTrimmedTextContent( - (NodeList) xPath.compile(XPATH_EXPR_CLASS_IPC).evaluate(xmlDocument, XPathConstants.NODESET)); - if (CollectionUtils.isNotEmpty(ipcClasses)) { - patentBuilder.setIpcClassSymbol(ipcClasses); - } - - NodeList epodocNameNodes = (NodeList) xPath.compile(xPathExpApplicantEpodocName).evaluate(xmlDocument, XPathConstants.NODESET); - List applicantEpodocNames = extractNonEmptyTrimmedTextContent(epodocNameNodes); - if (CollectionUtils.isNotEmpty(applicantEpodocNames)) { - patentBuilder.setApplicantNames(cleanNames(applicantEpodocNames)); - patentBuilder.setApplicantCountryCodes(extractCountryCodes(applicantEpodocNames)); - } - - return patentBuilder; - - } catch (Exception e) { - throw new PatentMetadataParserException("error while parsing XML contents: " + source, e); - } - } - - // ------------------------------- PRIVATE ------------------------------------ - - - private static String extractTrimmedValueForPreferredEnglishLanguage(NodeList nodes) { - String otherTitle = null; - for (int i = 0; i < nodes.getLength(); i++) { - Node currentNode = nodes.item(i); - Node langNode = currentNode.getAttributes().getNamedItem(ATTRIB_NAME_LANG); - if (langNode != null && ATTRIB_VALUE_LANG_EN.equals(langNode.getTextContent())) { - return currentNode.getTextContent().trim(); - } else { - otherTitle = currentNode.getTextContent().trim(); - } - } - return otherTitle; - } - - private static List extractNonEmptyTrimmedTextContent(NodeList nodes) { - List results = Lists.newArrayList(); - for (int i = 0; i < nodes.getLength(); i++) { - Node currentNode = nodes.item(i); - String textContent = currentNode.getTextContent(); - if (StringUtils.isNotBlank(textContent)) { - results.add(textContent.trim()); - } - } - return results; - } - - private static CharSequence extractFirstNonEmptyTrimmedTextContent(NodeList nodes) { - for (int i = 0; i < nodes.getLength(); i++) { - Node currentNode = nodes.item(i); - String textContent = currentNode.getTextContent(); - if (StringUtils.isNotBlank(textContent)) { - return textContent.trim(); - } - } - return null; - } - - /** - * Extracts earliest date comparing date strings lexicographically. - */ - private static String extractEarliestDate(NodeList nodes) { - String earliest = null; - for (int i = 0; i < nodes.getLength(); i++) { - Node currentNode = nodes.item(i); - String textContent = currentNode.getTextContent(); - if (StringUtils.isNotBlank(textContent)) { - String trimmedText = textContent.trim(); - if (earliest==null || trimmedText.compareTo(earliest) < 0) { - earliest = trimmedText; - } - } - } - return earliest; - } - - /** - * Converts date whenever specified in expected format, propagates unconverted date otherwise. - */ - private static String convertDate(String source) { - if (StringUtils.isNotBlank(source)) { - try { - LocalDate parsedSource = LocalDate.parse(source, DATE_FORMAT_SOURCE); - if (source.equals(DATE_FORMAT_SOURCE.format(parsedSource))) { - return DATE_FORMAT_TARGET.format(parsedSource); - } else { - return source; - } - } catch (DateTimeParseException e) { - log.warn("propagating source date without conversion: source date '" + source - + "' is not defined in expected format: " + DATE_FORMAT_PATTERN_SOURCE); - return source; - } - } else { - return null; - } - } - - private static List cleanNames(List names) { - if (CollectionUtils.isNotEmpty(names)) { - return names.stream().map(OpsPatentMetadataXPathBasedParser::cleanName) - .collect(Collectors.toList()); - } else { - return names; - } - } - - private static CharSequence cleanName(CharSequence name) { - if (StringUtils.isNotBlank(name)) { - String strName = name.toString().replaceAll("\\u2002", " "); - int idx = strName.indexOf('['); - if (idx > 0) { - return strName.substring(0, idx).trim(); - } else if (strName.charAt(strName.length()-1) == ',') { - return strName.substring(0, name.length()-1).trim(); - } - } - return name; - - } - - private static List extractCountryCodes(List epodocNames) { - if (CollectionUtils.isNotEmpty(epodocNames)) { - return epodocNames.stream().map(OpsPatentMetadataXPathBasedParser::extractCountryCode) - .collect(Collectors.toList()); - } else { - return null; - } - } - - private static CharSequence extractCountryCode(CharSequence epodocName) { - if (StringUtils.isNotBlank(epodocName)) { - String[] countryCodeCandidates = StringUtils.substringsBetween(epodocName.toString(), "[", "]"); - if (ArrayUtils.isNotEmpty(countryCodeCandidates)) { - return countryCodeCandidates[countryCodeCandidates.length -1]; - } - } - return ""; - } - - private void instantiateParser() { - this.xPathExpApplnDocIdEpodoc = MessageFormat.format(XPATH_EXPR_TEMPLATE_APLN_REFERENCE_DOC_ID, DOCUMENT_ID_TYPE_EPODOC); - this.xPathExpApplicantEpodocName = MessageFormat.format(XPATH_EXPR_APPLICANT_NAME, DATA_FORMAT_EPODOC); - } - - /** - * This method is part of deserialization mechanism. - */ - private void readObject(ObjectInputStream inputStream) throws IOException, ClassNotFoundException { - inputStream.defaultReadObject(); - instantiateParser(); - } - -} diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/parser/PatentMetadataParser.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/parser/PatentMetadataParser.java deleted file mode 100644 index aa007d6cd..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/parser/PatentMetadataParser.java +++ /dev/null @@ -1,20 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent.parser; - -import java.io.Serializable; - -import eu.dnetlib.iis.referenceextraction.patent.schemas.Patent; - -/** - * Patent metadata parser. - * - * @author mhorst - * - */ -public interface PatentMetadataParser extends Serializable { - - /** - * Parses XML file and produces {@link Patent.Builder} object by supplementing - * the object provided as parameter. - */ - Patent.Builder parse(CharSequence source, Patent.Builder patentBuilder) throws PatentMetadataParserException; -} diff --git a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/parser/PatentMetadataParserException.java b/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/parser/PatentMetadataParserException.java deleted file mode 100644 index 5f717d7a4..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/main/java/eu/dnetlib/iis/wf/referenceextraction/patent/parser/PatentMetadataParserException.java +++ /dev/null @@ -1,19 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent.parser; - -/** - * @author mhorst - * - */ -public class PatentMetadataParserException extends Exception { - - /** - * - */ - private static final long serialVersionUID = 4924395679997903210L; - - - public PatentMetadataParserException(String message, Throwable cause) { - super(message, cause); - } - -} diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app/import.txt b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app/import.txt index c507c2fba..4c295df9f 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app/import.txt +++ b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app/import.txt @@ -1,3 +1,2 @@ ## This is a classpath-based import file (this header is required) -sqlite_builder classpath eu/dnetlib/iis/wf/referenceextraction/patent/sqlite_builder/oozie_app main_sqlite classpath eu/dnetlib/iis/wf/referenceextraction/patent/main_sqlite/oozie_app \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app/workflow.xml b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app/workflow.xml index 040bb65a9..f70941fd5 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app/workflow.xml @@ -7,17 +7,14 @@ input document text with eu.dnetlib.iis.metadataextraction.schemas.DocumentText records - input_patent - input patent with eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent records + input_patent_db + com.cloudera.spark.lineage.NavigatorAppListener + pre-built sqlite database with patents output_document_to_patent output document to patent with eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent records - - output_patent_metadata - output patent metadata with eu.dnetlib.iis.referenceextraction.patent.schemas.Patent records - output_report_root_path base directory for storing reports @@ -56,85 +53,6 @@ spark2EventLogDir spark 2.* event log dir location - - referenceextraction_patent_number_of_emitted_filesa - 1000 - number of files created by patent metadata retriever module - - - patentMetadataRetrieverFacadeFactoryClassname - eu.dnetlib.iis.wf.referenceextraction.patent.OpenPatentWebServiceFacadeFactory - patent retriever factory class name - - - patentServiceAuthnConsumerKey - $UNDEFINED$ - remote EPO endpoint API authentication consumer key, required by OpenPatentWebServiceFacadeFactory - - - patentServiceAuthnConsumerSecret - $UNDEFINED$ - remote EPO endpoint API authentication consumer secret, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointAuthHost - $UNDEFINED$ - remote EPO endpoint authentication host, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointAuthPort - 443 - remote EPO endpoint authentication port, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointAuthScheme - https - remote EPO endpoint authentication scheme, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointAuthUriRoot - $UNDEFINED$ - remote EPO endpoint authentication URI root, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointOpsHost - $UNDEFINED$ - remote EPO endpoint OPS host, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointOpsPort - 443 - remote EPO endpoint OPS port, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointOpsScheme - https - remote EPO endpoint OPS scheme, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointOpsUriRoot - $UNDEFINED$ - remote EPO endpoint OPS URI root, required by OpenPatentWebServiceFacadeFactory - - - patentServiceEndpointThrottleSleepTime - 10000 - sleep time (in millis) to be applied when receiving 403 error code from EPO endpoint due to exhaustive querying - - - patentServiceEndpointRetriesCount - 20 - number of retries when receiving recoverable error from EPO endpoint (mostly 403) - - - patentLockManagerFactoryClassName - eu.dnetlib.iis.common.lock.ZookeeperLockManagerFactory - lock manager factory class name, to be used for synchronizing access to cache directory - - - cacheRootDir - patents retrieval cache root directory - @@ -156,53 +74,7 @@ - - - - - yarn-cluster - cluster - patent-referenceextraction-input-transformer - eu.dnetlib.iis.wf.referenceextraction.patent.input.PatentReferenceExtractionInputTransformerJob - ${oozieTopWfApplicationPath}/lib/iis-wf-referenceextraction-${projectVersion}.jar - - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - - -inputPath=${input_patent} - -outputPath=${workingDir}/patents_for_matching - - - - - - - - ${wf:appPath()}/sqlite_builder - - - - workingDir - ${workingDir}/sqlite_builder/working_dir - - - input_patent - ${workingDir}/patents_for_matching - - - output_patent_db - ${workingDir}/patents.db - - - - - - + @@ -213,105 +85,8 @@ workingDir ${workingDir}/main_sqlite/working_dir - - input_patent_db - ${workingDir}/patents.db - - - - - - - - yarn-cluster - cluster - patent-metadata-retriever-input-transformer - eu.dnetlib.iis.wf.referenceextraction.patent.input.PatentMetadataRetrieverInputTransformerJob - ${oozieTopWfApplicationPath}/lib/iis-wf-referenceextraction-${projectVersion}.jar - - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - - -inputImportedPatentPath=${input_patent} - -inputMatchedPatentPath=${output_document_to_patent} - -outputPath=${workingDir}/metadata_retriever_input - - - - - - - - yarn-cluster - cluster - patent-metadata-retriever - eu.dnetlib.iis.wf.referenceextraction.patent.PatentMetadataRetrieverJob - ${oozieTopWfApplicationPath}/lib/iis-wf-referenceextraction-${projectVersion}.jar - - --executor-memory=${sparkExecutorMemory} - --executor-cores=1 - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - - -inputPath=${workingDir}/metadata_retriever_input - -numberOfEmittedFiles=${referenceextraction_patent_number_of_emitted_filesa} - -outputPath=${workingDir}/metadata_retriever_output - -outputFaultPath=${workingDir}/metadata_retriever_faults - -outputReportPath=${output_report_root_path}/patent_metadata_retrieval - -lockManagerFactoryClassName=${patentLockManagerFactoryClassName} - -cacheRootDir=${cacheRootDir} - -patentServiceFacadeFactoryClassName=${patentMetadataRetrieverFacadeFactoryClassname} - -DauthnConsumerKey=${patentServiceAuthnConsumerKey} - -DauthnConsumerSecret=${patentServiceAuthnConsumerSecret} - -DendpointAuthHost=${patentServiceEndpointAuthHost} - -DendpointAuthPort=${patentServiceEndpointAuthPort} - -DendpointAuthScheme=${patentServiceEndpointAuthScheme} - -DendpointAuthUriRoot=${patentServiceEndpointAuthUriRoot} - -DendpointOpsHost=${patentServiceEndpointOpsHost} - -DendpointOpsPort=${patentServiceEndpointOpsPort} - -DendpointOpsScheme=${patentServiceEndpointOpsScheme} - -DendpointOpsUriRoot=${patentServiceEndpointOpsUriRoot} - -DendpointThrottleSleepTime=${patentServiceEndpointThrottleSleepTime} - -DendpointRetriesCount=${patentServiceEndpointRetriesCount} - -DendpointReadTimeout=60000 - -DendpointConnectionTimeout=60000 - - - - - - - - yarn-cluster - cluster - patent-metadata-extractor - eu.dnetlib.iis.wf.referenceextraction.patent.PatentMetadataExtractorJob - ${oozieTopWfApplicationPath}/lib/iis-wf-referenceextraction-${projectVersion}.jar - - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - - -inputImportedPatentPath=${input_patent} - -inputDocumentTextPath=${workingDir}/metadata_retriever_output - -outputPath=${output_patent_metadata} - -outputFaultPath=${workingDir}/metadata_extraction_faults - -outputReportPath=${output_report_root_path}/patent_metadata_extraction - diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main_sqlite/oozie_app/lib/scripts/patents.sql b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main_sqlite/oozie_app/lib/scripts/patents.sql index 4019e33e1..6e4656203 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main_sqlite/oozie_app/lib/scripts/patents.sql +++ b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main_sqlite/oozie_app/lib/scripts/patents.sql @@ -8,18 +8,20 @@ stdinput() ); --create temp table results as select * from ( -select jdict('documentId', docid, 'appln_nr', id, 'confidenceLevel', 0.8,'textsnippet',context) from ( -select docid, appln_nr as id, prev||" "||middle||" "||next as context, appln_nr from -(setschema 'docid,prev,middle,next' select c1 as docid, textwindow2s(keywords(c2),25,1,10, "(?:\D|\b)\d{6,12}\b") from (setschema 'c1,c2' select * from pubs)), patents -where regexpr("(?:\D|\b)(\d{6,12})\b",middle) = appln_nr -and (regexprmatches("european patent|patent application|patent office|ep patent|eu patent|patent",lower(context)) or - regexprmatches("\bEPO(?:\d|\b)",context) or regexprmatches ("\bEP\s*"||appln_nr,context) - ) -and (not regexprmatches("holotype|journal pone|journal pntd|paratype|scientometrics|specimen|dissection|\bnih\b|hepth|barcode|\bstrain|accession|\bbacter\b|patent ductus|patent foramen|arxiv|cern|biol|clin|letters|report",lower(context))) -and (not regexprmatches("[0-9]\."||appln_nr,middle) ) - +select jdict('documentId', docid, 'lensId', id, 'confidenceLevel', 0.8,'textsnippet',context) from ( +select docid, patents.c1 as id, patents.c2 as patentcode, prev||" "||middle||" "||next as context from +(setschema 'docid,prev,middle,next' select c1 as docid, textwindow2s(keywords(c2),7,1,3, "(?:\D|\b)[A-Z]?\d{6,}(\b|\D)") from (setschema 'c1,c2' select * from pubs)), patents +where regexpr("(\d{6,})", middle) = normal +and ( ( ( regexprmatches("patent",lower(context)) and ( ( +regexprmatches("\b"||c4||"\b",prev||" "||next) and not (regexprmatches((select jmergeregexp(jgroup("\b"||jurisdiction||"(?:\b|\d+)")) from jurisdictions where jurisdiction != c4),string_split_value(j2s(prev,middle), c4))) ) + or regexprmatches("\b"||c4,middle) ) and regexprmatches(c2, middle) ) + + and (not regexprmatches("holotype|journal pone|journal pntd|paratype|scientometrics|specimen|dissection|\bnih\b|hepth|barcode|\bstrain|accession|\bbacter\b|patent ductus|patent foramen|arxiv|cern|biol|clin|letters|report",lower(context))) + and (not regexprmatches("[0-9]\."||normal,middle) ) + ) + or regexprmatches(lower(c4||c2||c5), lower(middle)) + ) ); --select jdict('documentId', docid, 'appln_nr', id, 'confidenceLevel', 0.8,'textsnippet',context,'author',regexprmatches(authors,context)) from results; - diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main_sqlite/oozie_app/workflow.xml b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main_sqlite/oozie_app/workflow.xml index a1e1bc5cf..193ad059a 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main_sqlite/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main_sqlite/oozie_app/workflow.xml @@ -14,6 +14,16 @@ output_document_to_patent output document to patent + + output_report_root_path + base directory for storing reports + + + output_report_relative_path + document_to_patent + directory for storing report (relative to output_report_root_path) + + @@ -98,10 +108,21 @@ ${input_patent_db}#patent.db - + + + + eu.dnetlib.iis.common.java.ProcessWrapper + eu.dnetlib.iis.common.report.ReportGenerator + -Preport.processing.referenceExtraction.patent.references=${hadoop:counters('main')[RECORDS][MAP_OUT]} + -Oreport=${output_report_root_path}/${output_report_relative_path} + + + + + Unfortunately, the process failed -- error message: [${wf:errorMessage(wf:lastErrorNode())}] diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/sqlite_builder/oozie_app/lib/scripts/base_lens.db b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/sqlite_builder/oozie_app/lib/scripts/base_lens.db new file mode 100644 index 000000000..a8b3a7a73 Binary files /dev/null and b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/sqlite_builder/oozie_app/lib/scripts/base_lens.db differ diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/sqlite_builder/oozie_app/lib/scripts/buildpatentdb.sql b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/sqlite_builder/oozie_app/lib/scripts/buildpatentdb.sql index 85acd78eb..ca663667e 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/sqlite_builder/oozie_app/lib/scripts/buildpatentdb.sql +++ b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/sqlite_builder/oozie_app/lib/scripts/buildpatentdb.sql @@ -2,8 +2,13 @@ drop table if exists patents; create temp table jsoninp as select * from stdinput(); -create table patents as - select c1 as appln_nr from - (setschema 'c1' select jsonpath(c1,"appln_nr") from (select * from jsoninp)); -create index patentsindex on patents(appln_nr); \ No newline at end of file +create table patents as select c1,c2,normal,c4,c5 +from +(setschema 'c1,c2,normal,c4,c5' + select c1 as c1, c2 as c2, normal as normal, c4 as c4, c5 as c5 + from + (select * from (setschema 'c1,c2,normal,c4,c5' select jsonpath(c1, 'c1','c2','normal','c4','c5') from jsoninp))); +-- this is minimalistic input, required by the mining script, we are not interested in the rest of the fields, so we can ignore them + +create index patents_norm on patents(normal, c2, c4, c1); \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/OpenPatentWebServiceFacadeFactoryTest.java b/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/OpenPatentWebServiceFacadeFactoryTest.java deleted file mode 100644 index 2cfb5e38a..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/OpenPatentWebServiceFacadeFactoryTest.java +++ /dev/null @@ -1,208 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent; - -import com.google.common.collect.Maps; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import eu.dnetlib.iis.wf.referenceextraction.FacadeContentRetriever; -import org.junit.jupiter.api.Test; - -import java.nio.charset.StandardCharsets; -import java.util.Base64; -import java.util.Map; - -import static eu.dnetlib.iis.wf.referenceextraction.patent.OpenPatentWebServiceFacadeFactory.*; -import static org.junit.jupiter.api.Assertions.*; - -public class OpenPatentWebServiceFacadeFactoryTest { - - @Test - public void testCreate() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - - // execute - FacadeContentRetriever service = factory.instantiate(conf); - - // assert - assertNotNull(service); - assertTrue(service instanceof OpenPatentWebServiceFacade); - } - - @Test - public void testCreateInvalidConnectionTimeout() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.put(PARAM_SERVICE_ENDPOINT_CONNECTION_TIMEOUT, "non-int"); - - // execute - assertThrows(NumberFormatException.class, () -> factory.instantiate(conf)); - } - - @Test - public void testCreateInvalidReadTimeout() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.put(PARAM_SERVICE_ENDPOINT_READ_TIMEOUT, "non-int"); - - // execute - assertThrows(NumberFormatException.class, () -> factory.instantiate(conf)); - } - - @Test - public void testCreateMissingConsumerKey() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.remove(PARAM_CONSUMER_KEY); - - // execute - assertThrows(IllegalArgumentException.class, () -> factory.instantiate(conf)); - } - - @Test - public void testCreateMissingAuthHost() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.remove(PARAM_SERVICE_ENDPOINT_AUTH_HOST); - - // execute - assertThrows(IllegalArgumentException.class, () -> factory.instantiate(conf)); - } - - @Test - public void testCreateMissingAuthPort() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.remove(PARAM_SERVICE_ENDPOINT_AUTH_PORT); - - // execute - assertThrows(IllegalArgumentException.class, () -> factory.instantiate(conf)); - } - - @Test - public void testCreateInvalidAuthPort() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.put(PARAM_SERVICE_ENDPOINT_AUTH_PORT, "non-int"); - - // execute - assertThrows(NumberFormatException.class, () -> factory.instantiate(conf)); - } - - @Test - public void testCreateMissingAuthScheme() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.remove(PARAM_SERVICE_ENDPOINT_AUTH_SCHEME); - - // execute - assertThrows(IllegalArgumentException.class, () -> factory.instantiate(conf)); - - } - - @Test - public void testCreateMissingAuthUriRoot() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.remove(PARAM_SERVICE_ENDPOINT_AUTH_URI_ROOT); - - // execute - assertThrows(IllegalArgumentException.class, () -> factory.instantiate(conf)); - } - - @Test - public void testCreateMissingOpsHost() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.remove(PARAM_SERVICE_ENDPOINT_OPS_HOST); - - // execute - assertThrows(IllegalArgumentException.class, () -> factory.instantiate(conf)); - } - - @Test - public void testCreateMissingOpsPort() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.remove(PARAM_SERVICE_ENDPOINT_OPS_PORT); - - // execute - assertThrows(IllegalArgumentException.class, () -> factory.instantiate(conf)); - } - - @Test - public void testCreateInvalidOpsPort() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.put(PARAM_SERVICE_ENDPOINT_OPS_PORT, "non-int"); - - // execute - assertThrows(NumberFormatException.class, () -> factory.instantiate(conf)); - } - - @Test - public void testCreateMissingOpsScheme() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.remove(PARAM_SERVICE_ENDPOINT_OPS_SCHEME); - - // execute - assertThrows(IllegalArgumentException.class, () -> factory.instantiate(conf)); - } - - @Test - public void testCreateMissingOpsUriRoot() { - // given - OpenPatentWebServiceFacadeFactory factory = new OpenPatentWebServiceFacadeFactory(); - Map conf = prepareValidConfiguration(); - conf.remove(PARAM_SERVICE_ENDPOINT_OPS_URI_ROOT); - - // execute - assertThrows(IllegalArgumentException.class, () -> factory.instantiate(conf)); - } - - @Test - public void testBuildCredential() { - // given - String key = "someKey"; - String secret = "someSecret"; - - // execute - String credential = OpenPatentWebServiceFacadeFactory.buildCredential(key, secret); - - // assert - assertNotNull(credential); - assertEquals(Base64.getEncoder().encodeToString((key+':'+secret).getBytes(StandardCharsets.UTF_8)), credential); - } - - private Map prepareValidConfiguration() { - Map conf = Maps.newHashMap(); - - conf.put(PARAM_CONSUMER_KEY, "key"); - conf.put(PARAM_CONSUMER_SECRET, "secret"); - - conf.put(PARAM_SERVICE_ENDPOINT_AUTH_HOST, "ops.epo.org"); - conf.put(PARAM_SERVICE_ENDPOINT_AUTH_PORT, "443"); - conf.put(PARAM_SERVICE_ENDPOINT_AUTH_SCHEME, "https"); - conf.put(PARAM_SERVICE_ENDPOINT_AUTH_URI_ROOT, "/3.2/auth/accesstoken"); - - conf.put(PARAM_SERVICE_ENDPOINT_OPS_HOST, "ops.epo.org"); - conf.put(PARAM_SERVICE_ENDPOINT_OPS_PORT, "443"); - conf.put(PARAM_SERVICE_ENDPOINT_OPS_SCHEME, "https"); - conf.put(PARAM_SERVICE_ENDPOINT_OPS_URI_ROOT, "/3.2/rest-services/published-data/publication/docdb"); - - return conf; - } - -} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/OpenPatentWebServiceFacadeTest.java b/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/OpenPatentWebServiceFacadeTest.java deleted file mode 100644 index cc22c7ea8..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/OpenPatentWebServiceFacadeTest.java +++ /dev/null @@ -1,574 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent; - -import com.google.gson.Gson; -import com.google.gson.JsonParser; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import eu.dnetlib.iis.wf.referenceextraction.FacadeContentRetrieverResponse; -import eu.dnetlib.iis.wf.referenceextraction.RetryLimitExceededException; -import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpHost; -import org.apache.http.HttpRequest; -import org.apache.http.StatusLine; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; - -import java.io.*; -import java.nio.charset.StandardCharsets; - -import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.Mockito.*; - -@ExtendWith(MockitoExtension.class) -public class OpenPatentWebServiceFacadeTest { - private final String consumerCredential = "myCredential"; - - private final HttpHost authHost = new HttpHost("someAuthHost", 443, "https"); - private final HttpHost opsHost = new HttpHost("someOpsHost", 443, "https"); - private final String authUriRoot = "/auth_uri"; - private final String opsUriRoot = "ops_uri"; - - @Mock - private CloseableHttpClient httpClient; - - @Test - @DisplayName("Open patent web service facade builds url to query") - public void testGetPatentMetadataUrl() { - // given - ImportedPatent.Builder patentBuilder = initializeWithDummyValues(); - patentBuilder.setPublnAuth("pubAuth"); - patentBuilder.setPublnKind("pubKind"); - patentBuilder.setPublnNr("pubNr"); - OpenPatentWebServiceFacade service = prepareValidService(); - - // execute - String url = service.buildUrl(patentBuilder.build()); - - // assert - assertNotNull(url); - assertEquals(opsUriRoot + "/" + patentBuilder.getPublnAuth() + "." + patentBuilder.getPublnNr() + "." - + patentBuilder.getPublnKind() + "/biblio", url); - } - - @Test - @DisplayName("Open patent web service facade build http request for patent") - public void testBuildPatentMetaRequest() { - // given - String url = "/url/to/patent"; - String bearerToken = "someToken"; - - // execute - HttpRequest httpRequest = OpenPatentWebServiceFacade.buildPatentMetaRequest(url, bearerToken); - - // assert - assertNotNull(httpRequest); - assertEquals(url, httpRequest.getRequestLine().getUri()); - assertEquals("Bearer " + bearerToken, httpRequest.getLastHeader("Authorization").getValue()); - } - - @Test - @DisplayName("Open patent web service facade builds authorization request") - public void testBuildAuthRequest() throws Exception { - // given - String consumerCredential = "someCredential"; - - // execute - HttpRequest httpRequest = OpenPatentWebServiceFacade.buildAuthRequest(consumerCredential, authUriRoot); - - // assert - assertNotNull(httpRequest); - assertEquals(authUriRoot, httpRequest.getRequestLine().getUri()); - assertEquals("Basic " + consumerCredential, httpRequest.getLastHeader("Authorization").getValue()); - assertTrue(httpRequest instanceof HttpPost); - HttpPost postRequest = (HttpPost) httpRequest; - assertNotNull(postRequest.getEntity()); - assertTrue(postRequest.getEntity() instanceof UrlEncodedFormEntity); - - String content = IOUtils.toString(((UrlEncodedFormEntity)postRequest.getEntity()).getContent(), StandardCharsets.UTF_8); - assertNotNull(content); - assertEquals("grant_type=client_credentials", content); - - } - - @Test - @DisplayName("Open patent web service facade reauthentication works correctly") - public void testReauthenticate() throws Exception { - // given - OpenPatentWebServiceFacade service = prepareValidService(); - CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); - StatusLine statusLine = mock(StatusLine.class); - HttpEntity httpEntity = mock(HttpEntity.class); - String accessToken = "someAccessToken"; - - when(httpClient.execute(any(HttpHost.class),any(HttpRequest.class))).thenReturn(httpResponse); - when(httpResponse.getStatusLine()).thenReturn(statusLine); - when(statusLine.getStatusCode()).thenReturn(200); - when(httpResponse.getEntity()).thenReturn(httpEntity); - - Gson gson = new Gson(); - String pageContents = gson.toJson(new AuthenticationResponse(accessToken)); - InputStream pageInputStream = new ByteArrayInputStream(pageContents.getBytes()); - when(httpEntity.getContent()).thenReturn(pageInputStream); - - // execute - service.reauthenticate(); - String token = service.getSecurityToken(); - - // assert - assertNotNull(token); - assertEquals(accessToken, token); - } - - @Test - @DisplayName("Open patent web service facade reauthentication throws exception") - public void testReauthenticateResultingNon200() throws Exception { - // given - OpenPatentWebServiceFacade service = prepareValidService(); - CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); - StatusLine statusLine = mock(StatusLine.class); - HttpEntity httpEntity = mock(HttpEntity.class); - - when(httpClient.execute(any(HttpHost.class),any(HttpRequest.class))).thenReturn(httpResponse); - when(httpResponse.getStatusLine()).thenReturn(statusLine); - when(statusLine.getStatusCode()).thenReturn(404); - when(httpResponse.getEntity()).thenReturn(httpEntity); - - // execute - assertThrows(PatentWebServiceFacadeException.class, service::reauthenticate); - } - - @Test - @DisplayName("Open patent web service facade security token creation works correctly") - public void testGetSecurityToken() throws Exception { - // given - OpenPatentWebServiceFacade service = prepareValidService(); - CloseableHttpResponse httpResponse = mock(CloseableHttpResponse.class); - StatusLine statusLine = mock(StatusLine.class); - HttpEntity httpEntity = mock(HttpEntity.class); - String accessToken = "someAccessToken"; - - when(httpClient.execute(any(HttpHost.class),any(HttpRequest.class))).thenReturn(httpResponse); - when(httpResponse.getStatusLine()).thenReturn(statusLine); - when(statusLine.getStatusCode()).thenReturn(200); - when(httpResponse.getEntity()).thenReturn(httpEntity); - - Gson gson = new Gson(); - String pageContents = gson.toJson(new AuthenticationResponse(accessToken)); - InputStream pageInputStream = new ByteArrayInputStream(pageContents.getBytes()); - when(httpEntity.getContent()).thenReturn(pageInputStream); - - // execute - String token = service.getSecurityToken(); - - // assert - assertNotNull(token); - assertEquals(accessToken, token); - - // execute 2nd time - token = service.getSecurityToken(); - - // assert - assertNotNull(token); - assertEquals(accessToken, token); - // 2nd call should not result in reauthentication, token should be returned straight away - verify(httpClient, times(1)).execute(any(HttpHost.class),any(HttpRequest.class)); - } - - @Nested - public class OpenPatentWebServiceFacadeWithHTTP200FamilyTest { - - @Test - @DisplayName("Open patent web service facade retrieves content successfully for HTTP 200 server reply and valid entity") - public void testGetPatentMetadataForHttp200() throws Exception { - // given - ImportedPatent.Builder patentBuilder = initializeWithDummyValues(); - String expectedResult = "this is expected result"; - OpenPatentWebServiceFacade service = prepareValidService(); - - // authentication mock - CloseableHttpResponse authnHttpResponse = mock(CloseableHttpResponse.class); - StatusLine authnStatusLine = mock(StatusLine.class); - HttpEntity authnHttpEntity = mock(HttpEntity.class); - String accessToken = "someAccessToken"; - - when(authnHttpResponse.getStatusLine()).thenReturn(authnStatusLine); - when(authnStatusLine.getStatusCode()).thenReturn(200); - when(authnHttpResponse.getEntity()).thenReturn(authnHttpEntity); - - Gson gson = new Gson(); - String pageContents = gson.toJson(new AuthenticationResponse(accessToken)); - InputStream pageInputStream = new ByteArrayInputStream(pageContents.getBytes()); - when(authnHttpEntity.getContent()).thenReturn(pageInputStream); - - // metadata retrieval mock - CloseableHttpResponse getPatentHttpResponse = mock(CloseableHttpResponse.class); - StatusLine getPatentStatusLine = mock(StatusLine.class); - HttpEntity getPatentHttpEntity = mock(HttpEntity.class); - when(getPatentHttpResponse.getStatusLine()).thenReturn(getPatentStatusLine); - when(getPatentStatusLine.getStatusCode()).thenReturn(200); - when(getPatentHttpResponse.getEntity()).thenReturn(getPatentHttpEntity); - when(getPatentHttpEntity.getContent()).thenReturn(new ByteArrayInputStream(expectedResult.getBytes())); - - when(httpClient.execute(any(HttpHost.class),any(HttpRequest.class))).thenReturn(authnHttpResponse, getPatentHttpResponse); - - // execute - FacadeContentRetrieverResponse response = service.retrieveContent(patentBuilder.build()); - - // assert - assertNotNull(response); - assertEquals(FacadeContentRetrieverResponse.Success.class, response.getClass()); - assertEquals(expectedResult, response.getContent()); - } - - @Test - @DisplayName("Open patent web service facade returns persistent failure for HTTP 200 server reply and null entity") - public void testGetPatentMetadataForHttp200WithNullEntity() throws Exception { - // given - ImportedPatent.Builder patentBuilder = initializeWithDummyValues(); - String expectedResult = "this is expected result"; - OpenPatentWebServiceFacade service = prepareValidService(); - - // authentication mock - CloseableHttpResponse authnHttpResponse = mock(CloseableHttpResponse.class); - StatusLine authnStatusLine = mock(StatusLine.class); - HttpEntity authnHttpEntity = mock(HttpEntity.class); - String accessToken = "someAccessToken"; - - when(authnHttpResponse.getStatusLine()).thenReturn(authnStatusLine); - when(authnStatusLine.getStatusCode()).thenReturn(200); - when(authnHttpResponse.getEntity()).thenReturn(authnHttpEntity); - - Gson gson = new Gson(); - String pageContents = gson.toJson(new AuthenticationResponse(accessToken)); - InputStream pageInputStream = new ByteArrayInputStream(pageContents.getBytes()); - when(authnHttpEntity.getContent()).thenReturn(pageInputStream); - - // metadata retrieval mock - CloseableHttpResponse getPatentHttpResponse = mock(CloseableHttpResponse.class); - StatusLine getPatentStatusLine = mock(StatusLine.class); - when(getPatentHttpResponse.getStatusLine()).thenReturn(getPatentStatusLine); - when(getPatentStatusLine.getStatusCode()).thenReturn(200); - when(getPatentHttpResponse.getEntity()).thenReturn(null); - - when(httpClient.execute(any(HttpHost.class),any(HttpRequest.class))).thenReturn(authnHttpResponse, getPatentHttpResponse); - - // execute - FacadeContentRetrieverResponse response = service.retrieveContent(patentBuilder.build()); - - // assert - assertNotNull(response); - assertEquals(FacadeContentRetrieverResponse.PersistentFailure.class, response.getClass()); - assertEquals(PatentWebServiceFacadeException.class, response.getException().getClass()); - } - } - - @Nested - public class OpenPatentWebServiceFacadeWithHTTP400FamilyTest { - - @Test - @DisplayName("Open patent web service facade retrieves content successfully for HTTP 400 followed by HTTP 200 server replies and valid entity") - public void testGetPatentMetadataForHttp400() throws Exception { - // given - ImportedPatent.Builder patentBuilder = initializeWithDummyValues(); - String expectedResult = "this is expected result"; - OpenPatentWebServiceFacade service = prepareValidService(); - - // authentication mock - String accessToken = "someAccessToken"; - Gson gson = new Gson(); - String pageContents = gson.toJson(new AuthenticationResponse(accessToken)); - - CloseableHttpResponse authnHttpResponse1 = mock(CloseableHttpResponse.class); - StatusLine authnStatusLine = mock(StatusLine.class); - HttpEntity authnHttpEntity1 = mock(HttpEntity.class); - - when(authnHttpResponse1.getStatusLine()).thenReturn(authnStatusLine); - when(authnStatusLine.getStatusCode()).thenReturn(200); - when(authnHttpResponse1.getEntity()).thenReturn(authnHttpEntity1); - when(authnHttpEntity1.getContent()).thenReturn(new ByteArrayInputStream(pageContents.getBytes())); - - CloseableHttpResponse authnHttpResponse2 = mock(CloseableHttpResponse.class); - HttpEntity authnHttpEntity2 = mock(HttpEntity.class); - - when(authnHttpResponse2.getStatusLine()).thenReturn(authnStatusLine); - when(authnHttpResponse2.getEntity()).thenReturn(authnHttpEntity2); - when(authnHttpEntity2.getContent()).thenReturn(new ByteArrayInputStream(pageContents.getBytes())); - - // metadata retrieval mock - CloseableHttpResponse getPatentHttpResponse1 = mock(CloseableHttpResponse.class); - StatusLine getPatentStatusLine1 = mock(StatusLine.class); - when(getPatentHttpResponse1.getStatusLine()).thenReturn(getPatentStatusLine1); - when(getPatentStatusLine1.getStatusCode()).thenReturn(400); - - CloseableHttpResponse getPatentHttpResponse2 = mock(CloseableHttpResponse.class); - StatusLine getPatentStatusLine2 = mock(StatusLine.class); - HttpEntity getPatentHttpEntity2 = mock(HttpEntity.class); - when(getPatentHttpResponse2.getStatusLine()).thenReturn(getPatentStatusLine2); - when(getPatentStatusLine2.getStatusCode()).thenReturn(200); - when(getPatentHttpResponse2.getEntity()).thenReturn(getPatentHttpEntity2); - when(getPatentHttpEntity2.getContent()).thenReturn(new ByteArrayInputStream(expectedResult.getBytes())); - - when(httpClient.execute(any(HttpHost.class), any(HttpRequest.class))).thenReturn(authnHttpResponse1, - getPatentHttpResponse1, authnHttpResponse2, getPatentHttpResponse2); - - // execute - FacadeContentRetrieverResponse response = service.retrieveContent(patentBuilder.build()); - - // assert - assertNotNull(response); - assertEquals(FacadeContentRetrieverResponse.Success.class, response.getClass()); - assertEquals(expectedResult, response.getContent()); - } - - @Test - @DisplayName("Open patent web service facade retrieves content successfully for HTTP 403 followed by HTTP 200 server replies and valid entity") - public void testGetPatentMetadataForHttp403() throws Exception { - // given - ImportedPatent.Builder patentBuilder = initializeWithDummyValues(); - String expectedResult = "this is expected result"; - OpenPatentWebServiceFacade service = prepareValidService(); - - // authentication mock - String accessToken = "someAccessToken"; - Gson gson = new Gson(); - String pageContents = gson.toJson(new AuthenticationResponse(accessToken)); - - CloseableHttpResponse authnHttpResponse = mock(CloseableHttpResponse.class); - StatusLine authnStatusLine = mock(StatusLine.class); - HttpEntity authnHttpEntity = mock(HttpEntity.class); - - when(authnHttpResponse.getStatusLine()).thenReturn(authnStatusLine); - when(authnStatusLine.getStatusCode()).thenReturn(200); - when(authnHttpResponse.getEntity()).thenReturn(authnHttpEntity); - when(authnHttpEntity.getContent()).thenReturn(new ByteArrayInputStream(pageContents.getBytes())); - - // metadata retrieval mock - CloseableHttpResponse getPatentHttpResponse1 = mock(CloseableHttpResponse.class); - StatusLine getPatentStatusLine1 = mock(StatusLine.class); - when(getPatentHttpResponse1.getStatusLine()).thenReturn(getPatentStatusLine1); - when(getPatentStatusLine1.getStatusCode()).thenReturn(403); - - CloseableHttpResponse getPatentHttpResponse2 = mock(CloseableHttpResponse.class); - StatusLine getPatentStatusLine2 = mock(StatusLine.class); - HttpEntity getPatentHttpEntity2 = mock(HttpEntity.class); - when(getPatentHttpResponse2.getStatusLine()).thenReturn(getPatentStatusLine2); - when(getPatentStatusLine2.getStatusCode()).thenReturn(200); - when(getPatentHttpResponse2.getEntity()).thenReturn(getPatentHttpEntity2); - when(getPatentHttpEntity2.getContent()).thenReturn(new ByteArrayInputStream(expectedResult.getBytes())); - - when(httpClient.execute(any(HttpHost.class), any(HttpRequest.class))).thenReturn(authnHttpResponse, - getPatentHttpResponse1, getPatentHttpResponse2); - - // execute - FacadeContentRetrieverResponse response = service.retrieveContent(patentBuilder.build()); - - // assert - assertNotNull(response); - assertEquals(FacadeContentRetrieverResponse.Success.class, response.getClass()); - assertEquals(expectedResult, response.getContent()); - } - - @Test - @DisplayName("Open patent web service facade returns transient failure when max retry limit is exceeded") - public void testGetPatentMetadataForHttp403RetryCountExceeded() throws Exception { - // given - ImportedPatent.Builder patentBuilder = initializeWithDummyValues(); - OpenPatentWebServiceFacade service = prepareValidService(); - - // authentication mock - String accessToken = "someAccessToken"; - Gson gson = new Gson(); - String pageContents = gson.toJson(new AuthenticationResponse(accessToken)); - - CloseableHttpResponse authnHttpResponse = mock(CloseableHttpResponse.class); - StatusLine authnStatusLine = mock(StatusLine.class); - HttpEntity authnHttpEntity = mock(HttpEntity.class); - - when(authnHttpResponse.getStatusLine()).thenReturn(authnStatusLine); - when(authnStatusLine.getStatusCode()).thenReturn(200); - when(authnHttpResponse.getEntity()).thenReturn(authnHttpEntity); - when(authnHttpEntity.getContent()).thenReturn(new ByteArrayInputStream(pageContents.getBytes())); - - // metadata retrieval mock - CloseableHttpResponse getPatentHttpResponse = mock(CloseableHttpResponse.class); - StatusLine getPatentStatusLine = mock(StatusLine.class); - when(getPatentHttpResponse.getStatusLine()).thenReturn(getPatentStatusLine); - when(getPatentStatusLine.getStatusCode()).thenReturn(403); - - when(httpClient.execute(any(HttpHost.class), any(HttpRequest.class))).thenReturn(authnHttpResponse, - getPatentHttpResponse); - - // execute - FacadeContentRetrieverResponse response = service.retrieveContent(patentBuilder.build()); - - // assert - assertNotNull(response); - assertEquals(FacadeContentRetrieverResponse.TransientFailure.class, response.getClass()); - assertEquals(RetryLimitExceededException.class, response.getException().getClass()); - } - - @Test - @DisplayName("Open patent web service facade returns persistent failure for HTTP 404 server reply") - public void testGetPatentMetadataForHttp404() throws Exception { - // given - ImportedPatent.Builder patentBuilder = initializeWithDummyValues(); - OpenPatentWebServiceFacade service = prepareValidService(); - - // authentication mock - CloseableHttpResponse authnHttpResponse = mock(CloseableHttpResponse.class); - StatusLine authnStatusLine = mock(StatusLine.class); - HttpEntity authnHttpEntity = mock(HttpEntity.class); - String accessToken = "someAccessToken"; - - when(authnHttpResponse.getStatusLine()).thenReturn(authnStatusLine); - when(authnStatusLine.getStatusCode()).thenReturn(200); - when(authnHttpResponse.getEntity()).thenReturn(authnHttpEntity); - - Gson gson = new Gson(); - String pageContents = gson.toJson(new AuthenticationResponse(accessToken)); - InputStream pageInputStream = new ByteArrayInputStream(pageContents.getBytes()); - when(authnHttpEntity.getContent()).thenReturn(pageInputStream); - - // metadata retrieval mock - CloseableHttpResponse getPatentHttpResponse = mock(CloseableHttpResponse.class); - StatusLine getPatentStatusLine = mock(StatusLine.class); - when(getPatentHttpResponse.getStatusLine()).thenReturn(getPatentStatusLine); - when(getPatentStatusLine.getStatusCode()).thenReturn(404); - - when(httpClient.execute(any(HttpHost.class),any(HttpRequest.class))).thenReturn(authnHttpResponse, getPatentHttpResponse); - - // execute - FacadeContentRetrieverResponse response = service.retrieveContent(patentBuilder.build()); - - // assert - assertNotNull(response); - assertEquals(FacadeContentRetrieverResponse.PersistentFailure.class, response.getClass()); - assertEquals(PatentWebServiceFacadeException.class, response.getException().getClass()); - } - } - - @Nested - public class OpenPatentWebServiceFacadeWithHTTP500FamilyTest { - - @Test - @DisplayName("Open patent web service facade returns persistent failure for HTTP 500 server reply") - public void testGetPatentMetadataForHttp500() throws Exception { - // given - ImportedPatent.Builder patentBuilder = initializeWithDummyValues(); - OpenPatentWebServiceFacade service = prepareValidService(); - - // authentication mock - CloseableHttpResponse authnHttpResponse = mock(CloseableHttpResponse.class); - StatusLine authnStatusLine = mock(StatusLine.class); - HttpEntity authnHttpEntity = mock(HttpEntity.class); - String accessToken = "someAccessToken"; - - when(authnHttpResponse.getStatusLine()).thenReturn(authnStatusLine); - when(authnStatusLine.getStatusCode()).thenReturn(200); - when(authnHttpResponse.getEntity()).thenReturn(authnHttpEntity); - - Gson gson = new Gson(); - String pageContents = gson.toJson(new AuthenticationResponse(accessToken)); - InputStream pageInputStream = new ByteArrayInputStream(pageContents.getBytes()); - when(authnHttpEntity.getContent()).thenReturn(pageInputStream); - - // metadata retrieval mock - CloseableHttpResponse getPatentHttpResponse = mock(CloseableHttpResponse.class); - StatusLine getPatentStatusLine = mock(StatusLine.class); - when(getPatentHttpResponse.getStatusLine()).thenReturn(getPatentStatusLine); - when(getPatentStatusLine.getStatusCode()).thenReturn(500); - - when(httpClient.execute(any(HttpHost.class),any(HttpRequest.class))).thenReturn(authnHttpResponse, getPatentHttpResponse); - - // execute - FacadeContentRetrieverResponse response = service.retrieveContent(patentBuilder.build()); - - // assert - assertNotNull(response); - assertEquals(FacadeContentRetrieverResponse.PersistentFailure.class, response.getClass()); - assertEquals(PatentWebServiceFacadeException.class, response.getException().getClass()); - } - } - - @Test - @DisplayName("Open patent web service facade returns transient failure when content retrieval throws an exception") - public void testGetPatentMetadataWithAnException() { - // given - OpenPatentWebServiceFacade service = new OpenPatentWebServiceFacade(httpClient, authHost, authUriRoot, opsHost, - opsUriRoot, consumerCredential, 1, 1, new JsonParser()) { - @Override - protected FacadeContentRetrieverResponse retrieveContentOrThrow(String url, int retryCount) { - throw new RuntimeException("failed"); - } - }; - - // execute - FacadeContentRetrieverResponse response = service.retrieveContent(mock(ImportedPatent.class)); - - // assert - assertNotNull(response); - assertEquals(FacadeContentRetrieverResponse.TransientFailure.class, response.getClass()); - assertEquals(RuntimeException.class, response.getException().getClass()); - } - - @Test - @DisplayName("Open patent web service facade is serializable") - public void testSerializeAndDeserialize() throws Exception { - // given - OpenPatentWebServiceFacade service = new OpenPatentWebServiceFacade(ConnectionDetailsBuilder.newBuilder() - .withConnectionTimeout(10000).withReadTimeout(10000).withAuthHostName("authn-host").withAuthPort(8080) - .withAuthScheme("https").withAuthUriRoot(authUriRoot).withOpsHostName("ops-host").withOpsPort(8090) - .withOpsScheme("http").withOpsUriRoot(opsUriRoot).withConsumerCredential(consumerCredential) - .withThrottleSleepTime(60000).withMaxRetriesCount(1).build()); - - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - ObjectOutputStream oos = new ObjectOutputStream(bos); - - // execute - oos.writeObject(service); - oos.flush(); - oos.close(); - ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(bos.toByteArray())); - OpenPatentWebServiceFacade deserService = (OpenPatentWebServiceFacade) ois.readObject(); - ois.close(); - - // assert - assertNotNull(deserService); - } - - private static ImportedPatent.Builder initializeWithDummyValues() { - ImportedPatent.Builder patentBuilder = ImportedPatent.newBuilder(); - String irrelevant = "irrelevant"; - patentBuilder.setApplnAuth(irrelevant); - patentBuilder.setApplnNr(irrelevant); - patentBuilder.setPublnAuth(irrelevant); - patentBuilder.setPublnKind(irrelevant); - patentBuilder.setPublnNr(irrelevant); - return patentBuilder; - } - - private static class AuthenticationResponse { - - @SuppressWarnings("unused") - private String access_token; - - public AuthenticationResponse(String access_token) { - this.access_token = access_token; - } - - } - - private OpenPatentWebServiceFacade prepareValidService() { - return new OpenPatentWebServiceFacade(httpClient, authHost, authUriRoot, opsHost, - opsUriRoot, consumerCredential, 1, 1, new JsonParser()); - } - -} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentMetadataExtractorJobTest.java b/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentMetadataExtractorJobTest.java deleted file mode 100644 index a1e0e1160..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentMetadataExtractorJobTest.java +++ /dev/null @@ -1,249 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent; - -import com.google.common.collect.Lists; -import eu.dnetlib.iis.audit.schemas.Fault; -import eu.dnetlib.iis.common.ClassPathResourceProvider; -import eu.dnetlib.iis.common.SlowTest; -import eu.dnetlib.iis.common.java.io.DataStore; -import eu.dnetlib.iis.common.java.io.HdfsTestUtils; -import eu.dnetlib.iis.common.schemas.ReportEntry; -import eu.dnetlib.iis.common.utils.AvroTestUtils; -import eu.dnetlib.iis.metadataextraction.schemas.DocumentText; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import eu.dnetlib.iis.referenceextraction.patent.schemas.Patent; -import eu.dnetlib.iis.wf.referenceextraction.patent.parser.PatentMetadataParserException; -import org.apache.hadoop.conf.Configuration; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import pl.edu.icm.sparkutils.test.SparkJob; -import pl.edu.icm.sparkutils.test.SparkJobBuilder; -import pl.edu.icm.sparkutils.test.SparkJobExecutor; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; - -/** - * {@link PatentMetadataExtractorJob} test class. - * - */ -@SlowTest -public class PatentMetadataExtractorJobTest { - - static final String xmlResourcesRootClassPath = "/eu/dnetlib/iis/wf/referenceextraction/patent/data/"; - - private SparkJobExecutor executor = new SparkJobExecutor(); - - @TempDir - public Path workingDir; - - private Path inputImportedPatentDir; - private Path inputDocumentTextDir; - private Path outputDir; - private Path outputFaultDir; - private Path outputReportDir; - - @BeforeEach - public void before() { - inputImportedPatentDir = workingDir.resolve("input_imported_patent"); - inputDocumentTextDir = workingDir.resolve("input_document_text"); - outputDir = workingDir.resolve("output"); - outputFaultDir = workingDir.resolve("fault"); - outputReportDir = workingDir.resolve("report"); - } - - @Test - public void testExtractMetadata() throws IOException { - // given - String matchedPatentId = "1234"; - List importedPatent = Lists.newArrayList( - buildImportedPatent("XX", matchedPatentId), - buildImportedPatent("YY", "5678")); - List documentText = Lists.newArrayList( - buildDocumentText(matchedPatentId, xmlResourcesRootClassPath + "WO.0042078.A1.xml")); - - AvroTestUtils.createLocalAvroDataStore(importedPatent, inputImportedPatentDir.toString()); - AvroTestUtils.createLocalAvroDataStore(documentText, inputDocumentTextDir.toString()); - - SparkJob sparkJob = buildSparkJob(); - - // when - executor.execute(sparkJob); - - // then - List calculatedResult = AvroTestUtils.readLocalAvroDataStore(outputDir.toString()); - assertNotNull(calculatedResult); - assertEquals(1, calculatedResult.size()); - Patent parsedPatent = calculatedResult.get(0); - assertNotNull(parsedPatent); - assertEquals(matchedPatentId, parsedPatent.getApplnNr().toString()); - assertEquals("XX", parsedPatent.getApplnAuth().toString()); - assertEquals("WO2000EP00003", parsedPatent.getApplnNrEpodoc().toString()); - - List generatedFaults = AvroTestUtils.readLocalAvroDataStore(outputFaultDir.toString()); - assertNotNull(generatedFaults); - assertEquals(0, generatedFaults.size()); - - assertEquals(1, - HdfsTestUtils.countFiles(new Configuration(), outputReportDir.toString(), DataStore.AVRO_FILE_EXT)); - assertReports(AvroTestUtils.readLocalAvroDataStore(outputReportDir.toString()), 1, 0); - } - - @Test - public void testExtractMetadataNoCountryCode() throws IOException { - // given - String matchedPatentId = "1234"; - List importedPatent = Lists.newArrayList( - buildImportedPatent("XX", matchedPatentId), - buildImportedPatent("YY", "5678")); - List documentText = Lists.newArrayList( - buildDocumentText(matchedPatentId, xmlResourcesRootClassPath + "WO.0042078.A1.no_country_code.xml")); - - AvroTestUtils.createLocalAvroDataStore(importedPatent, inputImportedPatentDir.toString()); - AvroTestUtils.createLocalAvroDataStore(documentText, inputDocumentTextDir.toString()); - - SparkJob sparkJob = buildSparkJob(); - - // when - executor.execute(sparkJob); - - // then - List calculatedResult = AvroTestUtils.readLocalAvroDataStore(outputDir.toString()); - assertNotNull(calculatedResult); - assertEquals(1, calculatedResult.size()); - Patent parsedPatent = calculatedResult.get(0); - assertNotNull(parsedPatent); - assertEquals(matchedPatentId, parsedPatent.getApplnNr().toString()); - assertEquals("XX", parsedPatent.getApplnAuth().toString()); - assertEquals("WO2000EP00003", parsedPatent.getApplnNrEpodoc().toString()); - - List generatedFaults = AvroTestUtils.readLocalAvroDataStore(outputFaultDir.toString()); - assertNotNull(generatedFaults); - assertEquals(0, generatedFaults.size()); - - assertEquals(1, - HdfsTestUtils.countFiles(new Configuration(), outputReportDir.toString(), DataStore.AVRO_FILE_EXT)); - assertReports(AvroTestUtils.readLocalAvroDataStore(outputReportDir.toString()), 1, 0); - } - - @Test - public void testExtractMetadataFromInvalidXmlFile() throws IOException { - // given - String patentId = "1234"; - List importedPatent = Lists.newArrayList( - buildImportedPatent("XX", patentId)); - DocumentText.Builder documentTextBuilder = DocumentText.newBuilder(); - documentTextBuilder.setId(patentId); - documentTextBuilder.setText(""); - List documentText = Lists.newArrayList( - documentTextBuilder.build()); - - AvroTestUtils.createLocalAvroDataStore(importedPatent, inputImportedPatentDir.toString()); - AvroTestUtils.createLocalAvroDataStore(documentText, inputDocumentTextDir.toString()); - - SparkJob sparkJob = buildSparkJob(); - - // when - executor.execute(sparkJob); - - // then - List calculatedResult = AvroTestUtils.readLocalAvroDataStore(outputDir.toString()); - assertNotNull(calculatedResult); - assertEquals(0, calculatedResult.size()); - - List generatedFaults = AvroTestUtils.readLocalAvroDataStore(outputFaultDir.toString()); - assertNotNull(generatedFaults); - assertEquals(1, generatedFaults.size()); - Fault fault = generatedFaults.get(0); - assertEquals(patentId, fault.getInputObjectId().toString()); - assertEquals(PatentMetadataParserException.class.getCanonicalName(), fault.getCode().toString()); - - assertEquals(1, - HdfsTestUtils.countFiles(new Configuration(), outputReportDir.toString(), DataStore.AVRO_FILE_EXT)); - assertReports(AvroTestUtils.readLocalAvroDataStore(outputReportDir.toString()), 1, 1); - } - - @Test - public void testExtractMetadataForNotMatchableDocumentTextDatastore() throws IOException { - // given - List importedPatent = Lists.newArrayList( - buildImportedPatent("XX", "1234")); - List documentText = Lists.newArrayList( - buildDocumentText("5678", xmlResourcesRootClassPath + "WO.0042078.A1.xml")); - - AvroTestUtils.createLocalAvroDataStore(importedPatent, inputImportedPatentDir.toString()); - AvroTestUtils.createLocalAvroDataStore(documentText, inputDocumentTextDir.toString()); - - SparkJob sparkJob = buildSparkJob(); - - // when - executor.execute(sparkJob); - - // then - List calculatedResult = AvroTestUtils.readLocalAvroDataStore(outputDir.toString()); - assertNotNull(calculatedResult); - assertEquals(0, calculatedResult.size()); - - List generatedFaults = AvroTestUtils.readLocalAvroDataStore(outputFaultDir.toString()); - assertNotNull(generatedFaults); - assertEquals(0, generatedFaults.size()); - - assertEquals(1, - HdfsTestUtils.countFiles(new Configuration(), outputReportDir.toString(), DataStore.AVRO_FILE_EXT)); - assertReports(AvroTestUtils.readLocalAvroDataStore(outputReportDir.toString()), 0, 0); - } - - private void assertReports(List reports, int processedTotal, int processedFault) { - assertNotNull(reports); - assertEquals(2, reports.size()); - - Map reportMap = reports.stream() - .collect(Collectors.toMap(x -> x.getKey().toString(), x -> x.getValue().toString())); - - String counterValue = reportMap.get(PatentMetadataExtractorJob.COUNTER_PROCESSED_TOTAL); - assertNotNull(counterValue); - assertEquals(processedTotal, Integer.parseInt(counterValue)); - - counterValue = reportMap.get(PatentMetadataExtractorJob.COUNTER_PROCESSED_FAULT); - assertNotNull(counterValue); - assertEquals(processedFault, Integer.parseInt(counterValue)); - } - - private ImportedPatent buildImportedPatent(String applnAuth, String applnNr) { - ImportedPatent.Builder importedPatentBuilder = ImportedPatent.newBuilder(); - importedPatentBuilder.setApplnAuth(applnAuth); - importedPatentBuilder.setApplnNr(applnNr); - importedPatentBuilder.setPublnAuth("irrelevant"); - importedPatentBuilder.setPublnNr("irrelevant"); - importedPatentBuilder.setPublnKind("irrelevant"); - return importedPatentBuilder.build(); - } - - private DocumentText buildDocumentText(String id, String textClassPathLocation) { - DocumentText.Builder documentTextBuilder = DocumentText.newBuilder(); - documentTextBuilder.setId(id); - String textContent = ClassPathResourceProvider.getResourceContent(textClassPathLocation); - documentTextBuilder.setText(textContent); - return documentTextBuilder.build(); - } - - private SparkJob buildSparkJob() { - return SparkJobBuilder.create() - .setAppName(getClass().getName()) - .setMainClass(PatentMetadataExtractorJob.class) - .addArg("-inputImportedPatentPath", inputImportedPatentDir.toString()) - .addArg("-inputDocumentTextPath", inputDocumentTextDir.toString()) - .addArg("-outputPath", outputDir.toString()) - .addArg("-outputFaultPath", outputFaultDir.toString()) - .addArg("-outputReportPath", outputReportDir.toString()) - .addJobProperty("spark.driver.host", "localhost") - .build(); - } - -} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentMetadataRetrieverJobTest.java b/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentMetadataRetrieverJobTest.java deleted file mode 100644 index 9fa7b8575..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/PatentMetadataRetrieverJobTest.java +++ /dev/null @@ -1,349 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent; - -import eu.dnetlib.iis.audit.schemas.Fault; -import eu.dnetlib.iis.common.ClassPathResourceProvider; -import eu.dnetlib.iis.common.SlowTest; -import eu.dnetlib.iis.common.cache.CacheMetadataManagingProcess; -import eu.dnetlib.iis.common.cache.CacheStorageUtils; -import eu.dnetlib.iis.common.cache.CacheStorageUtils.CacheRecordType; -import eu.dnetlib.iis.common.java.io.DataStore; -import eu.dnetlib.iis.common.java.io.HdfsTestUtils; -import eu.dnetlib.iis.common.lock.ZookeeperLockManagerFactory; -import eu.dnetlib.iis.common.schemas.ReportEntry; -import eu.dnetlib.iis.common.utils.AvroAssertTestUtil; -import eu.dnetlib.iis.common.utils.AvroTestUtils; -import eu.dnetlib.iis.common.utils.JsonAvroTestUtils; -import eu.dnetlib.iis.metadataextraction.schemas.DocumentText; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import org.apache.curator.test.TestingServer; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ha.ZKFailoverController; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import pl.edu.icm.sparkutils.test.SparkJob; -import pl.edu.icm.sparkutils.test.SparkJobBuilder; -import pl.edu.icm.sparkutils.test.SparkJobExecutor; - -import java.io.IOException; -import java.nio.file.Path; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -@SlowTest -public class PatentMetadataRetrieverJobTest { - - private SparkJobExecutor executor = new SparkJobExecutor(); - - @TempDir - public Path workingDir; - - private Path inputDir; - private Path input2Dir; - private Path outputDir; - private Path output2Dir; - private Path outputFaultDir; - private Path outputFault2Dir; - private Path outputReportDir; - private Path outputReport2Dir; - private Path cacheRootDir; - - private static TestingServer zookeeperServer; - - @BeforeAll - public static void beforeAll() throws Exception { - zookeeperServer = new TestingServer(true); - } - - @BeforeEach - public void beforeEach() { - inputDir = workingDir.resolve("input"); - input2Dir = workingDir.resolve("input2"); - outputDir = workingDir.resolve("output"); - output2Dir = workingDir.resolve("output2"); - outputFaultDir = workingDir.resolve("fault"); - outputFault2Dir = workingDir.resolve("fault2"); - outputReportDir = workingDir.resolve("report"); - outputReport2Dir = workingDir.resolve("report2"); - cacheRootDir = workingDir.resolve("cache"); - } - - @AfterAll - public static void after() throws IOException { - zookeeperServer.close(); - } - - @Test - public void testRetrievePatentMetaFromStubbedRetrieverAndInitializeCacheWithPersistentFaults() throws IOException { - // given - String inputPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/input.json"); - String outputPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/output.json"); - String reportPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report.json"); - String cachePath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text1.json"); - - CacheMetadataManagingProcess cacheManager = new CacheMetadataManagingProcess(); - - AvroTestUtils.createLocalAvroDataStore(JsonAvroTestUtils.readJsonDataStore(inputPath, ImportedPatent.class), - inputDir.toString()); - - SparkJob sparkJob = buildSparkJob(inputDir.toString(), outputDir.toString(), outputFaultDir.toString(), outputReportDir.toString(), - "eu.dnetlib.iis.wf.referenceextraction.patent.TestServiceFacadeFactories$StubServiceFacadeFactoryWithPersistentFailure"); - - // when - executor.execute(sparkJob); - - // then - Configuration conf = new Configuration(); - - // text output contents - assertEquals(2, HdfsTestUtils.countFiles(conf, outputDir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputDir.toString(), outputPath, DocumentText.class); - - // fault output contents - need to validate faults programmatically due to dynamic timestamp generation - assertEquals(2, HdfsTestUtils.countFiles(conf, outputFaultDir.toString(), DataStore.AVRO_FILE_EXT)); - validateFaults(AvroTestUtils.readLocalAvroDataStore(outputFaultDir.toString())); - - String cacheId = cacheManager.getExistingCacheId(conf, new org.apache.hadoop.fs.Path(cacheRootDir.toString())); - assertNotNull(cacheId); - - // text cache contents - org.apache.hadoop.fs.Path textCacheLocation = CacheStorageUtils - .getCacheLocation(new org.apache.hadoop.fs.Path(cacheRootDir.toString()), cacheId, CacheRecordType.data); - assertEquals(2, HdfsTestUtils.countFiles(conf, textCacheLocation.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(textCacheLocation.toString(), cachePath, DocumentText.class); - - // fault output contents - org.apache.hadoop.fs.Path faultCacheLocation = CacheStorageUtils - .getCacheLocation(new org.apache.hadoop.fs.Path(cacheRootDir.toString()), cacheId, CacheRecordType.fault); - assertEquals(2, HdfsTestUtils.countFiles(conf, faultCacheLocation.toString(), DataStore.AVRO_FILE_EXT)); - validateFaults(AvroTestUtils.readLocalAvroDataStore(faultCacheLocation.toString())); - - // evaluating report - assertEquals(1, HdfsTestUtils.countFiles(conf, outputReportDir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputReportDir.toString(), reportPath, ReportEntry.class); - } - - @Test - public void testRetrievePatentMetaFromStubbedRetrieverAndNotInitializeCacheWithTransientFaults() throws IOException { - // given - String inputPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/input.json"); - String outputPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/output.json"); - String reportPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report.json"); - String cachePath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text12.json"); - - CacheMetadataManagingProcess cacheManager = new CacheMetadataManagingProcess(); - - AvroTestUtils.createLocalAvroDataStore(JsonAvroTestUtils.readJsonDataStore(inputPath, ImportedPatent.class), - inputDir.toString()); - - SparkJob sparkJob = buildSparkJob(inputDir.toString(), outputDir.toString(), outputFaultDir.toString(), outputReportDir.toString(), - "eu.dnetlib.iis.wf.referenceextraction.patent.TestServiceFacadeFactories$StubServiceFacadeFactoryWithTransientFailure"); - - // when - executor.execute(sparkJob); - - // then - Configuration conf = new Configuration(); - - // text output contents - assertEquals(2, HdfsTestUtils.countFiles(conf, outputDir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputDir.toString(), outputPath, DocumentText.class); - - // fault output contents - need to validate faults programmatically due to dynamic timestamp generation - assertEquals(2, HdfsTestUtils.countFiles(conf, outputFaultDir.toString(), DataStore.AVRO_FILE_EXT)); - validateFaults(AvroTestUtils.readLocalAvroDataStore(outputFaultDir.toString())); - - String cacheId = cacheManager.getExistingCacheId(conf, new org.apache.hadoop.fs.Path(cacheRootDir.toString())); - assertNotNull(cacheId); - - // text cache contents - org.apache.hadoop.fs.Path textCacheLocation = CacheStorageUtils - .getCacheLocation(new org.apache.hadoop.fs.Path(cacheRootDir.toString()), cacheId, CacheRecordType.data); - assertEquals(2, HdfsTestUtils.countFiles(conf, textCacheLocation.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(textCacheLocation.toString(), cachePath, DocumentText.class); - - // fault output contents - org.apache.hadoop.fs.Path faultCacheLocation = CacheStorageUtils - .getCacheLocation(new org.apache.hadoop.fs.Path(cacheRootDir.toString()), cacheId, CacheRecordType.fault); - assertEquals(2, HdfsTestUtils.countFiles(conf, faultCacheLocation.toString(), DataStore.AVRO_FILE_EXT)); - assertTrue(AvroTestUtils.readLocalAvroDataStore(faultCacheLocation.toString()).isEmpty()); - - // evaluating report - assertEquals(1, HdfsTestUtils.countFiles(conf, outputReportDir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputReportDir.toString(), reportPath, ReportEntry.class); - } - - @Test - public void testRetrievePatentMetaFromStubbedRetrieverAndUpdateCache() throws IOException { - // given - String inputPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/input.json"); - String input2Path = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/input2.json"); - String outputPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/output.json"); - String output2Path = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/output2.json"); - String reportPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report.json"); - String report2Path = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report_update.json"); - String cachePath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text2.json"); - - CacheMetadataManagingProcess cacheManager = new CacheMetadataManagingProcess(); - - AvroTestUtils.createLocalAvroDataStore(JsonAvroTestUtils.readJsonDataStore(inputPath, ImportedPatent.class), - inputDir.toString()); - AvroTestUtils.createLocalAvroDataStore(JsonAvroTestUtils.readJsonDataStore(input2Path, ImportedPatent.class), - input2Dir.toString()); - - // when - executor.execute(buildSparkJob(inputDir.toString(), outputDir.toString(), outputFaultDir.toString(), outputReportDir.toString(), - "eu.dnetlib.iis.wf.referenceextraction.patent.TestServiceFacadeFactories$StubServiceFacadeFactoryWithPersistentFailure")); - executor.execute(buildSparkJob(input2Dir.toString(), output2Dir.toString(), outputFault2Dir.toString(), outputReport2Dir.toString(), - "eu.dnetlib.iis.wf.referenceextraction.patent.TestServiceFacadeFactories$StubServiceFacadeFactoryWithPersistentFailure")); - - // then - Configuration conf = new Configuration(); - - // text output contents - assertEquals(2, HdfsTestUtils.countFiles(conf, outputDir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputDir.toString(), outputPath, DocumentText.class); - - assertEquals(2, HdfsTestUtils.countFiles(conf, output2Dir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(output2Dir.toString(), output2Path, DocumentText.class); - - // fault output contents - need to validate faults programmatically due to dynamic timestamp generation - assertEquals(2, HdfsTestUtils.countFiles(conf, outputFaultDir.toString(), DataStore.AVRO_FILE_EXT)); - validateFaults(AvroTestUtils.readLocalAvroDataStore(outputFaultDir.toString())); - - assertEquals(2, HdfsTestUtils.countFiles(conf, outputFault2Dir.toString(), DataStore.AVRO_FILE_EXT)); - assertTrue(AvroTestUtils.readLocalAvroDataStore(outputFault2Dir.toString()).isEmpty()); - - String cacheId = cacheManager.getExistingCacheId(conf, new org.apache.hadoop.fs.Path(cacheRootDir.toString())); - assertNotNull(cacheId); - - // text cache contents - org.apache.hadoop.fs.Path textCacheLocation = CacheStorageUtils - .getCacheLocation(new org.apache.hadoop.fs.Path(cacheRootDir.toString()), cacheId, CacheRecordType.data); - assertEquals(2, HdfsTestUtils.countFiles(conf, textCacheLocation.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(textCacheLocation.toString(), cachePath, DocumentText.class); - - // fault output contents - org.apache.hadoop.fs.Path faultCacheLocation = CacheStorageUtils - .getCacheLocation(new org.apache.hadoop.fs.Path(cacheRootDir.toString()), cacheId, CacheRecordType.fault); - assertEquals(2, HdfsTestUtils.countFiles(conf, faultCacheLocation.toString(), DataStore.AVRO_FILE_EXT)); - validateFaults(AvroTestUtils.readLocalAvroDataStore(faultCacheLocation.toString())); - - // evaluating report - assertEquals(1, HdfsTestUtils.countFiles(conf, outputReportDir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputReportDir.toString(), reportPath, ReportEntry.class); - - assertEquals(1, HdfsTestUtils.countFiles(conf, outputReport2Dir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputReport2Dir.toString(), report2Path, ReportEntry.class); - } - - @Test - public void testObtainPatentMetaFromCacheWithoutCallingPatentServiceFacade() throws IOException { - // given - String inputPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/input.json"); - String outputPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/output.json"); - String reportPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report.json"); - String report2Path = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report_from_cache.json"); - String cachePath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text1.json"); - - CacheMetadataManagingProcess cacheManager = new CacheMetadataManagingProcess(); - - AvroTestUtils.createLocalAvroDataStore(JsonAvroTestUtils.readJsonDataStore(inputPath, ImportedPatent.class), - inputDir.toString()); - - // when - executor.execute(buildSparkJob(inputDir.toString(), outputDir.toString(), outputFaultDir.toString(), outputReportDir.toString(), - "eu.dnetlib.iis.wf.referenceextraction.patent.TestServiceFacadeFactories$StubServiceFacadeFactoryWithPersistentFailure")); - executor.execute(buildSparkJob(inputDir.toString(), output2Dir.toString(), outputFault2Dir.toString(), outputReport2Dir.toString(), - "eu.dnetlib.iis.wf.referenceextraction.patent.TestServiceFacadeFactories$ExceptionThrowingFacadeFactory")); - - // then - Configuration conf = new Configuration(); - - // text output contents - assertEquals(2, HdfsTestUtils.countFiles(conf, outputDir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputDir.toString(), outputPath, DocumentText.class); - - assertEquals(2, HdfsTestUtils.countFiles(conf, output2Dir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(output2Dir.toString(), outputPath, DocumentText.class); - - // fault output contents - need to validate faults programmatically due to dynamic timestamp generation - assertEquals(2, HdfsTestUtils.countFiles(conf, outputFaultDir.toString(), DataStore.AVRO_FILE_EXT)); - validateFaults(AvroTestUtils.readLocalAvroDataStore(outputFaultDir.toString())); - - String cacheId = cacheManager.getExistingCacheId(conf, new org.apache.hadoop.fs.Path(cacheRootDir.toString())); - assertNotNull(cacheId); - - // text cache contents - org.apache.hadoop.fs.Path textCacheLocation = CacheStorageUtils - .getCacheLocation(new org.apache.hadoop.fs.Path(cacheRootDir.toString()), cacheId, CacheRecordType.data); - assertEquals(2, HdfsTestUtils.countFiles(conf, textCacheLocation.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(textCacheLocation.toString(), cachePath, DocumentText.class); - - // fault cache contents - org.apache.hadoop.fs.Path faultCacheLocation = CacheStorageUtils - .getCacheLocation(new org.apache.hadoop.fs.Path(cacheRootDir.toString()), cacheId, CacheRecordType.fault); - assertEquals(2, HdfsTestUtils.countFiles(conf, faultCacheLocation.toString(), DataStore.AVRO_FILE_EXT)); - validateFaults(AvroTestUtils.readLocalAvroDataStore(faultCacheLocation.toString())); - - // evaluating report - assertEquals(1, HdfsTestUtils.countFiles(conf, outputReportDir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputReportDir.toString(), reportPath, ReportEntry.class); - - assertEquals(1, HdfsTestUtils.countFiles(conf, outputReport2Dir.toString(), DataStore.AVRO_FILE_EXT)); - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputReport2Dir.toString(), report2Path, ReportEntry.class); - - assertEquals(1, HdfsTestUtils.countFiles(conf, outputReport2Dir.toString(), DataStore.AVRO_FILE_EXT)); - assertTrue(AvroTestUtils.readLocalAvroDataStore(outputFault2Dir.toString()).isEmpty()); - } - - private void validateFaults(List faults) { - assertNotNull(faults); - assertEquals(1, faults.size()); - assertEquals("00000002", faults.get(0).getInputObjectId().toString()); - assertEquals("unable to find element", faults.get(0).getMessage().toString()); - assertEquals(PatentWebServiceFacadeException.class.getCanonicalName(), faults.get(0).getCode().toString()); - } - - private SparkJob buildSparkJob(String inputPath, String outputPath, String outputFaultPath, - String outputReportPath, String patentServiceFacadeFactoryClassName) { - return SparkJobBuilder.create() - .setAppName(getClass().getName()) - .setMainClass(PatentMetadataRetrieverJob.class) - .addArg("-inputPath", inputPath) - .addArg("-numberOfEmittedFiles", String.valueOf(2)) - .addArg("-lockManagerFactoryClassName", ZookeeperLockManagerFactory.class.getName()) - .addArg("-cacheRootDir", cacheRootDir.toString()) - .addArg("-outputPath", outputPath) - .addArg("-outputFaultPath", outputFaultPath) - .addArg("-outputReportPath", outputReportPath) - .addArg("-patentServiceFacadeFactoryClassName", patentServiceFacadeFactoryClassName) - .addArg("-DtestParam", "testValue") - .addJobProperty("spark.driver.host", "localhost") - .addJobProperty(ZKFailoverController.ZK_QUORUM_KEY, "localhost:" + zookeeperServer.getPort()) - .build(); - } -} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/TestServiceFacadeFactories.java b/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/TestServiceFacadeFactories.java deleted file mode 100644 index 2ed9c5eaf..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/TestServiceFacadeFactories.java +++ /dev/null @@ -1,170 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent; - -import com.google.common.base.Preconditions; -import eu.dnetlib.iis.common.ClassPathResourceProvider; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import eu.dnetlib.iis.wf.importer.facade.ServiceFacadeFactory; -import eu.dnetlib.iis.wf.referenceextraction.FacadeContentRetriever; -import eu.dnetlib.iis.wf.referenceextraction.FacadeContentRetrieverResponse; - -import java.io.Serializable; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.function.Function; - -/** - * Patent service facade factories used for testing. - */ -public class TestServiceFacadeFactories { - - private TestServiceFacadeFactories() { - } - - /** - * Simple mock retrieving XML contents as files from classpath. Relies on - * publn_auth, publn_nr, publn_kind fields defined in {@link ImportedPatent} - * while generating filename: - *

- * publn_auth + '.' + publn_nr + '.' + publn_kind + ".xml" - *

- * Throws an exception if a content is retrieved more than once. - * - * @author mhorst - */ - public static class FileContentReturningFacadeFactory implements ServiceFacadeFactory>, Serializable { - - private static final String classPathRoot = "/eu/dnetlib/iis/wf/referenceextraction/patent/data/mock_facade_storage/"; - - public static class Retriever extends FacadeContentRetriever { - public final Set retrievedUrls = new HashSet<>(); - - @Override - protected String buildUrl(ImportedPatent objToBuildUrl) { - StringBuilder strBuilder = new StringBuilder(); - strBuilder.append(objToBuildUrl.getPublnAuth()); - strBuilder.append('.'); - strBuilder.append(objToBuildUrl.getPublnNr()); - strBuilder.append('.'); - strBuilder.append(objToBuildUrl.getPublnKind()); - strBuilder.append(".xml"); - return strBuilder.toString(); - } - - @Override - protected FacadeContentRetrieverResponse retrieveContentOrThrow(String url, int retryCount) throws Exception { - if (retrievedUrls.contains(url)) { - throw new RuntimeException("requesting already processed url: " + url); - } - FacadeContentRetrieverResponse.Success result = FacadeContentRetrieverResponse - .success(ClassPathResourceProvider.getResourceContent(classPathRoot + url)); - retrievedUrls.add(url); - return result; - } - } - - @Override - public FacadeContentRetriever instantiate(Map parameters) { - return new Retriever(); - } - } - - private static abstract class StubServiceFacadeFactoryWithFailure implements ServiceFacadeFactory>, Serializable { - private static final String expectedParamName = "testParam"; - private static final String expectedParamValue = "testValue"; - - public static class Retriever extends FacadeContentRetriever { - private final Set retrievedUrls = new HashSet<>(); - private final Function> failureProducerFn; - - public Retriever(Function> failureProducerFn) { - this.failureProducerFn = failureProducerFn; - } - - @Override - protected String buildUrl(ImportedPatent objToBuildUrl) { - StringBuilder strBuilder = new StringBuilder(); - strBuilder.append(objToBuildUrl.getApplnAuth()); - strBuilder.append('-'); - strBuilder.append(objToBuildUrl.getApplnNr()); - strBuilder.append('-'); - strBuilder.append(objToBuildUrl.getPublnAuth()); - strBuilder.append('-'); - strBuilder.append(objToBuildUrl.getPublnNr()); - strBuilder.append('-'); - strBuilder.append(objToBuildUrl.getPublnKind()); - return strBuilder.toString(); - } - - @Override - protected FacadeContentRetrieverResponse retrieveContentOrThrow(String url, int retryCount) throws Exception { - if (retrievedUrls.contains(url)) { - throw new RuntimeException("requesting already processed url: " + url); - } - FacadeContentRetrieverResponse result = url.contains("non-existing") ? - failureProducerFn.apply(new PatentWebServiceFacadeException("unable to find element")) - : FacadeContentRetrieverResponse.success(url); - retrievedUrls.add(url); - return result; - } - } - - protected abstract FacadeContentRetrieverResponse.Failure failure(Exception e); - - @Override - public FacadeContentRetriever instantiate(Map parameters) { - String paramValue = parameters.get(expectedParamName); - Preconditions.checkArgument(expectedParamValue.equals(paramValue), - "'%s' parameter value: '%s' is different than the expected one: '%s'", expectedParamName, paramValue, expectedParamValue); - return new Retriever((Function> & Serializable) this::failure); - } - } - - /** - * Simple stub factory producing {@link FacadeContentRetriever} and persistent failure on error. Throws an exception - * if a content is retrieved more than once. - */ - public static class StubServiceFacadeFactoryWithPersistentFailure extends StubServiceFacadeFactoryWithFailure { - - @Override - protected FacadeContentRetrieverResponse.Failure failure(Exception e) { - return FacadeContentRetrieverResponse.persistentFailure(e); - } - } - - /** - * Simple stub factory producing {@link FacadeContentRetriever} and transient failure on error. Throws an exception - * if a content is retrieved more than once. - */ - public static class StubServiceFacadeFactoryWithTransientFailure extends StubServiceFacadeFactoryWithFailure { - - @Override - protected FacadeContentRetrieverResponse.Failure failure(Exception e) { - return FacadeContentRetrieverResponse.transientFailure(e); - } - } - - /** - * Factory throwing an exception. - */ - public static class ExceptionThrowingFacadeFactory implements ServiceFacadeFactory>, Serializable { - - public static class Retriever extends FacadeContentRetriever { - - @Override - protected String buildUrl(ImportedPatent objToBuildUrl) { - return "/url/to/patent"; - } - - @Override - protected FacadeContentRetrieverResponse retrieveContentOrThrow(String url, int retryCount) throws Exception { - throw new RuntimeException("unexpected call for url: " + url); - } - } - - @Override - public FacadeContentRetriever instantiate(Map parameters) { - return new Retriever(); - } - } -} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/input/PatentMetadataRetrieverInputTransformerJobTest.java b/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/input/PatentMetadataRetrieverInputTransformerJobTest.java deleted file mode 100644 index a3e6fa80f..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/input/PatentMetadataRetrieverInputTransformerJobTest.java +++ /dev/null @@ -1,79 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent.input; - -import eu.dnetlib.iis.common.ClassPathResourceProvider; -import eu.dnetlib.iis.common.SlowTest; -import eu.dnetlib.iis.common.utils.AvroAssertTestUtil; -import eu.dnetlib.iis.common.utils.AvroTestUtils; -import eu.dnetlib.iis.common.utils.JsonAvroTestUtils; -import eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import pl.edu.icm.sparkutils.test.SparkJob; -import pl.edu.icm.sparkutils.test.SparkJobBuilder; -import pl.edu.icm.sparkutils.test.SparkJobExecutor; - -import java.io.IOException; -import java.nio.file.Path; - -/** - * {@link PatentMetadataRetrieverInputTransformerJob} test class. - * - * @author mhorst - * - */ -@SlowTest -public class PatentMetadataRetrieverInputTransformerJobTest { - - private SparkJobExecutor executor = new SparkJobExecutor(); - - @TempDir - public Path workingDir; - - private Path inputImportedPatentDir; - private Path inputMatchedPatentDir; - private Path outputDir; - - @BeforeEach - public void before() { - inputImportedPatentDir = workingDir.resolve("input-imported"); - inputMatchedPatentDir = workingDir.resolve("input-matched"); - outputDir = workingDir.resolve("output"); - } - - @Test - public void shouldConvertAvroDatastoreForMetadataRetrieval() throws IOException { - // given - String inputImportedPatentPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/transformer/input_imported_patent.json"); - String inputMatchedPatentPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/transformer/input_matched_patent.json"); - - String outputTransformedPatentPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/transformer/output.json"); - - AvroTestUtils.createLocalAvroDataStore(JsonAvroTestUtils.readJsonDataStore(inputImportedPatentPath, ImportedPatent.class), inputImportedPatentDir.toString()); - AvroTestUtils.createLocalAvroDataStore(JsonAvroTestUtils.readJsonDataStore(inputMatchedPatentPath, DocumentToPatent.class), inputMatchedPatentDir.toString()); - - SparkJob sparkJob = buildSparkJob(); - - // when - executor.execute(sparkJob); - - // then - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputDir.toString(), outputTransformedPatentPath, ImportedPatent.class); - } - - private SparkJob buildSparkJob() { - return SparkJobBuilder.create() - .setAppName(getClass().getName()) - .setMainClass(PatentMetadataRetrieverInputTransformerJob.class) - .addArg("-inputImportedPatentPath", inputImportedPatentDir.toString()) - .addArg("-inputMatchedPatentPath", inputMatchedPatentDir.toString()) - .addArg("-outputPath", outputDir.toString()) - .addJobProperty("spark.driver.host", "localhost") - .build(); - } - -} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/input/PatentReferenceExtractionInputTransformerJobTest.java b/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/input/PatentReferenceExtractionInputTransformerJobTest.java deleted file mode 100644 index e7eee18ac..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/input/PatentReferenceExtractionInputTransformerJobTest.java +++ /dev/null @@ -1,64 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent.input; - -import eu.dnetlib.iis.common.ClassPathResourceProvider; -import eu.dnetlib.iis.common.SlowTest; -import eu.dnetlib.iis.common.utils.AvroAssertTestUtil; -import eu.dnetlib.iis.common.utils.AvroTestUtils; -import eu.dnetlib.iis.common.utils.JsonAvroTestUtils; -import eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent; -import eu.dnetlib.iis.referenceextraction.patent.schemas.PatentReferenceExtractionInput; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; -import pl.edu.icm.sparkutils.test.SparkJob; -import pl.edu.icm.sparkutils.test.SparkJobBuilder; -import pl.edu.icm.sparkutils.test.SparkJobExecutor; - -import java.io.IOException; -import java.nio.file.Path; - -@SlowTest -public class PatentReferenceExtractionInputTransformerJobTest { - private SparkJobExecutor executor = new SparkJobExecutor(); - - @TempDir - public Path workingDir; - - private Path inputDir; - private Path outputDir; - - @BeforeEach - public void before() { - inputDir = workingDir.resolve("input"); - outputDir = workingDir.resolve("output"); - } - - @Test - public void shouldConvertAvroDatastoreForReferenceExtraction() throws IOException { - // given - String inputPatentPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/input_transformer/imported_patent.json"); - String outputTransformedPatentPath = ClassPathResourceProvider - .getResourcePath("eu/dnetlib/iis/wf/referenceextraction/patent/data/input_transformer/patent_transformed.json"); - AvroTestUtils.createLocalAvroDataStore(JsonAvroTestUtils.readJsonDataStore(inputPatentPath, ImportedPatent.class), inputDir.toString()); - - SparkJob sparkJob = buildSparkJob(); - - // when - executor.execute(sparkJob); - - // then - AvroAssertTestUtil.assertEqualsWithJsonIgnoreOrder(outputDir.toString(), outputTransformedPatentPath, PatentReferenceExtractionInput.class); - } - - private SparkJob buildSparkJob() { - return SparkJobBuilder.create() - .setAppName(getClass().getName()) - .setMainClass(PatentReferenceExtractionInputTransformerJob.class) - .addArg("-inputPath", inputDir.toString()) - .addArg("-outputPath", outputDir.toString()) - .addJobProperty("spark.driver.host", "localhost") - .build(); - } - -} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/parser/OpsPatentMetadataXPathBasedParserTest.java b/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/parser/OpsPatentMetadataXPathBasedParserTest.java deleted file mode 100644 index 51324a1a5..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/java/eu/dnetlib/iis/wf/referenceextraction/patent/parser/OpsPatentMetadataXPathBasedParserTest.java +++ /dev/null @@ -1,337 +0,0 @@ -package eu.dnetlib.iis.wf.referenceextraction.patent.parser; - -import com.google.common.collect.Lists; -import eu.dnetlib.iis.common.ClassPathResourceProvider; -import eu.dnetlib.iis.referenceextraction.patent.schemas.Patent; -import org.apache.commons.collections.CollectionUtils; -import org.junit.jupiter.api.Test; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import java.io.StringWriter; -import java.util.List; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * {@link OpsPatentMetadataXPathBasedParser} test class. - * - * @author mhorst - * - */ -public class OpsPatentMetadataXPathBasedParserTest { - - static final String xmlResourcesRootClassPath = "/eu/dnetlib/iis/wf/referenceextraction/patent/data/"; - - private OpsPatentMetadataXPathBasedParser parser = new OpsPatentMetadataXPathBasedParser(); - - - // -------------------------------- TESTS ---------------------------- - - @Test - public void testExtractMetadataFromValidXMLfile() throws Exception { - // given - String xmlContents = ClassPathResourceProvider - .getResourceContent(xmlResourcesRootClassPath + "WO.0042078.A1.xml"); - - // execute - Patent.Builder patent = parser.parse(xmlContents, getPatentBuilderInitializedWithRequiredFields()); - - // assert - assertNotNull(patent); - assertTrue(patent.hasApplnTitle()); - assertEquals("AQUEOUS PEROXIDE EMULSIONS", patent.getApplnTitle()); - assertNotNull(patent.getApplnAbstract()); - assertEquals("Simple abstract in English", patent.getApplnAbstract()); - assertNotNull(patent.getIpcClassSymbol()); - assertEquals(1, patent.getIpcClassSymbol().size()); - assertEquals("C08F2/20", patent.getIpcClassSymbol().get(0)); - assertEquals("WO2000EP00003", patent.getApplnNrEpodoc()); - assertEquals("2000-01-06", patent.getApplnFilingDate()); - assertEquals("2000-07-20", patent.getEarliestPublnDate()); - - assertNotNull(patent.getApplicantNames()); - assertEquals(3, patent.getApplicantNames().size()); - assertEquals("AKZO NOBEL NV", patent.getApplicantNames().get(0)); - assertEquals("WESTMIJZE HANS", patent.getApplicantNames().get(1)); - assertEquals("BOEN HO O", patent.getApplicantNames().get(2)); - - assertNotNull(patent.getApplicantCountryCodes()); - assertEquals(3, patent.getApplicantCountryCodes().size()); - assertEquals("NL", patent.getApplicantCountryCodes().get(0)); - assertEquals("NL", patent.getApplicantCountryCodes().get(1)); - assertEquals("NL", patent.getApplicantCountryCodes().get(2)); - } - - @Test - public void testExtractFromDocumentWithAllFieldsMissing() throws Exception { - // given - String xmlContents = buildXMLContents(null, null, null, null, null, null, null); - - // execute - Patent.Builder patentBuilder = parser.parse(xmlContents, getPatentBuilderInitializedWithRequiredFields()); - - // assert - Patent patent = patentBuilder.build(); - assertNotNull(patent); - assertNull(patent.getApplnTitle()); - assertNull(patent.getApplnAbstract()); - assertNull(patent.getApplnNrEpodoc()); - assertNull(patent.getApplnFilingDate()); - assertNull(patent.getEarliestPublnDate()); - assertNull(patent.getIpcClassSymbol()); - } - - @Test - public void testExtractFromDocumentWithAllFieldsSet() throws Exception { - // given - String title = "some title"; - String abstractText = "some abstract"; - String classIpcText = "some class IPC text"; - String applnEpodocId = "1234"; - String applnDate = "20200501"; - String publnDate = "20000530"; - List epodocApplicantNames = Lists.newArrayList("John Smith [PL]"); - - String xmlContents = buildXMLContents(title, abstractText, classIpcText, applnEpodocId, applnDate, publnDate, - epodocApplicantNames); - - // execute - Patent.Builder patentBuilder = parser.parse(xmlContents, getPatentBuilderInitializedWithRequiredFields()); - - // assert - Patent patent = patentBuilder.build(); - assertNotNull(patent); - assertEquals(title, patent.getApplnTitle()); - assertNotNull(patent.getApplnAbstract()); - assertEquals(abstractText, patent.getApplnAbstract()); - assertNotNull(patent.getIpcClassSymbol()); - assertEquals(1, patent.getIpcClassSymbol().size()); - assertEquals(classIpcText, patent.getIpcClassSymbol().get(0)); - assertEquals(applnEpodocId, patent.getApplnNrEpodoc()); - assertEquals("2020-05-01", patent.getApplnFilingDate()); - assertEquals("2000-05-30", patent.getEarliestPublnDate()); - - assertNotNull(patent.getApplicantNames()); - assertEquals(1, patent.getApplicantNames().size()); - assertEquals("John Smith", patent.getApplicantNames().get(0)); - - assertNotNull(patent.getApplicantCountryCodes()); - assertEquals(1, patent.getApplicantCountryCodes().size()); - assertEquals("PL", patent.getApplicantCountryCodes().get(0)); - } - - @Test - public void testExtractFromDocumentWithPublnDateAlreadyDefinedInTargetFormat() throws Exception { - // given - String publnDate = "2000-05-30"; - - // execute - Patent.Builder patentBuilder = parser.parse(buildXMLContentsWithPublnDate(publnDate), getPatentBuilderInitializedWithRequiredFields()); - - // assert - Patent patent = patentBuilder.build(); - assertNotNull(patent); - assertEquals("2000-05-30", patent.getEarliestPublnDate()); - } - - @Test - public void testExtractFromDocumentWithPublnDateDefinedInUnsupportedFormat() throws Exception { - // given - String publnDate = "30/05/99"; - - // execute - Patent.Builder patentBuilder = parser.parse(buildXMLContentsWithPublnDate(publnDate), getPatentBuilderInitializedWithRequiredFields()); - - // assert - Patent patent = patentBuilder.build(); - assertNotNull(patent); - assertEquals("30/05/99", patent.getEarliestPublnDate()); - } - - @Test - public void testExtractFromDocumentWithApplicantNamesWithoutCountry() throws Exception { - // given - String title = null; - String abstractText = null; - String classIpcText = null; - String applnEpodocId = null; - String applnDate = null; - String publnDate = null; - List epodocApplicantNames = Lists.newArrayList("John Smith"); - - String xmlContents = buildXMLContents(title, abstractText, classIpcText, applnEpodocId, applnDate, publnDate, - epodocApplicantNames); - - // execute - Patent.Builder patentBuilder = parser.parse(xmlContents, getPatentBuilderInitializedWithRequiredFields()); - - // assert - Patent patent = patentBuilder.build(); - assertNotNull(patent); - assertNotNull(patent.getApplicantNames()); - assertEquals("John Smith", patent.getApplicantNames().get(0)); - - assertNotNull(patent.getApplicantCountryCodes()); - assertEquals(1, patent.getApplicantCountryCodes().size()); - assertEquals("", patent.getApplicantCountryCodes().get(0)); - } - - @Test - public void testExtractFromDocumentWithApplicantNamesToBeCleanedUp() throws Exception { - // given - String title = null; - String abstractText = null; - String classIpcText = null; - String applnEpodocId = null; - String applnDate = null; - String publnDate = null; - List epodocApplicantNames = Lists.newArrayList("Smith J., ", " Doe J."); - - String xmlContents = buildXMLContents(title, abstractText, classIpcText, applnEpodocId, applnDate, publnDate, - epodocApplicantNames); - - // execute - Patent.Builder patentBuilder = parser.parse(xmlContents, getPatentBuilderInitializedWithRequiredFields()); - - // assert - Patent patent = patentBuilder.build(); - assertNotNull(patent); - assertNotNull(patent.getApplicantCountryCodes()); - assertEquals("", patent.getApplicantCountryCodes().get(0)); - assertEquals("", patent.getApplicantCountryCodes().get(1)); - - assertNotNull(patent.getApplicantNames()); - assertEquals(2, patent.getApplicantNames().size()); - assertEquals("Smith J.", patent.getApplicantNames().get(0)); - assertEquals("Doe J.", patent.getApplicantNames().get(1)); - } - - // -------------------------------- PRIVATE ---------------------------- - - private Patent.Builder getPatentBuilderInitializedWithRequiredFields() { - Patent.Builder patentBuilder = Patent.newBuilder(); - patentBuilder.setApplnAuth("irrelevant"); - patentBuilder.setApplnNr("irrelevant"); - return patentBuilder; - } - - private String buildXMLContentsWithPublnDate(String publnDate) throws ParserConfigurationException, TransformerException { - return buildXMLContents(null, null, null, null, null, publnDate, null); - } - - private String buildXMLContents(String title, String abstractText, String classIpcText, String applnEpodocId, - String applnDate, String publnDate, List epodocApplicantNames) throws ParserConfigurationException, TransformerException { - - DocumentBuilderFactory docFactory = DocumentBuilderFactory.newInstance(); - DocumentBuilder docBuilder = docFactory.newDocumentBuilder(); - - Document doc = docBuilder.newDocument(); - Element rootElement = doc.createElement("world-patent-data"); - doc.appendChild(rootElement); - - Element exchangeDocs = doc.createElement("exchange-documents"); - rootElement.appendChild(exchangeDocs); - - Element exchangeDoc = doc.createElement("exchange-document"); - exchangeDocs.appendChild(exchangeDoc); - - if (abstractText != null) { - Element abstractEl = doc.createElement("abstract"); - // currently the lang is unspecified - abstractEl.appendChild(doc.createTextNode(abstractText)); - exchangeDoc.appendChild(abstractEl); - } - - Element biblioData = doc.createElement("bibliographic-data"); - exchangeDoc.appendChild(biblioData); - - if (title != null) { - Element titleEl = doc.createElement("invention-title"); - titleEl.appendChild(doc.createTextNode(title)); - biblioData.appendChild(titleEl); - } - - if (classIpcText != null) { - Element classIpc = doc.createElement("classification-ipc"); - biblioData.appendChild(classIpc); - - Element classIpcTextEl = doc.createElement("text"); - classIpcTextEl.appendChild(doc.createTextNode(classIpcText)); - classIpc.appendChild(classIpcTextEl); - - } - - { - Element applnReference = doc.createElement("application-reference"); - biblioData.appendChild(applnReference); - - Element documentId = doc.createElement("document-id"); - documentId.setAttribute("document-id-type", "epodoc"); - applnReference.appendChild(documentId); - - if (applnEpodocId != null) { - Element docNumber = doc.createElement("doc-number"); - docNumber.appendChild(doc.createTextNode(applnEpodocId)); - documentId.appendChild(docNumber); - } - if (applnDate != null) { - Element date = doc.createElement("date"); - date.appendChild(doc.createTextNode(applnDate)); - documentId.appendChild(date); - } - } - - { - Element publnReference = doc.createElement("publication-reference"); - biblioData.appendChild(publnReference); - - Element documentId = doc.createElement("document-id"); - documentId.setAttribute("document-id-type", "epodoc"); - publnReference.appendChild(documentId); - - if (publnDate != null) { - Element date = doc.createElement("date"); - date.appendChild(doc.createTextNode(publnDate)); - documentId.appendChild(date); - } - } - - Element parties = doc.createElement("parties"); - biblioData.appendChild(parties); - - Element applicants = doc.createElement("applicants"); - parties.appendChild(applicants); - - if (CollectionUtils.isNotEmpty(epodocApplicantNames)) { - for (String epodocApplicantName : epodocApplicantNames) { - Element applicant = doc.createElement("applicant"); - applicant.setAttribute("data-format", "epodoc"); - applicants.appendChild(applicant); - - Element applicantName = doc.createElement("applicant-name"); - applicant.appendChild(applicantName); - - Element name = doc.createElement("name"); - name.appendChild(doc.createTextNode(epodocApplicantName)); - applicantName.appendChild(name); - } - } - - TransformerFactory transformerFactory = TransformerFactory.newInstance(); - Transformer transformer = transformerFactory.newTransformer(); - DOMSource source = new DOMSource(doc); - StringWriter strWriter = new StringWriter(); - transformer.transform(source, new StreamResult(strWriter)); - return strWriter.toString(); - } - -} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/WO.0042078.A1.no_country_code.xml b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/WO.0042078.A1.no_country_code.xml deleted file mode 100644 index 667c0075b..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/WO.0042078.A1.no_country_code.xml +++ /dev/null @@ -1,102 +0,0 @@ - - - - - - - - WO - 0042078 - A1 - 20000721 - - - WO0042078 - 20000720 - - - - C08F2/20 - - - - EP - 0000003 - W - - - WO2000EP00003 - 20000106 - - - EP0000003 - - - - - - - AKZO NOBEL NV - - - - - WESTMIJZE HANS - - - - - BOEN HO O - - - - - AKZO NOBEL N.V, - - - - - WESTMIJZE, HANS, - - - - - O, BOEN, HO - - - - - - - WESTMIJZE HANS [NL] - - - - - O BOEN HO [NL] - - - - - WESTMIJZE, HANS, - - - - - O, BOEN, HO - - - - - EMULSIONS AQUEUSES DE PEROXYDE - AQUEOUS PEROXIDE EMULSIONS - - -

Simple abstract in English

- - -

Simple abstract in French

- - - - diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/WO.0042078.A1.xml b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/WO.0042078.A1.xml deleted file mode 100644 index 3bd08dbd0..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/WO.0042078.A1.xml +++ /dev/null @@ -1,102 +0,0 @@ - - - - - - - - WO - 0042078 - A1 - 20000721 - - - WO0042078 - 20000720 - - - - C08F2/20 - - - - EP - 0000003 - W - - - WO2000EP00003 - 20000106 - - - EP0000003 - - - - - - - AKZO NOBEL NV [NL] - - - - - WESTMIJZE HANS [NL] - - - - - BOEN HO O [NL] - - - - - AKZO NOBEL N.V, - - - - - WESTMIJZE, HANS, - - - - - O, BOEN, HO - - - - - - - WESTMIJZE HANS [NL] - - - - - O BOEN HO [NL] - - - - - WESTMIJZE, HANS, - - - - - O, BOEN, HO - - - - - EMULSIONS AQUEUSES DE PEROXYDE - AQUEOUS PEROXIDE EMULSIONS - - -

Simple abstract in English

- - -

Simple abstract in French

- - - - diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/document_to_patent.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/document_to_patent.json index 09d23ac9b..6977c1d33 100644 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/document_to_patent.json +++ b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/document_to_patent.json @@ -1,42 +1,18 @@ { - "documentId": "50|dedup_wf_001::4f3578410aba958f2c228051f1a711e8", - "appln_nr": "08159385", + "documentId": "50|dedup_wf_001::77290501959cbe2dc541f6f760c84ad8", + "lensId": "190-148-596-073-56X", "confidenceLevel": 0.8, - "textsnippet": "have read the journal s policy and have the following conflicts PH and GPJMvdD have filed a patent application entitled Vaccine against amyloid folding intermediate EP08159385 .7 2008 WO2010002251 A1 2010 The RIVM is a Government" -} -{ - "documentId": "50|dedup_wf_001::dc4a71f4bd13a6423d69dc362adc5ba0", - "appln_nr": "11305584", - "confidenceLevel": 0.8, - "textsnippet": "WIBG within BGCN homolog Drosophila ZNF706 zinc finger protein 706 Conflicts of interest A patent was submitted by INSERM TRANSFERT in May 2011 submission number 11305584 Receiving Office European Patent Office The Hague Authors contributions IA" -} -{ - "documentId": "50|dedup_wf_001::22702d5c2030360cf41cf43798a02980", - "appln_nr": "97202904", - "confidenceLevel": 0.8, - "textsnippet": "of galvanic industries or at remote locations where energy and refinery metallurgists are at a premium A patent for this process was filed under number 97202904.5 2309 14 Acknowledgments The authors express their appreciation to G" -} -{ - "documentId": "50|od______1093::dd1fa9c8f4845773ca7ad9e06c50a110", - "appln_nr": "99480075", - "confidenceLevel": 0.8, - "textsnippet": "FR99 00485 EURECOM 09 PCT Mar 1999 39 J L Dugelay and C Rey Method of Marking a Multimedia Document Having Improved Robustness Pending Patent EUP99480075 .3 EURECOM 14 EP May 2001 40 Y Fisher Fractal" -} -{ - "documentId": "50|dedup_wf_001::2cd68a226c23900f926931f43d8cc4af", - "appln_nr": "9806838", - "confidenceLevel": 0.8, - "textsnippet": "transmembrane TRAP tartrate resistant acid phosphatase Competing interests RAWvL and JH are the inventors of a patent entitled Methods and means for modifying complement activation WO9806838 Neither author has received any financial reimbursement for this patent" + "textsnippet": "online http patentscope wipo int search en WO2005013941 accessed on 10" } { "documentId": "50|dedup_wf_001::4b7b9d8520c64fbafe3d3ba2883597e4", - "appln_nr": "2013058681", + "lensId": "024-040-013-839-479", "confidenceLevel": 0.8, - "textsnippet": "6 dioxo 2 3 6 7 tetrahydro 1H purine 8 yl fragment and constituting A 2A adenosine receptor antagonists and the use thereof European Patent WO2013058681 Koles L Furst S Illes P Purine ionotropic P2X receptors" + "textsnippet": "antagonists and the use thereof European Patent WO2013058681 Koles L Furst" } { - "documentId": "50|dedup_wf_001::e14edbd13c929f68c75ca321ff2f629a", - "appln_nr": "93430009", + "documentId": "50|dedup_wf_001::4b7b9d8520c64fbafe3d3ba2883597e4", + "lensId": "154-511-862-831-246", "confidenceLevel": 0.8, - "textsnippet": "A L Vallet J L 1993 Process for salting drying and smoking cold meat products and a device for carrying this out Patent No 93430009.6 filing date July 10 Dieuzeide R Novella M 1951 Essai" + "textsnippet": "antagonists and the use thereof European Patent WO2013058681 Koles L Furst" } \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/extracted_patent_metadata.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/extracted_patent_metadata.json deleted file mode 100644 index 7c7ba08a0..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/extracted_patent_metadata.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "appln_auth": "EP", - "appln_nr": "2013058681", - "appln_filing_date": "2000-01-06", - "appln_nr_epodoc": "WO2000EP00011", - "earliest_publn_date": "2000-01-01", - "appln_abstract": "This is abstract", - "appln_title": "The great invention", - "ipc_class_symbol": [ - "C08F2/20" - ], - "applicant_names": [ - "Jan Nowak" - ], - "applicant_country_codes": [ - "PL" - ] -} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/imported_patent.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/imported_patent.json deleted file mode 100644 index ab2c2da65..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/imported_patent.json +++ /dev/null @@ -1,14 +0,0 @@ -{"appln_auth": "EP", "appln_nr":"00103094", "publn_auth": "EP", "publn_nr": "1037159", "publn_kind": "A3"} -{"appln_auth": "EP", "appln_nr":"01119799", "publn_auth": "EP", "publn_nr": "1186609", "publn_kind": "A3"} -{"appln_auth": "EP", "appln_nr":"06021113", "publn_auth": "EP", "publn_nr": "1775372", "publn_kind": "B1"} -{"appln_auth": "EP", "appln_nr":"07121953", "publn_auth": "EP", "publn_nr": "1944442", "publn_kind": "B1"} -{"appln_auth": "EP", "appln_nr":"07021065", "publn_auth": "EP", "publn_nr": "1944445", "publn_kind": "A3"} -{"appln_auth": "EP", "appln_nr":"8100053", "publn_auth": "EP", "publn_nr": "1944446", "publn_kind": "A3"} -{"appln_auth": "EP", "appln_nr":"08159385", "publn_auth": "EP", "publn_nr": "0000000", "publn_kind": "A3"} -{"appln_auth": "EP", "appln_nr":"11305584", "publn_auth": "EP", "publn_nr": "0000000", "publn_kind": "A3"} -{"appln_auth": "EP", "appln_nr":"2005013941", "publn_auth": "WO", "publn_nr": "2007068278", "publn_kind": "A1"} -{"appln_auth": "EP", "appln_nr":"97202904", "publn_auth": "EP", "publn_nr": "0908525", "publn_kind": "A1"} -{"appln_auth": "EP", "appln_nr":"99480075", "publn_auth": "EP", "publn_nr": "1031944", "publn_kind": "A1"} -{"appln_auth": "EP", "appln_nr":"9806838", "publn_auth": "EP", "publn_nr": "2321860", "publn_kind": "B1"} -{"appln_auth": "EP", "appln_nr":"2013058681", "publn_auth": "WO", "publn_nr": "2013189646", "publn_kind": "A1"} -{"appln_auth": "EP", "appln_nr":"93430009", "publn_auth": "EP", "publn_nr": "0587515", "publn_kind": "B1"} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/input_transformer/imported_patent.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/input_transformer/imported_patent.json deleted file mode 100644 index ab2c2da65..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/input_transformer/imported_patent.json +++ /dev/null @@ -1,14 +0,0 @@ -{"appln_auth": "EP", "appln_nr":"00103094", "publn_auth": "EP", "publn_nr": "1037159", "publn_kind": "A3"} -{"appln_auth": "EP", "appln_nr":"01119799", "publn_auth": "EP", "publn_nr": "1186609", "publn_kind": "A3"} -{"appln_auth": "EP", "appln_nr":"06021113", "publn_auth": "EP", "publn_nr": "1775372", "publn_kind": "B1"} -{"appln_auth": "EP", "appln_nr":"07121953", "publn_auth": "EP", "publn_nr": "1944442", "publn_kind": "B1"} -{"appln_auth": "EP", "appln_nr":"07021065", "publn_auth": "EP", "publn_nr": "1944445", "publn_kind": "A3"} -{"appln_auth": "EP", "appln_nr":"8100053", "publn_auth": "EP", "publn_nr": "1944446", "publn_kind": "A3"} -{"appln_auth": "EP", "appln_nr":"08159385", "publn_auth": "EP", "publn_nr": "0000000", "publn_kind": "A3"} -{"appln_auth": "EP", "appln_nr":"11305584", "publn_auth": "EP", "publn_nr": "0000000", "publn_kind": "A3"} -{"appln_auth": "EP", "appln_nr":"2005013941", "publn_auth": "WO", "publn_nr": "2007068278", "publn_kind": "A1"} -{"appln_auth": "EP", "appln_nr":"97202904", "publn_auth": "EP", "publn_nr": "0908525", "publn_kind": "A1"} -{"appln_auth": "EP", "appln_nr":"99480075", "publn_auth": "EP", "publn_nr": "1031944", "publn_kind": "A1"} -{"appln_auth": "EP", "appln_nr":"9806838", "publn_auth": "EP", "publn_nr": "2321860", "publn_kind": "B1"} -{"appln_auth": "EP", "appln_nr":"2013058681", "publn_auth": "WO", "publn_nr": "2013189646", "publn_kind": "A1"} -{"appln_auth": "EP", "appln_nr":"93430009", "publn_auth": "EP", "publn_nr": "0587515", "publn_kind": "B1"} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/input_transformer/patent_transformed.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/input_transformer/patent_transformed.json deleted file mode 100644 index 446dfc60c..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/input_transformer/patent_transformed.json +++ /dev/null @@ -1,14 +0,0 @@ -{"appln_nr":"00103094"} -{"appln_nr":"01119799"} -{"appln_nr":"06021113"} -{"appln_nr":"07121953"} -{"appln_nr":"07021065"} -{"appln_nr":"8100053"} -{"appln_nr":"08159385"} -{"appln_nr":"11305584"} -{"appln_nr":"2005013941"} -{"appln_nr":"97202904"} -{"appln_nr":"99480075"} -{"appln_nr":"9806838"} -{"appln_nr":"2013058681"} -{"appln_nr":"93430009"} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/mock_facade_storage/WO.2013189646.A1.xml b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/mock_facade_storage/WO.2013189646.A1.xml deleted file mode 100644 index fd43faa3a..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/mock_facade_storage/WO.2013189646.A1.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - - - WO - 2013189646 - A1 - 20000101 - - - WO2013189646 - 20000120 - - - - C08F2/20 - - - - WO2000EP00011 - 20000106 - - - - - - - Jan Nowak [PL] - - - - - Jan Nowak - - - - - - - Jan Nowak [PL] - - - - - Jan Nowak - - - - - The great invention - - -

This is abstract

- - - - diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/raw_patent.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/raw_patent.json new file mode 100644 index 000000000..b15b6be6a --- /dev/null +++ b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/raw_patent.json @@ -0,0 +1,5 @@ +{"c1":"001-200-008-561-076","c2":"H0683302", "normal":"0683302", "c4":"JP", "c5":"B2"} +{"c1":"001-200-010-686-433","c2":"2001087790", "normal":"2001087790", "c4":"WO", "c5":"A1"} +{"c1":"024-040-013-839-479","c2":"2013058681", "normal":"2013058681", "c4":"WO", "c5":"A3"} +{"c1":"154-511-862-831-246","c2":"2013058681", "normal":"2013058681", "c4":"WO", "c5":"A2"} +{"c1":"190-148-596-073-56X","c2":"2005013941", "normal":"2005013941", "c4":"WO", "c5":"A1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/report_patent.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/report_patent.json new file mode 100644 index 000000000..6aaf5046c --- /dev/null +++ b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/report_patent.json @@ -0,0 +1 @@ +{"key":"processing.referenceExtraction.patent.references", "type":"COUNTER", "value": "3"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/report_patent_metadata_extraction.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/report_patent_metadata_extraction.json deleted file mode 100644 index fee88f9f7..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/report_patent_metadata_extraction.json +++ /dev/null @@ -1,2 +0,0 @@ -{"key":"processing.referenceExtraction.patent.metadataextraction.processed.total", "type":"COUNTER", "value": "1"} -{"key":"processing.referenceExtraction.patent.metadataextraction.processed.fault", "type":"COUNTER", "value": "0"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/report_patent_metadata_retrieval.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/report_patent_metadata_retrieval.json deleted file mode 100644 index 92f3acc49..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/report_patent_metadata_retrieval.json +++ /dev/null @@ -1,3 +0,0 @@ -{"key":"processing.referenceExtraction.patent.retrieval.fromCache.total", "type":"COUNTER", "value": "0"} -{"key":"processing.referenceExtraction.patent.retrieval.processed.total", "type":"COUNTER", "value": "7"} -{"key":"processing.referenceExtraction.patent.retrieval.processed.fault", "type":"COUNTER", "value": "6"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text1.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text1.json deleted file mode 100644 index 1dc05dbd0..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text1.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"00000001","text":"EP-00000001-EP-1118543-B1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text12.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text12.json deleted file mode 100644 index 1dc05dbd0..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text12.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"00000001","text":"EP-00000001-EP-1118543-B1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text2.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text2.json deleted file mode 100644 index dd0056e73..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/cache_text2.json +++ /dev/null @@ -1,2 +0,0 @@ -{"id":"00000001","text":"EP-00000001-EP-1118543-B1"} -{"id":"00000003","text":"EP-00000003-EP-1118533-B1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/input.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/input.json deleted file mode 100644 index 8f79b138a..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/input.json +++ /dev/null @@ -1,2 +0,0 @@ -{"appln_auth":"EP","appln_nr":"00000001","publn_auth":"EP","publn_nr":"1118543","publn_kind":"B1"} -{"appln_auth":"EP","appln_nr":"00000002","publn_auth":"EP","publn_nr":"non-existing","publn_kind":"B1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/input2.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/input2.json deleted file mode 100644 index 83c35d59b..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/input2.json +++ /dev/null @@ -1 +0,0 @@ -{"appln_auth":"EP","appln_nr":"00000003","publn_auth":"EP","publn_nr":"1118533","publn_kind":"B1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/output.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/output.json deleted file mode 100644 index 1dc05dbd0..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/output.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"00000001","text":"EP-00000001-EP-1118543-B1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/output2.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/output2.json deleted file mode 100644 index 67ad4a17b..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/output2.json +++ /dev/null @@ -1 +0,0 @@ -{"id":"00000003","text":"EP-00000003-EP-1118533-B1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report.json deleted file mode 100644 index 3b8dfa90a..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report.json +++ /dev/null @@ -1,3 +0,0 @@ -{"key":"processing.referenceExtraction.patent.retrieval.fromCache.total", "type":"COUNTER", "value": "0"} -{"key":"processing.referenceExtraction.patent.retrieval.processed.total", "type":"COUNTER", "value": "2"} -{"key":"processing.referenceExtraction.patent.retrieval.processed.fault", "type":"COUNTER", "value": "1"} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report_from_cache.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report_from_cache.json deleted file mode 100644 index 3cae5e65e..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report_from_cache.json +++ /dev/null @@ -1,3 +0,0 @@ -{"key":"processing.referenceExtraction.patent.retrieval.fromCache.total", "type":"COUNTER", "value": "2"} -{"key":"processing.referenceExtraction.patent.retrieval.processed.total", "type":"COUNTER", "value": "0"} -{"key":"processing.referenceExtraction.patent.retrieval.processed.fault", "type":"COUNTER", "value": "0"} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report_update.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report_update.json deleted file mode 100644 index 376a0f818..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/report_update.json +++ /dev/null @@ -1,3 +0,0 @@ -{"key":"processing.referenceExtraction.patent.retrieval.fromCache.total", "type":"COUNTER", "value": "0"} -{"key":"processing.referenceExtraction.patent.retrieval.processed.total", "type":"COUNTER", "value": "1"} -{"key":"processing.referenceExtraction.patent.retrieval.processed.fault", "type":"COUNTER", "value": "0"} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/transformer/input_imported_patent.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/transformer/input_imported_patent.json deleted file mode 100644 index 782a65969..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/transformer/input_imported_patent.json +++ /dev/null @@ -1,5 +0,0 @@ -{"appln_auth":"EP","appln_nr":"00000001","publn_auth":"EP","publn_nr":"1118543","publn_kind":"B1"} -{"appln_auth":"EP","appln_nr":"00000002","publn_auth":"EP","publn_nr":"1217010","publn_kind":"B1"} -{"appln_auth":"EP","appln_nr":"0000003","publn_auth":"WO","publn_nr":"0042078","publn_kind":"A1"} -{"appln_auth":"EP","appln_nr":"0000004","publn_auth":"WO","publn_nr":"0060207","publn_kind":"A1"} -{"appln_auth":"EP","appln_nr":"0000005","publn_auth":"WO","publn_nr":"0040431","publn_kind":"A1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/transformer/input_matched_patent.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/transformer/input_matched_patent.json deleted file mode 100644 index 46a5e9e6b..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/transformer/input_matched_patent.json +++ /dev/null @@ -1,2 +0,0 @@ -{"documentId":"doc-id1","appln_nr":"00000001","confidenceLevel":0.8,"textsnippet":null} -{"documentId":"doc-id2","appln_nr":"0000005","confidenceLevel":0.8,"textsnippet":null} diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/transformer/output.json b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/transformer/output.json deleted file mode 100644 index 7fb00fe27..000000000 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/data/retriever/transformer/output.json +++ /dev/null @@ -1,2 +0,0 @@ -{"appln_auth":"EP","appln_nr":"00000001","publn_auth":"EP","publn_nr":"1118543","publn_kind":"B1"} -{"appln_auth":"EP","appln_nr":"0000005","publn_auth":"WO","publn_nr":"0040431","publn_kind":"A1"} \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest/oozie_app/import.txt b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest/oozie_app/import.txt index 3bfcfe9ba..f0e026703 100644 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest/oozie_app/import.txt +++ b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest/oozie_app/import.txt @@ -1,2 +1,3 @@ ## This is a classpath-based import file (this header is required) referenceextraction_patent classpath eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app +sqlite_builder classpath eu/dnetlib/iis/wf/referenceextraction/patent/sqlite_builder/oozie_app \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest/oozie_app/workflow.xml b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest/oozie_app/workflow.xml index e73c12598..00e25c2e1 100644 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest/oozie_app/workflow.xml @@ -30,61 +30,68 @@ eu.dnetlib.iis.metadataextraction.schemas.DocumentText, eu/dnetlib/iis/wf/referenceextraction/patent/data/document_text.json} - -C{imported_patent, - eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent, - eu/dnetlib/iis/wf/referenceextraction/patent/data/imported_patent.json} + -C{raw_patent, + eu.dnetlib.iis.referenceextraction.patent.schemas.PatentReferenceExtractionInput, + eu/dnetlib/iis/wf/referenceextraction/patent/data/raw_patent.json} -Odocument_text=${workingDir}/producer/document_text - -Oimported_patent=${workingDir}/producer/imported_patent + -Oraw_patent=${workingDir}/producer/raw_patent - + - + - ${wf:appPath()}/referenceextraction_patent + ${wf:appPath()}/sqlite_builder workingDir - ${workingDir}/referenceextraction_patent/working_dir + ${workingDir}/sqlite_builder/working_dir - patentMetadataRetrieverFacadeFactoryClassname - eu.dnetlib.iis.wf.referenceextraction.patent.TestServiceFacadeFactories$FileContentReturningFacadeFactory + input_patent + ${workingDir}/producer/raw_patent - patentLockManagerFactoryClassName - eu.dnetlib.iis.common.lock.LockManagerFactoryMock + output_patent_db + ${workingDir}/sqlite_builder/lens.db + + + + + + + + + + ${wf:appPath()}/referenceextraction_patent + + + + workingDir + ${workingDir}/referenceextraction_patent/working_dir input_document_text ${workingDir}/producer/document_text - input_patent - ${workingDir}/producer/imported_patent + input_patent_db + ${workingDir}/sqlite_builder/lens.db output_document_to_patent ${workingDir}/referenceextraction_patent/output - - output_patent_metadata - ${workingDir}/referenceextraction_patent/output_patent_metadata - output_report_root_path ${workingDir}/referenceextraction_patent/report - cacheRootDir - ${workingDir}/referenceextraction_patent/working_dir - - - referenceextraction_patent_number_of_emitted_filesa - 1 + output_report_relative_path + document_to_patent @@ -100,22 +107,12 @@ eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent, eu/dnetlib/iis/wf/referenceextraction/patent/data/document_to_patent.json} - -C{patent_metadata, - eu.dnetlib.iis.referenceextraction.patent.schemas.Patent, - eu/dnetlib/iis/wf/referenceextraction/patent/data/extracted_patent_metadata.json} - - -C{report_patent_metadata_retrieval, - eu.dnetlib.iis.common.schemas.ReportEntry, - eu/dnetlib/iis/wf/referenceextraction/patent/data/report_patent_metadata_retrieval.json} - - -C{report_patent_metadata_extraction, + -C{report_patent, eu.dnetlib.iis.common.schemas.ReportEntry, - eu/dnetlib/iis/wf/referenceextraction/patent/data/report_patent_metadata_extraction.json} + eu/dnetlib/iis/wf/referenceextraction/patent/data/report_patent.json} -Idocument_to_patent=${workingDir}/referenceextraction_patent/output - -Ipatent_metadata=${workingDir}/referenceextraction_patent/output_patent_metadata - -Ireport_patent_metadata_retrieval=${workingDir}/referenceextraction_patent/report/patent_metadata_retrieval - -Ireport_patent_metadata_extraction=${workingDir}/referenceextraction_patent/report/patent_metadata_extraction + -Ireport_patent=${workingDir}/referenceextraction_patent/report/document_to_patent diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_patent_input/oozie_app/import.txt b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_patent_input/oozie_app/import.txt index 3bfcfe9ba..f0e026703 100644 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_patent_input/oozie_app/import.txt +++ b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_patent_input/oozie_app/import.txt @@ -1,2 +1,3 @@ ## This is a classpath-based import file (this header is required) referenceextraction_patent classpath eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app +sqlite_builder classpath eu/dnetlib/iis/wf/referenceextraction/patent/sqlite_builder/oozie_app \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_patent_input/oozie_app/workflow.xml b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_patent_input/oozie_app/workflow.xml index d1d3db0d2..bed339cb6 100644 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_patent_input/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_patent_input/oozie_app/workflow.xml @@ -30,61 +30,68 @@ eu.dnetlib.iis.metadataextraction.schemas.DocumentText, eu/dnetlib/iis/wf/referenceextraction/patent/data/document_text.json} - -C{imported_patent, - eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent, + -C{raw_patent, + eu.dnetlib.iis.referenceextraction.patent.schemas.PatentReferenceExtractionInput, eu/dnetlib/iis/common/data/empty.json} -Odocument_text=${workingDir}/producer/document_text - -Oimported_patent=${workingDir}/producer/imported_patent + -Oraw_patent=${workingDir}/producer/raw_patent - + - + - ${wf:appPath()}/referenceextraction_patent + ${wf:appPath()}/sqlite_builder workingDir - ${workingDir}/referenceextraction_patent/working_dir + ${workingDir}/sqlite_builder/working_dir + + + input_patent + ${workingDir}/producer/raw_patent - patentMetadataRetrieverFacadeFactoryClassname - eu.dnetlib.iis.wf.referenceextraction.patent.TestServiceFacadeFactories$FileContentReturningFacadeFactory + output_patent_db + ${workingDir}/sqlite_builder/lens.db + + + + + + + + + ${wf:appPath()}/referenceextraction_patent + + - patentLockManagerFactoryClassName - eu.dnetlib.iis.common.lock.LockManagerFactoryMock + workingDir + ${workingDir}/referenceextraction_patent/working_dir input_document_text ${workingDir}/producer/document_text - input_patent - ${workingDir}/producer/imported_patent + input_patent_db + ${workingDir}/sqlite_builder/lens.db output_document_to_patent ${workingDir}/referenceextraction_patent/output - - output_patent_metadata - ${workingDir}/referenceextraction_patent/output_patent_metadata - output_report_root_path ${workingDir}/referenceextraction_patent/report - cacheRootDir - ${workingDir}/referenceextraction_patent/working_dir - - - referenceextraction_patent_number_of_emitted_filesa - 1 + output_report_relative_path + document_to_patent @@ -100,12 +107,7 @@ eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent, eu/dnetlib/iis/common/data/empty.json} - -C{patent_metadata, - eu.dnetlib.iis.referenceextraction.patent.schemas.Patent, - eu/dnetlib/iis/common/data/empty.json} - -Idocument_to_patent=${workingDir}/referenceextraction_patent/output - -Ipatent_metadata=${workingDir}/referenceextraction_patent/output_patent_metadata diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_text_input/oozie_app/import.txt b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_text_input/oozie_app/import.txt index 3bfcfe9ba..f0e026703 100644 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_text_input/oozie_app/import.txt +++ b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_text_input/oozie_app/import.txt @@ -1,2 +1,3 @@ ## This is a classpath-based import file (this header is required) referenceextraction_patent classpath eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app +sqlite_builder classpath eu/dnetlib/iis/wf/referenceextraction/patent/sqlite_builder/oozie_app \ No newline at end of file diff --git a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_text_input/oozie_app/workflow.xml b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_text_input/oozie_app/workflow.xml index 241388dac..c693c5562 100644 --- a/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_text_input/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-referenceextraction/src/test/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/sampletest_empty_text_input/oozie_app/workflow.xml @@ -30,61 +30,68 @@ eu.dnetlib.iis.metadataextraction.schemas.DocumentText, eu/dnetlib/iis/common/data/empty.json} - -C{imported_patent, - eu.dnetlib.iis.referenceextraction.patent.schemas.ImportedPatent, - eu/dnetlib/iis/wf/referenceextraction/patent/data/imported_patent.json} + -C{raw_patent, + eu.dnetlib.iis.referenceextraction.patent.schemas.PatentReferenceExtractionInput, + eu/dnetlib/iis/wf/referenceextraction/patent/data/raw_patent.json} -Odocument_text=${workingDir}/producer/document_text - -Oimported_patent=${workingDir}/producer/imported_patent + -Oraw_patent=${workingDir}/producer/raw_patent - + - + - ${wf:appPath()}/referenceextraction_patent + ${wf:appPath()}/sqlite_builder workingDir - ${workingDir}/referenceextraction_patent/working_dir + ${workingDir}/sqlite_builder/working_dir + + + input_patent + ${workingDir}/producer/raw_patent - patentMetadataRetrieverFacadeFactoryClassname - eu.dnetlib.iis.wf.referenceextraction.patent.TestServiceFacadeFactories$FileContentReturningFacadeFactory + output_patent_db + ${workingDir}/sqlite_builder/lens.db + + + + + + + + + ${wf:appPath()}/referenceextraction_patent + + - patentLockManagerFactoryClassName - eu.dnetlib.iis.common.lock.LockManagerFactoryMock + workingDir + ${workingDir}/referenceextraction_patent/working_dir input_document_text ${workingDir}/producer/document_text - input_patent - ${workingDir}/producer/imported_patent + input_patent_db + ${workingDir}/sqlite_builder/lens.db output_document_to_patent ${workingDir}/referenceextraction_patent/output - - output_patent_metadata - ${workingDir}/referenceextraction_patent/output_patent_metadata - output_report_root_path ${workingDir}/referenceextraction_patent/report - cacheRootDir - ${workingDir}/referenceextraction_patent/working_dir - - - referenceextraction_patent_number_of_emitted_filesa - 1 + output_report_relative_path + document_to_patent @@ -100,12 +107,7 @@ eu.dnetlib.iis.referenceextraction.patent.schemas.DocumentToPatent, eu/dnetlib/iis/common/data/empty.json} - -C{patent_metadata, - eu.dnetlib.iis.referenceextraction.patent.schemas.Patent, - eu/dnetlib/iis/common/data/empty.json} - -Idocument_to_patent=${workingDir}/referenceextraction_patent/output - -Ipatent_metadata=${workingDir}/referenceextraction_patent/output_patent_metadata