From 4654ec85a43297c034e7e69fdd9f719e89429124 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 14 Jun 2026 12:39:01 +0200 Subject: [PATCH 1/2] #78 - Clean up NER types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Drop 27 marker subtypes from NamedEntity type system (Person, Organization, Location, Date, Time, Money, Percent, Quantity, Ordinal, Cardinal, WorkOfArt, Animal, Plant, Substance, Disease, Event, Law, Language, ContactInfo, Game, Fac, FacDesc, Gpe, GpeDesc, Norp, OrgDesc, PerDesc, Product, ProductDesc, Nationality) — keep only `NamedEntity` with its `value` + `identifier` features - Delete 11 per-model CoreNLP NER tag-to-type `.map` files; `MappingProvider` falls through to `BASE_TYPE=NamedEntity` and records the source tag in `value` - Keep the generic NER mapping mechanism (`MappingProviderFactory.createNerMappingProvider`, `mappingLocation` parameter, `ner-default-variants.map`) so users can still supply custom `.map` files - Rewrite `DKPro2Gate` to route GATE annotation types (`Person`/`Location`/`Organization`/`NamedEntity`) off `NamedEntity.value` instead of `instanceof` on subtypes; drop the now-redundant `dkproType` GATE feature - Collapse the two GATE writer NER tests into one (`oneWayNamedEntity`); delete obsolete `ner2002_conll.map` + `ner2002_ref_specific.xml`; rename `ner2002_ref_generic.xml` → `ner2002_ref.xml` and update to expected GATE types - Update brat reader/writer tests and `.ann` reference fixtures to use `NamedEntity` instead of `Person`/`Location`/`Organization` subtypes - Update brat reader Javadoc example to reference `NamedEntity` instead of `Location` - Update CoreNLP NER test assertions from `Person(...)`/`Location(...)`/`Organization(...)` to `NamedEntity(...)` - Strip 30 deleted subtype entries from `typesystemmapping.yaml`; update brat documentation example to map `(LOC|PER|ORG)` → `NamedEntity` --- .githooks/pre-commit | 2 +- .../main/resources/desc/type/NamedEntity.xml | 150 -------- .../corenlp/lib/ner-de-dewac_175m_600.crf.map | 9 - .../ner-de-germeval2014.hgc_175m_600.crf.map | 5 - .../corenlp/lib/ner-de-hgc_175m_600.crf.map | 9 - ...ner-en-all.3class.caseless.distsim.crf.map | 5 - .../lib/ner-en-all.3class.distsim.crf.map | 4 - ...r-en-conll.4class.caseless.distsim.crf.map | 5 - .../lib/ner-en-conll.4class.distsim.crf.map | 5 - ...ner-en-muc.7class.caseless.distsim.crf.map | 9 - .../lib/ner-en-muc.7class.distsim.crf.map | 9 - ...-en-nowiki.3class.caseless.distsim.crf.map | 4 - .../lib/ner-es-ancora.distsim.s512.crf.map | 6 - .../CoreNlpNamedEntityRecognizerTest.java | 42 +-- ...PosTaggerAndNamedEntityRecognizerTest.java | 10 +- .../format-reference/sectionIntroBrat.adoc | 10 +- .../script/mappings/typesystemmapping.yaml | 113 ------- .../org/dkpro/core/io/brat/BratReader.java | 2 +- .../core/io/brat/BratReaderWriterTest.java | 26 +- .../resources/brat/document1-ref-mapping.ann | 15 +- .../src/test/resources/brat/document1-ref.ann | 10 +- .../core/io/gate/internal/DKPro2Gate.java | 25 +- .../dkpro/core/io/gate/GateXmlWriterTest.java | 20 +- .../resources/conll/2002/ner2002_conll.map | 4 - ...r2002_ref_specific.xml => ner2002_ref.xml} | 40 +-- .../conll/2002/ner2002_ref_generic.xml | 320 ------------------ 26 files changed, 78 insertions(+), 781 deletions(-) delete mode 100644 dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-de-dewac_175m_600.crf.map delete mode 100644 dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-de-germeval2014.hgc_175m_600.crf.map delete mode 100644 dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-de-hgc_175m_600.crf.map delete mode 100644 dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-all.3class.caseless.distsim.crf.map delete mode 100644 dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-all.3class.distsim.crf.map delete mode 100644 dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-conll.4class.caseless.distsim.crf.map delete mode 100644 dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-conll.4class.distsim.crf.map delete mode 100644 dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-muc.7class.caseless.distsim.crf.map delete mode 100644 dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-muc.7class.distsim.crf.map delete mode 100644 dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-nowiki.3class.caseless.distsim.crf.map delete mode 100644 dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-es-ancora.distsim.s512.crf.map delete mode 100644 dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_conll.map rename dkpro-core-io-gate-asl/src/test/resources/conll/2002/{ner2002_ref_specific.xml => ner2002_ref.xml} (93%) delete mode 100644 dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_ref_generic.xml diff --git a/.githooks/pre-commit b/.githooks/pre-commit index eba5d14be2..b8931505f2 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -26,7 +26,7 @@ printf '%s\n' "$MODULES" | tr ',' '\n' | sed 's/^/ - /' REPO_ROOT=$(git rev-parse --show-toplevel) cd "$REPO_ROOT" -if ! mvn -q -pl "$MODULES" -Pspotless,checkstyle spotless:check checkstyle:check; then +if ! mvn -q -pl "$MODULES" -Pspotless,checkstyle,experimental spotless:check checkstyle:check; then cat >&2 <<'EOF' [pre-commit] Verification failed. diff --git a/dkpro-core-api-ner-asl/src/main/resources/desc/type/NamedEntity.xml b/dkpro-core-api-ner-asl/src/main/resources/desc/type/NamedEntity.xml index 2715d3f0f8..ce4dc64ca3 100644 --- a/dkpro-core-api-ner-asl/src/main/resources/desc/type/NamedEntity.xml +++ b/dkpro-core-api-ner-asl/src/main/resources/desc/type/NamedEntity.xml @@ -22,155 +22,5 @@ - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.PerDesc - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Norp - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.OrgDesc - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Gpe - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.GpeDesc - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Product - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.ProductDesc - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Date - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Time - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Percent - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Money - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Quantity - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Ordinal - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Cardinal - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Event - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Plant - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Animal - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Substance - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Disease - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.WorkOfArt - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Law - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Language - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.ContactInfo - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Game - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Fac - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.FacDesc - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.Nationality - - de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity - diff --git a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-de-dewac_175m_600.crf.map b/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-de-dewac_175m_600.crf.map deleted file mode 100644 index fb61cad157..0000000000 --- a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-de-dewac_175m_600.crf.map +++ /dev/null @@ -1,9 +0,0 @@ -#O -I-PER=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person -I-LOC=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -#I-MISC -I-ORG=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -#B-MISC -B-LOC=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -B-ORG=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -*=de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity diff --git a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-de-germeval2014.hgc_175m_600.crf.map b/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-de-germeval2014.hgc_175m_600.crf.map deleted file mode 100644 index e7c53d13de..0000000000 --- a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-de-germeval2014.hgc_175m_600.crf.map +++ /dev/null @@ -1,5 +0,0 @@ -PERSON=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person -LOCATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -ORGANIZATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -#MISC -*=de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity diff --git a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-de-hgc_175m_600.crf.map b/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-de-hgc_175m_600.crf.map deleted file mode 100644 index fb61cad157..0000000000 --- a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-de-hgc_175m_600.crf.map +++ /dev/null @@ -1,9 +0,0 @@ -#O -I-PER=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person -I-LOC=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -#I-MISC -I-ORG=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -#B-MISC -B-LOC=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -B-ORG=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -*=de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity diff --git a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-all.3class.caseless.distsim.crf.map b/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-all.3class.caseless.distsim.crf.map deleted file mode 100644 index eacb379ec6..0000000000 --- a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-all.3class.caseless.distsim.crf.map +++ /dev/null @@ -1,5 +0,0 @@ -ORGANIZATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -PERSON=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person -LOCATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -#MISC -*=de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity \ No newline at end of file diff --git a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-all.3class.distsim.crf.map b/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-all.3class.distsim.crf.map deleted file mode 100644 index f509828f1d..0000000000 --- a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-all.3class.distsim.crf.map +++ /dev/null @@ -1,4 +0,0 @@ -ORGANIZATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -PERSON=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person -LOCATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -*=de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity diff --git a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-conll.4class.caseless.distsim.crf.map b/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-conll.4class.caseless.distsim.crf.map deleted file mode 100644 index eacb379ec6..0000000000 --- a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-conll.4class.caseless.distsim.crf.map +++ /dev/null @@ -1,5 +0,0 @@ -ORGANIZATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -PERSON=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person -LOCATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -#MISC -*=de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity \ No newline at end of file diff --git a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-conll.4class.distsim.crf.map b/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-conll.4class.distsim.crf.map deleted file mode 100644 index eacb379ec6..0000000000 --- a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-conll.4class.distsim.crf.map +++ /dev/null @@ -1,5 +0,0 @@ -ORGANIZATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -PERSON=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person -LOCATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -#MISC -*=de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity \ No newline at end of file diff --git a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-muc.7class.caseless.distsim.crf.map b/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-muc.7class.caseless.distsim.crf.map deleted file mode 100644 index f094674886..0000000000 --- a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-muc.7class.caseless.distsim.crf.map +++ /dev/null @@ -1,9 +0,0 @@ -ORGANIZATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -PERSON=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person -LOCATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -DATE=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Date -MONEY=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Money -TIME=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Time -PERCENT=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Percent -#O -*=de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity \ No newline at end of file diff --git a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-muc.7class.distsim.crf.map b/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-muc.7class.distsim.crf.map deleted file mode 100644 index f094674886..0000000000 --- a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-muc.7class.distsim.crf.map +++ /dev/null @@ -1,9 +0,0 @@ -ORGANIZATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -PERSON=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person -LOCATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -DATE=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Date -MONEY=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Money -TIME=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Time -PERCENT=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Percent -#O -*=de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity \ No newline at end of file diff --git a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-nowiki.3class.caseless.distsim.crf.map b/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-nowiki.3class.caseless.distsim.crf.map deleted file mode 100644 index f509828f1d..0000000000 --- a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-en-nowiki.3class.caseless.distsim.crf.map +++ /dev/null @@ -1,4 +0,0 @@ -ORGANIZATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -PERSON=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person -LOCATION=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -*=de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity diff --git a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-es-ancora.distsim.s512.crf.map b/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-es-ancora.distsim.s512.crf.map deleted file mode 100644 index b142755dbc..0000000000 --- a/dkpro-core-corenlp-gpl/src/main/resources/org/dkpro/core/corenlp/lib/ner-es-ancora.distsim.s512.crf.map +++ /dev/null @@ -1,6 +0,0 @@ -LUG=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -#O -ORG=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -#OTROS -PERS=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person -*=de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity diff --git a/dkpro-core-corenlp-gpl/src/test/java/org/dkpro/core/corenlp/CoreNlpNamedEntityRecognizerTest.java b/dkpro-core-corenlp-gpl/src/test/java/org/dkpro/core/corenlp/CoreNlpNamedEntityRecognizerTest.java index e258ed3922..0a24bf0489 100644 --- a/dkpro-core-corenlp-gpl/src/test/java/org/dkpro/core/corenlp/CoreNlpNamedEntityRecognizerTest.java +++ b/dkpro-core-corenlp-gpl/src/test/java/org/dkpro/core/corenlp/CoreNlpNamedEntityRecognizerTest.java @@ -42,8 +42,8 @@ public void testEnglish() throws Exception JCas jcas = runTest("en", null, "IBM where John works is in Germany ."); - String[] ne = { "[ 0, 3]Organization(ORGANIZATION) (IBM)", - "[ 10, 14]Person(PERSON) (John)", "[ 27, 34]Location(LOCATION) (Germany)" }; + String[] ne = { "[ 0, 3]NamedEntity(ORGANIZATION) (IBM)", + "[ 10, 14]NamedEntity(PERSON) (John)", "[ 27, 34]NamedEntity(LOCATION) (Germany)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); } @@ -56,8 +56,8 @@ public void test3classCaselessEnglish() throws Exception JCas jcas = runTest("en", "all.3class.caseless.distsim.crf", "ibm where john works is in germany ."); - String[] ne = { "[ 0, 3]Organization(ORGANIZATION) (ibm)", - "[ 10, 14]Person(PERSON) (john)", "[ 27, 34]Location(LOCATION) (germany)" }; + String[] ne = { "[ 0, 3]NamedEntity(ORGANIZATION) (ibm)", + "[ 10, 14]NamedEntity(PERSON) (john)", "[ 27, 34]NamedEntity(LOCATION) (germany)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); } @@ -70,8 +70,8 @@ public void testNoWiki3classCaselessEnglish() throws Exception JCas jcas = runTest("en", "nowiki.3class.caseless.distsim.crf", "ibm where john works is in germany ."); - String[] ne = { "[ 0, 3]Organization(ORGANIZATION) (ibm)", - "[ 10, 14]Person(PERSON) (john)", "[ 27, 34]Location(LOCATION) (germany)" }; + String[] ne = { "[ 0, 3]NamedEntity(ORGANIZATION) (ibm)", + "[ 10, 14]NamedEntity(PERSON) (john)", "[ 27, 34]NamedEntity(LOCATION) (germany)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); } @@ -84,8 +84,8 @@ public void test4classEnglish() throws Exception JCas jcas = runTest("en", "conll.4class.distsim.crf", "IBM where John works is in Germany ."); - String[] ne = { "[ 0, 3]Organization(ORGANIZATION) (IBM)", - "[ 10, 14]Person(PERSON) (John)", "[ 27, 34]Location(LOCATION) (Germany)" }; + String[] ne = { "[ 0, 3]NamedEntity(ORGANIZATION) (IBM)", + "[ 10, 14]NamedEntity(PERSON) (John)", "[ 27, 34]NamedEntity(LOCATION) (Germany)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); } @@ -98,8 +98,8 @@ public void test4classCaselessEnglish() throws Exception JCas jcas = runTest("en", "conll.4class.caseless.distsim.crf", "ibm where john works is in germany ."); - String[] ne = { "[ 0, 3]Organization(ORGANIZATION) (ibm)", - "[ 10, 14]Person(PERSON) (john)", "[ 27, 34]Location(LOCATION) (germany)" }; + String[] ne = { "[ 0, 3]NamedEntity(ORGANIZATION) (ibm)", + "[ 10, 14]NamedEntity(PERSON) (john)", "[ 27, 34]NamedEntity(LOCATION) (germany)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); } @@ -112,8 +112,8 @@ public void test4classCaselessMixedEnglish() throws Exception JCas jcas = runTest("en", "conll.4class.caseless.distsim.crf", "IBM where john works is in Germany ."); - String[] ne = { "[ 0, 3]Organization(ORGANIZATION) (IBM)", - "[ 10, 14]Person(PERSON) (john)", "[ 27, 34]Location(LOCATION) (Germany)" }; + String[] ne = { "[ 0, 3]NamedEntity(ORGANIZATION) (IBM)", + "[ 10, 14]NamedEntity(PERSON) (john)", "[ 27, 34]NamedEntity(LOCATION) (Germany)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); } @@ -125,8 +125,8 @@ public void test7classEnglish() throws Exception JCas jcas = runTest("en", "muc.7class.distsim.crf", "IBM where John works is in Germany ."); - String[] ne = { "[ 0, 3]Organization(ORGANIZATION) (IBM)", - "[ 10, 14]Person(PERSON) (John)", "[ 27, 34]Location(LOCATION) (Germany)" }; + String[] ne = { "[ 0, 3]NamedEntity(ORGANIZATION) (IBM)", + "[ 10, 14]NamedEntity(PERSON) (John)", "[ 27, 34]NamedEntity(LOCATION) (Germany)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); } @@ -138,9 +138,9 @@ public void testGerman() throws Exception JCas jcas = runTest("de", null, "Markus arbeitet seit 10 Jahren bei SAP in Deutschland ."); - String[] ne = { "[ 0, 6]Person(PERSON) (Markus)", - "[ 35, 38]Organization(ORGANIZATION) (SAP)", - "[ 42, 53]Location(LOCATION) (Deutschland)" }; + String[] ne = { "[ 0, 6]NamedEntity(PERSON) (Markus)", + "[ 35, 38]NamedEntity(ORGANIZATION) (SAP)", + "[ 42, 53]NamedEntity(LOCATION) (Deutschland)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); } @@ -153,8 +153,8 @@ public void testHgcGerman() throws Exception JCas jcas = runTest("de", "hgc_175m_600.crf", "Markus arbeitet seit 10 Jahren bei SAP in Deutschland ."); - String[] ne = { "[ 0, 6]Person(I-PER) (Markus)", "[ 35, 38]Organization(I-ORG) (SAP)", - "[ 42, 53]Location(I-LOC) (Deutschland)" }; + String[] ne = { "[ 0, 6]NamedEntity(I-PER) (Markus)", "[ 35, 38]NamedEntity(I-ORG) (SAP)", + "[ 42, 53]NamedEntity(I-LOC) (Deutschland)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); } @@ -166,8 +166,8 @@ public void testSpanish() throws Exception JCas jcas = runTest("es", null, "Hace 10 años Markus trabaja en SAP en Alemania ."); - String[] ne = { "[ 13, 19]Person(PERS) (Markus)", "[ 31, 34]Organization(ORG) (SAP)", - "[ 38, 46]Location(LUG) (Alemania)" }; + String[] ne = { "[ 13, 19]NamedEntity(PERS) (Markus)", "[ 31, 34]NamedEntity(ORG) (SAP)", + "[ 38, 46]NamedEntity(LUG) (Alemania)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); } diff --git a/dkpro-core-corenlp-gpl/src/test/java/org/dkpro/core/corenlp/CoreNlpPosTaggerAndNamedEntityRecognizerTest.java b/dkpro-core-corenlp-gpl/src/test/java/org/dkpro/core/corenlp/CoreNlpPosTaggerAndNamedEntityRecognizerTest.java index 2b49bd9793..ef88edecee 100644 --- a/dkpro-core-corenlp-gpl/src/test/java/org/dkpro/core/corenlp/CoreNlpPosTaggerAndNamedEntityRecognizerTest.java +++ b/dkpro-core-corenlp-gpl/src/test/java/org/dkpro/core/corenlp/CoreNlpPosTaggerAndNamedEntityRecognizerTest.java @@ -37,8 +37,8 @@ public void thatDurationIsRecognized() throws Exception { JCas jcas = runTest("en", "John lives for 200 years ."); - String[] ne = { "[ 0, 4]Person(PERSON) (John)", "[ 15, 18]NamedEntity(DURATION) (200)", - "[ 19, 24]NamedEntity(DURATION) (years)" }; + String[] ne = { "[ 0, 4]NamedEntity(PERSON) (John)", + "[ 15, 18]NamedEntity(DURATION) (200)", "[ 19, 24]NamedEntity(DURATION) (years)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); } @@ -48,7 +48,7 @@ public void thatMoneyIsRecognized() throws Exception { JCas jcas = runTest("en", "John buys a laptop for 200 dollars ."); - String[] ne = { "[ 0, 4]Person(PERSON) (John)", "[ 23, 26]NamedEntity(MONEY) (200)", + String[] ne = { "[ 0, 4]NamedEntity(PERSON) (John)", "[ 23, 26]NamedEntity(MONEY) (200)", "[ 27, 34]NamedEntity(MONEY) (dollars)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); @@ -59,7 +59,7 @@ public void thatOrdinalNumbersAreRecognized() throws Exception { JCas jcas = runTest("en", "John made the second place in the run ."); - String[] ne = { "[ 0, 4]Person(PERSON) (John)", + String[] ne = { "[ 0, 4]NamedEntity(PERSON) (John)", "[ 14, 20]NamedEntity(ORDINAL) (second)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); @@ -70,7 +70,7 @@ public void thatCardinalNumbersAreRecognized() throws Exception { JCas jcas = runTest("en", "John bought one hundred laptops ."); - String[] ne = { "[ 0, 4]Person(PERSON) (John)", "[ 12, 15]NamedEntity(NUMBER) (one)", + String[] ne = { "[ 0, 4]NamedEntity(PERSON) (John)", "[ 12, 15]NamedEntity(NUMBER) (one)", "[ 16, 23]NamedEntity(NUMBER) (hundred)" }; AssertAnnotations.assertNamedEntity(ne, select(jcas, NamedEntity.class)); diff --git a/dkpro-core-doc/src/main/asciidoc/format-reference/sectionIntroBrat.adoc b/dkpro-core-doc/src/main/asciidoc/format-reference/sectionIntroBrat.adoc index f5abb7717a..0f1e23f164 100644 --- a/dkpro-core-doc/src/main/asciidoc/format-reference/sectionIntroBrat.adoc +++ b/dkpro-core-doc/src/main/asciidoc/format-reference/sectionIntroBrat.adoc @@ -63,7 +63,7 @@ features on the target UIMA types. The DKPro Core `BratReader` tries its best to map a given brat file into the UIMA type system of the CAS it is given. Thus, the `BratReader` is not bound strictly to the pre-defined DKPro Core types, but supports any custom types as well. Since the type names in UIMA are typically long (e.g. -`de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location`) and the names used in brat tend to be short +`de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity`) and the names used in brat tend to be short (e.g. `LOC`), an explicit mapping is usually required. This mapping can be provided as JSON which needs to be passed to the `PARAM_MAPPINGS` parameter of the `BratReader`. Note that the parameter takes actual JSON, not the path to a JSON file. @@ -102,12 +102,8 @@ For the purpose of mapping, brat event (`E`) and text (`T`) annotations are both { 'textTypeMapppings': [ { - 'from': 'LOC', - 'to': 'de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location' - }, - { - 'from': 'PER', - 'to': 'de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person' + 'from': '(LOC|PER|ORG)', + 'to': 'de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity' }, ... ], diff --git a/dkpro-core-doc/src/main/script/mappings/typesystemmapping.yaml b/dkpro-core-doc/src/main/script/mappings/typesystemmapping.yaml index 611db4e94d..9e3243463d 100644 --- a/dkpro-core-doc/src/main/script/mappings/typesystemmapping.yaml +++ b/dkpro-core-doc/src/main/script/mappings/typesystemmapping.yaml @@ -288,62 +288,6 @@ de.tudarmstadt.ukp.dkpro.core.api.metadata.type.TagsetDescription: tags: externalReferences: [] externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Animal: - features: {} - externalReferences: - - {source: NERD, id: 'http://nerd.eurecom.fr/ontology#Animal', rel: similar} -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Cardinal: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.ContactInfo: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Date: - features: {} - externalReferences: - - {source: LAPPS, id: 'http://vocab.lappsgrid.org/Date', rel: similar} - - {source: ISOcat, id: 'http://www.isocat.org/datcat/DC-6123', rel: similar} - - {source: 'schema.org', id: 'http://schema.org/Date', rel: similar} - - {source: OMTD-SHARE, id: 'http://w3id.org/meta-share/omtd-share/Date', rel: similar} -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Disease: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Event: - features: {} - externalReferences: - - {source: NERD, id: 'http://nerd.eurecom.fr/ontology#Event', rel: similar} -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Fac: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.FacDesc: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Game: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Gpe: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.GpeDesc: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Language: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Law: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location: - features: {} - externalReferences: - - {source: LAPPS, id: 'http://vocab.lappsgrid.org/Location', rel: similar} - - {source: ISOcat, id: 'http://www.isocat.org/datcat/DC-4339', rel: similar} - - {source: 'schema.org', id: 'http://schema.org/Location', rel: similar} - - {source: NERD, id: 'http://nerd.eurecom.fr/ontology#Location', rel: similar} - - {source: OMTD-SHARE, id: 'http://w3id.org/meta-share/omtd-share/Location', rel: similar} -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Money: - features: {} - externalReferences: [] de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity: features: value: @@ -358,63 +302,6 @@ de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity: - {source: OWL, id: 'http://www.w3.org/2002/07/owl#Thing', rel: similar} - {source: NIF, id: 'http://persistence.uni-leipzig.org/nlp2rdf/ontologies/nif-core#EntityOccurrence', rel: similar} - {source: OMTD-SHARE, id: 'http://w3id.org/meta-share/omtd-share/NamedEntity', rel: similar} -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Nationality: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Norp: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Ordinal: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.OrgDesc: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization: - features: {} - externalReferences: - - {source: LAPPS, id: 'http://vocab.lappsgrid.org/Organization', rel: similar} - - {source: ISOcat, id: 'http://www.isocat.org/datcat/DC-2979', rel: similar} - - {source: 'schema.org', id: 'http://schema.org/Organization', rel: similar} - - {source: NERD, id: 'http://nerd.eurecom.fr/ontology#Organization', rel: similar} - - {source: OMTD-SHARE, id: 'http://w3id.org/meta-share/omtd-share/Organization', rel: similar} -de.tudarmstadt.ukp.dkpro.core.api.ner.type.PerDesc: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Percent: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person: - features: {} - externalReferences: - - {source: LAPPS, id: 'http://vocab.lappsgrid.org/Person', rel: similar} - - {source: ISOcat, id: 'http://www.isocat.org/datcat/DC-2978', rel: similar} - - {source: 'schema.org', id: 'http://schema.org/Person', rel: similar} - - {source: NERD, id: 'http://nerd.eurecom.fr/ontology#Location', rel: similar} - - {source: OMTD-SHARE, id: 'http://w3id.org/meta-share/omtd-share/Person', rel: similar} -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Plant: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Product: - features: {} - externalReferences: - - {source: NERD, id: 'http://nerd.eurecom.fr/ontology#Product', rel: similar} -de.tudarmstadt.ukp.dkpro.core.api.ner.type.ProductDesc: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Quantity: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Substance: - features: {} - externalReferences: [] -de.tudarmstadt.ukp.dkpro.core.api.ner.type.Time: - features: {} - externalReferences: - - {source: NERD, id: 'http://nerd.eurecom.fr/ontology#Time', rel: similar} -de.tudarmstadt.ukp.dkpro.core.api.ner.type.WorkOfArt: - features: {} - externalReferences: [] de.tudarmstadt.ukp.dkpro.core.api.phonetics.type.PhoneticTranscription: features: transcription: diff --git a/dkpro-core-io-brat-asl/src/main/java/org/dkpro/core/io/brat/BratReader.java b/dkpro-core-io-brat-asl/src/main/java/org/dkpro/core/io/brat/BratReader.java index 2cbc6a32ff..bd8ca7c31f 100644 --- a/dkpro-core-io-brat-asl/src/main/java/org/dkpro/core/io/brat/BratReader.java +++ b/dkpro-core-io-brat-asl/src/main/java/org/dkpro/core/io/brat/BratReader.java @@ -133,7 +133,7 @@ public class BratReader * * 
      * 
-     * Country -> de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location
+     * Country -> de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity
      * 
      *
*/ diff --git a/dkpro-core-io-brat-asl/src/test/java/org/dkpro/core/io/brat/BratReaderWriterTest.java b/dkpro-core-io-brat-asl/src/test/java/org/dkpro/core/io/brat/BratReaderWriterTest.java index e6ab3bd3f5..dfa3d069b0 100644 --- a/dkpro-core-io-brat-asl/src/test/java/org/dkpro/core/io/brat/BratReaderWriterTest.java +++ b/dkpro-core-io-brat-asl/src/test/java/org/dkpro/core/io/brat/BratReaderWriterTest.java @@ -113,9 +113,7 @@ public void testWithShortNames() throws Exception { testRoundTrip(createReaderDescription(BratReader.class, BratReader.PARAM_TEXT_ANNOTATION_TYPE_MAPPINGS, - asList("Token -> de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token", - "Organization -> de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization", - "Location -> de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location")), + asList("Token -> de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token")), createEngineDescription(BratWriter.class, BratWriter.PARAM_ENABLE_TYPE_MAPPINGS, true), "brat/document0a.ann"); @@ -135,15 +133,15 @@ public void test1legacy() throws Exception { testOneWay(createReaderDescription(BratReader.class, BratReader.PARAM_TEXT_ANNOTATION_TYPE_MAPPINGS, - asList("Country -> de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location", - "Organization -> de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization", + asList("Country -> de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity", + "Organization -> de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity", "MERGE-ORG -> de.tudarmstadt.ukp.dkpro.core.io.brat.type.MergeOrg"), BratReader.PARAM_RELATION_TYPE_MAPPINGS, asList("Origin -> de.tudarmstadt.ukp.dkpro.core.io.brat.type.AnnotationRelation"), BratReader.PARAM_RELATION_TYPES, asList("de.tudarmstadt.ukp.dkpro.core.io.brat.type.AnnotationRelation:source:target{A}:value"), BratReader.PARAM_NOTE_MAPPINGS, - asList("de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization:value", + asList("de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity:value", "de.tudarmstadt.ukp.dkpro.core.io.brat.type.AnnotationRelation:comment", "de.tudarmstadt.ukp.dkpro.core.io.brat.type.MergeOrg:comment")), createEngineDescription(BratWriter.class, BratWriter.PARAM_RELATION_TYPES, asList( @@ -156,21 +154,21 @@ public void test1mapping() throws Exception { String mapping = String.join("\n", "{", " 'textTypeMapppings': [", " {", " 'from': 'Country',", - " 'to': 'de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location'", " },", + " 'to': 'de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity'", " },", " {", " 'from': 'Organization',", - " 'to': 'de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization'", " },", + " 'to': 'de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity'", " },", " {", " 'from': 'MERGE-ORG',", " 'to': 'de.tudarmstadt.ukp.dkpro.core.io.brat.type.MergeOrg'", " }", " ],", " 'relationTypeMapppings': [", " {", " 'from': 'Origin',", " 'to': 'de.tudarmstadt.ukp.dkpro.core.io.brat.type.AnnotationRelation'", " }", " ],", " 'spans': [", " {", - " 'type': 'de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location',", + " 'type': 'de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity',", " 'defaultFeatureValues': {", " 'value': 'LOC'", " }", " }", " ],", " 'relations': [", " {", " 'type': 'de.tudarmstadt.ukp.dkpro.core.io.brat.type.AnnotationRelation',", " 'arg1': 'source',", " 'arg2': 'target',", " 'flags2': 'A',", " 'subCatFeature': 'value'", " }", " ],", " 'comments': [", " {", - " 'type': 'de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization',", + " 'type': 'de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity',", " 'feature': 'value'", " },", " {", " 'type': 'de.tudarmstadt.ukp.dkpro.core.io.brat.type.AnnotationRelation',", " 'feature': 'comment'", " },", " {", @@ -207,9 +205,7 @@ public void testBratWithDiscontinuousFragmentNear(@TempDir File tempDir) throws ReaderAssert.assertThat( // BratReader.class, // BratReader.PARAM_TEXT_ANNOTATION_TYPE_MAPPINGS, - asList("Token -> de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token", - "Organization -> de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization", - "Location -> de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location")) + asList("Token -> de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token")) .readingFrom("src/test/resources/brat/document0c.ann") // .usingWriter( // BratWriter.class, // @@ -225,9 +221,7 @@ public void testBratWithDiscontinuousFragmentFar() throws Exception { testOneWay(createReaderDescription(BratReader.class, BratReader.PARAM_TEXT_ANNOTATION_TYPE_MAPPINGS, - asList("Token -> de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token", - "Organization -> de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization", - "Location -> de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location")), + asList("Token -> de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token")), createEngineDescription(BratWriter.class, BratWriter.PARAM_ENABLE_TYPE_MAPPINGS, true), "brat/document0d-ref.ann", "brat/document0d.ann"); diff --git a/dkpro-core-io-brat-asl/src/test/resources/brat/document1-ref-mapping.ann b/dkpro-core-io-brat-asl/src/test/resources/brat/document1-ref-mapping.ann index 93f2204d58..81f099bda0 100644 --- a/dkpro-core-io-brat-asl/src/test/resources/brat/document1-ref-mapping.ann +++ b/dkpro-core-io-brat-asl/src/test/resources/brat/document1-ref-mapping.ann @@ -1,9 +1,10 @@ -T1 de-tudarmstadt-ukp-dkpro-core-api-ner-type-Organization 0 4 Sony -A1 de-tudarmstadt-ukp-dkpro-core-api-ner-type-Organization_value T1 This is a named entity. -T2 de-tudarmstadt-ukp-dkpro-core-api-ner-type-Organization 33 41 Ericsson -T3 de-tudarmstadt-ukp-dkpro-core-api-ner-type-Location 75 81 Sweden -A2 de-tudarmstadt-ukp-dkpro-core-api-ner-type-Location_value T3 LOC +T1 de-tudarmstadt-ukp-dkpro-core-api-ner-type-NamedEntity 0 4 Sony +A1 de-tudarmstadt-ukp-dkpro-core-api-ner-type-NamedEntity_value T1 This is a named entity. +T2 de-tudarmstadt-ukp-dkpro-core-api-ner-type-NamedEntity 33 41 Ericsson +A2 de-tudarmstadt-ukp-dkpro-core-api-ner-type-NamedEntity_value T2 LOC +T3 de-tudarmstadt-ukp-dkpro-core-api-ner-type-NamedEntity 75 81 Sweden +A3 de-tudarmstadt-ukp-dkpro-core-api-ner-type-NamedEntity_value T3 LOC T4 de-tudarmstadt-ukp-dkpro-core-io-brat-type-MergeOrg 14 27 joint venture E1 de-tudarmstadt-ukp-dkpro-core-io-brat-type-MergeOrg:T4 Org:T1 Org1:T2 -A3 de-tudarmstadt-ukp-dkpro-core-io-brat-type-MergeOrg_comment E1 This is an event. -R1 de-tudarmstadt-ukp-dkpro-core-io-brat-type-AnnotationRelation source:T2 target:T3 \ No newline at end of file +A4 de-tudarmstadt-ukp-dkpro-core-io-brat-type-MergeOrg_comment E1 This is an event. +R1 de-tudarmstadt-ukp-dkpro-core-io-brat-type-AnnotationRelation source:T2 target:T3 diff --git a/dkpro-core-io-brat-asl/src/test/resources/brat/document1-ref.ann b/dkpro-core-io-brat-asl/src/test/resources/brat/document1-ref.ann index 10a3a85dcc..c026b54f91 100644 --- a/dkpro-core-io-brat-asl/src/test/resources/brat/document1-ref.ann +++ b/dkpro-core-io-brat-asl/src/test/resources/brat/document1-ref.ann @@ -1,8 +1,8 @@ -T1 de-tudarmstadt-ukp-dkpro-core-api-ner-type-Organization 0 4 Sony -A1 de-tudarmstadt-ukp-dkpro-core-api-ner-type-Organization_value T1 This is a named entity. -T2 de-tudarmstadt-ukp-dkpro-core-api-ner-type-Organization 33 41 Ericsson -T3 de-tudarmstadt-ukp-dkpro-core-api-ner-type-Location 75 81 Sweden +T1 de-tudarmstadt-ukp-dkpro-core-api-ner-type-NamedEntity 0 4 Sony +A1 de-tudarmstadt-ukp-dkpro-core-api-ner-type-NamedEntity_value T1 This is a named entity. +T2 de-tudarmstadt-ukp-dkpro-core-api-ner-type-NamedEntity 33 41 Ericsson +T3 de-tudarmstadt-ukp-dkpro-core-api-ner-type-NamedEntity 75 81 Sweden T4 de-tudarmstadt-ukp-dkpro-core-io-brat-type-MergeOrg 14 27 joint venture E1 de-tudarmstadt-ukp-dkpro-core-io-brat-type-MergeOrg:T4 Org:T1 Org1:T2 A2 de-tudarmstadt-ukp-dkpro-core-io-brat-type-MergeOrg_comment E1 This is an event. -R1 de-tudarmstadt-ukp-dkpro-core-io-brat-type-AnnotationRelation source:T2 target:T3 \ No newline at end of file +R1 de-tudarmstadt-ukp-dkpro-core-io-brat-type-AnnotationRelation source:T2 target:T3 diff --git a/dkpro-core-io-gate-asl/src/main/java/org/dkpro/core/io/gate/internal/DKPro2Gate.java b/dkpro-core-io-gate-asl/src/main/java/org/dkpro/core/io/gate/internal/DKPro2Gate.java index afe9cfb8d0..16610a4be7 100644 --- a/dkpro-core-io-gate-asl/src/main/java/org/dkpro/core/io/gate/internal/DKPro2Gate.java +++ b/dkpro-core-io-gate-asl/src/main/java/org/dkpro/core/io/gate/internal/DKPro2Gate.java @@ -33,10 +33,7 @@ import org.apache.uima.jcas.cas.TOP; import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS; -import de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location; import de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity; -import de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization; -import de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Lemma; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; @@ -112,23 +109,21 @@ else if (fs instanceof NamedEntity) { fm.put(TOKEN_LENGTH_FEATURE_NAME, ne.getCoveredText().length()); fm.put(TOKEN_STRING_FEATURE_NAME, ne.getCoveredText()); fm.put("value", ne.getValue()); - fm.put("dkproType", ne.getClass().getSimpleName()); - if (ne instanceof Person) { - as.add(Long.valueOf(ne.getBegin()), Long.valueOf(ne.getEnd()), - PERSON_ANNOTATION_TYPE, fm); + String gateType; + String value = ne.getValue(); + if ("PER".equals(value) || "PERSON".equals(value)) { + gateType = PERSON_ANNOTATION_TYPE; } - else if (ne instanceof Location) { - as.add(Long.valueOf(ne.getBegin()), Long.valueOf(ne.getEnd()), - LOCATION_ANNOTATION_TYPE, fm); + else if ("LOC".equals(value) || "LOCATION".equals(value)) { + gateType = LOCATION_ANNOTATION_TYPE; } - else if (ne instanceof Organization) { - as.add(Long.valueOf(ne.getBegin()), Long.valueOf(ne.getEnd()), - ORGANIZATION_ANNOTATION_TYPE, fm); + else if ("ORG".equals(value) || "ORGANIZATION".equals(value)) { + gateType = ORGANIZATION_ANNOTATION_TYPE; } else { - as.add(Long.valueOf(ne.getBegin()), Long.valueOf(ne.getEnd()), "NamedEntity", - fm); + gateType = "NamedEntity"; } + as.add(Long.valueOf(ne.getBegin()), Long.valueOf(ne.getEnd()), gateType, fm); } else if (fs instanceof Sentence) { Sentence s = (Sentence) fs; diff --git a/dkpro-core-io-gate-asl/src/test/java/org/dkpro/core/io/gate/GateXmlWriterTest.java b/dkpro-core-io-gate-asl/src/test/java/org/dkpro/core/io/gate/GateXmlWriterTest.java index b61f824fd1..910c1b626b 100644 --- a/dkpro-core-io-gate-asl/src/test/java/org/dkpro/core/io/gate/GateXmlWriterTest.java +++ b/dkpro-core-io-gate-asl/src/test/java/org/dkpro/core/io/gate/GateXmlWriterTest.java @@ -51,28 +51,12 @@ public void oneWayNamedAnnotationSet() throws Exception } @Test - public void oneWayNamedEntityGeneric() throws Exception + public void oneWayNamedEntity() throws Exception { testOneWay( // Conll2002Reader.class, // GateXmlWriter.class, // - "conll/2002/ner2002_ref_generic.xml", // + "conll/2002/ner2002_ref.xml", // "conll/2002/ner2002_test.conll"); } - - @Test - public void oneWayNamedEntitySpecific() throws Exception - { - testOneWay( // - createReaderDescription( // - Conll2002Reader.class, // - Conll2002Reader.PARAM_LANGUAGE, "en", // - Conll2002Reader.PARAM_NAMED_ENTITY_MAPPING_LOCATION, - "src/test/resources/conll/2002/ner2002_conll.map"), - createEngineDescription( // - GateXmlWriter.class), // - "conll/2002/ner2002_ref_specific.xml", // - "conll/2002/ner2002_test.conll"); - - } } diff --git a/dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_conll.map b/dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_conll.map deleted file mode 100644 index 0e99f61f36..0000000000 --- a/dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_conll.map +++ /dev/null @@ -1,4 +0,0 @@ -PER=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Person -LOC=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Location -ORG=de.tudarmstadt.ukp.dkpro.core.api.ner.type.Organization -*=de.tudarmstadt.ukp.dkpro.core.api.ner.type.NamedEntity diff --git a/dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_ref_specific.xml b/dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_ref.xml similarity index 93% rename from dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_ref_specific.xml rename to dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_ref.xml index 39df6ba9f5..9cada9bbbb 100644 --- a/dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_ref_specific.xml +++ b/dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_ref.xml @@ -11,40 +11,32 @@ - + length - 11 + 5 string - Real Madrid + Wolff value - ORG - - - dkproType - Organization + PER - + length - 5 + 9 string - Wolff + Argentina value - PER - - - dkproType - Person + LOC @@ -60,27 +52,19 @@ value PER - - dkproType - Person - - + length - 9 + 11 string - Argentina + Real Madrid value - LOC - - - dkproType - Location + ORG diff --git a/dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_ref_generic.xml b/dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_ref_generic.xml deleted file mode 100644 index f2659da887..0000000000 --- a/dkpro-core-io-gate-asl/src/test/resources/conll/2002/ner2002_ref_generic.xml +++ /dev/null @@ -1,320 +0,0 @@ - - - - - - - - -Wolff , currently a journalist in Argentina , played with Del Bosque in the final years of the seventies in Real Madrid . - - - - - - - length - 5 - - - string - Wolff - - - value - PER - - - dkproType - NamedEntity - - - - - length - 9 - - - string - Argentina - - - value - LOC - - - dkproType - NamedEntity - - - - - length - 10 - - - string - Del Bosque - - - value - PER - - - dkproType - NamedEntity - - - - - length - 11 - - - string - Real Madrid - - - value - ORG - - - dkproType - NamedEntity - - - - - - - length - 5 - - - string - Wolff - - - - - length - 1 - - - string - , - - - - - length - 9 - - - string - currently - - - - - length - 1 - - - string - a - - - - - length - 10 - - - string - journalist - - - - - length - 2 - - - string - in - - - - - length - 9 - - - string - Argentina - - - - - length - 1 - - - string - , - - - - - length - 6 - - - string - played - - - - - length - 4 - - - string - with - - - - - length - 3 - - - string - Del - - - - - length - 6 - - - string - Bosque - - - - - length - 2 - - - string - in - - - - - length - 3 - - - string - the - - - - - length - 5 - - - string - final - - - - - length - 5 - - - string - years - - - - - length - 2 - - - string - of - - - - - length - 3 - - - string - the - - - - - length - 9 - - - string - seventies - - - - - length - 2 - - - string - in - - - - - length - 4 - - - string - Real - - - - - length - 6 - - - string - Madrid - - - - - length - 1 - - - string - . - - - - - \ No newline at end of file From 48f814ac4200c1c7bc82fce5571fa98d757ba176 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 14 Jun 2026 13:07:00 +0200 Subject: [PATCH 2/2] #78 - Clean up NER types - Rewrite the type4:Person elements to type4:NamedEntity; xmi:ids stay stable so the view member references remain valid. --- dkpro-core-io-xmi-asl/src/test/resources/xmi/english.xmi | 8 ++++---- dkpro-core-io-xmi-asl/src/test/resources/xmi/english2.xmi | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dkpro-core-io-xmi-asl/src/test/resources/xmi/english.xmi b/dkpro-core-io-xmi-asl/src/test/resources/xmi/english.xmi index 58697c5f30..7aa30ceb2c 100644 --- a/dkpro-core-io-xmi-asl/src/test/resources/xmi/english.xmi +++ b/dkpro-core-io-xmi-asl/src/test/resources/xmi/english.xmi @@ -203,10 +203,10 @@ - - - - + + + + diff --git a/dkpro-core-io-xmi-asl/src/test/resources/xmi/english2.xmi b/dkpro-core-io-xmi-asl/src/test/resources/xmi/english2.xmi index 376b05b482..4cffda1fde 100644 --- a/dkpro-core-io-xmi-asl/src/test/resources/xmi/english2.xmi +++ b/dkpro-core-io-xmi-asl/src/test/resources/xmi/english2.xmi @@ -202,10 +202,10 @@ - - - - + + + +