From cbe2a26123cf15b91bb0a4826149994a3235af7a Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 14 Jun 2026 15:17:01 +0200 Subject: [PATCH 1/3] #1636 - Remove deprecated APIs for 3.0.0 - Remove deprecated SerializedCasReader and SerializedCasWriter (io-bincas), superseded by BinaryCasReader / BinaryCasWriter - Migrate serialized-format test coverage to BinaryCasWriter (format S+ for embedded type system, S for separate type system) - Fix stale {@link SerializedCasReader} javadoc reference in BinaryCasWriter - Remove deprecated LineBasedSentenceSegmenter and WhitespaceSegmenter (tokit), superseded by RegexSegmenter - Remove the dedicated tests for the removed classes (SerializedCasWriterReaderTest, LineBasedSentenceSegmenterTest, WhitespaceSegmenterTest) --- .../dkpro/core/io/bincas/BinaryCasWriter.java | 4 +- .../core/io/bincas/SerializedCasReader.java | 131 ------------- .../core/io/bincas/SerializedCasWriter.java | 153 --------------- .../io/bincas/BinaryCasWriterReaderTest.java | 26 +-- .../bincas/SerializedCasWriterReaderTest.java | 176 ----------------- .../tokit/LineBasedSentenceSegmenter.java | 61 ------ .../dkpro/core/tokit/WhitespaceSegmenter.java | 107 ----------- .../tokit/LineBasedSentenceSegmenterTest.java | 60 ------ .../core/tokit/WhitespaceSegmenterTest.java | 177 ------------------ 9 files changed, 11 insertions(+), 884 deletions(-) delete mode 100644 dkpro-core-io-bincas-asl/src/main/java/org/dkpro/core/io/bincas/SerializedCasReader.java delete mode 100644 dkpro-core-io-bincas-asl/src/main/java/org/dkpro/core/io/bincas/SerializedCasWriter.java delete mode 100644 dkpro-core-io-bincas-asl/src/test/java/org/dkpro/core/io/bincas/SerializedCasWriterReaderTest.java delete mode 100644 dkpro-core-tokit-asl/src/main/java/org/dkpro/core/tokit/LineBasedSentenceSegmenter.java delete mode 100644 dkpro-core-tokit-asl/src/main/java/org/dkpro/core/tokit/WhitespaceSegmenter.java delete mode 100644 dkpro-core-tokit-asl/src/test/java/org/dkpro/core/tokit/LineBasedSentenceSegmenterTest.java delete mode 100644 dkpro-core-tokit-asl/src/test/java/org/dkpro/core/tokit/WhitespaceSegmenterTest.java diff --git a/dkpro-core-io-bincas-asl/src/main/java/org/dkpro/core/io/bincas/BinaryCasWriter.java b/dkpro-core-io-bincas-asl/src/main/java/org/dkpro/core/io/bincas/BinaryCasWriter.java index 597317dcf7..9c1b5cf785 100644 --- a/dkpro-core-io-bincas-asl/src/main/java/org/dkpro/core/io/bincas/BinaryCasWriter.java +++ b/dkpro-core-io-bincas-asl/src/main/java/org/dkpro/core/io/bincas/BinaryCasWriter.java @@ -170,8 +170,8 @@ public class BinaryCasWriter * type system file should be compressed or not is detected from the file name extension (e.g. * ".gz").
* If this parameter is set, the type system and index repository are no longer serialized into - * the same file as the test of the CAS. The {@link SerializedCasReader} can currently not read - * such files. Use this only if you really know what you are doing.
+ * the same file as the test of the CAS. The {@link BinaryCasReader} can currently not read such + * files. Use this only if you really know what you are doing.
* This parameter has no effect if formats S+ or 6+ are used as the type system information is * embedded in each individual file. Otherwise, it is recommended that this parameter be set * unless some other mechanism is used to initialize the CAS with the same type system and index diff --git a/dkpro-core-io-bincas-asl/src/main/java/org/dkpro/core/io/bincas/SerializedCasReader.java b/dkpro-core-io-bincas-asl/src/main/java/org/dkpro/core/io/bincas/SerializedCasReader.java deleted file mode 100644 index be32b81f4c..0000000000 --- a/dkpro-core-io-bincas-asl/src/main/java/org/dkpro/core/io/bincas/SerializedCasReader.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.io.bincas; - -import static java.io.ObjectInputFilter.Config.createFilter; -import static java.lang.String.join; -import static org.apache.uima.cas.impl.Serialization.deserializeCASComplete; - -import java.io.File; -import java.io.IOException; -import java.io.ObjectInputFilter; -import java.io.ObjectInputStream; - -import org.apache.uima.cas.CAS; -import org.apache.uima.cas.impl.CASCompleteSerializer; -import org.apache.uima.cas.impl.CASImpl; -import org.apache.uima.cas.impl.CASMgrSerializer; -import org.apache.uima.cas.impl.CASSerializer; -import org.apache.uima.collection.CollectionException; -import org.apache.uima.fit.descriptor.ConfigurationParameter; -import org.apache.uima.fit.descriptor.ResourceMetaData; -import org.dkpro.core.api.io.ResourceCollectionReaderBase; -import org.dkpro.core.api.resources.CompressionUtils; - -import eu.openminted.share.annotations.api.DocumentationResource; - -/** - * @deprecated use {@code BinaryCasReader} instead. - */ -@ResourceMetaData(name = "UIMA Serialized CAS Reader") -@DocumentationResource("${docbase}/format-reference.html#format-${command}") -@Deprecated -public class SerializedCasReader - extends ResourceCollectionReaderBase -{ - private final static ObjectInputFilter SERIALIZED_CAS_INPUT_FILTER = createFilter(join(";", // - CASCompleteSerializer.class.getName(), // - CASSerializer.class.getName(), // - CASMgrSerializer.class.getName(), // - String.class.getName(), // - "!*")); - - /** - * The file from which to obtain the type system if it is not embedded in the serialized CAS. - */ - public static final String PARAM_TYPE_SYSTEM_LOCATION = "typeSystemLocation"; - @ConfigurationParameter(name = PARAM_TYPE_SYSTEM_LOCATION, mandatory = false) - private String typeSystemLocation; - - private CASMgrSerializer casMgrSerializer; - - @Override - public void getNext(CAS aCAS) throws IOException, CollectionException - { - Resource res = nextFile(); - try (var is = new ObjectInputStream( - CompressionUtils.getInputStream(res.getLocation(), res.getInputStream()));) { - is.setObjectInputFilter(SERIALIZED_CAS_INPUT_FILTER); - - var object = is.readObject(); - if (object instanceof CASCompleteSerializer) { - // Annotations and CAS metadata saved together - getLogger().debug("Reading CAS and type system from [" + res.getLocation() + "]"); - var serializer = (CASCompleteSerializer) object; - deserializeCASComplete(serializer, (CASImpl) aCAS); - } - else if (object instanceof CASSerializer) { - // Annotations and CAS metadata saved separately - var serializer = new CASCompleteSerializer(); - serializer.setCasMgrSerializer(readCasManager()); - serializer.setCasSerializer((CASSerializer) object); - getLogger().debug("Reading CAS from [" + res.getLocation() + "]"); - deserializeCASComplete(serializer, (CASImpl) aCAS); - } - else { - throw new IOException("Unknown serialized object found with type [" - + object.getClass().getName() + "]"); - } - } - catch (ClassNotFoundException e) { - throw new IOException(e); - } - } - - private CASMgrSerializer readCasManager() throws IOException - { - // If we already read the type system, return it - do not read it again. - if (casMgrSerializer != null) { - return casMgrSerializer; - } - - org.springframework.core.io.Resource r; - // Is absolute? - if (typeSystemLocation.indexOf(':') != -1 || typeSystemLocation.startsWith("/") - || typeSystemLocation.startsWith(File.separator)) { - // If the type system location is absolute, resolve it absolute - r = getResolver().getResource(locationToUrl(typeSystemLocation)); - } - else { - // If the type system is not absolute, resolve it relative to the base location - r = getResolver().getResource(getBase() + typeSystemLocation); - } - getLogger().debug("Reading type system from [" + r.getURI() + "]"); - - try (var is = new ObjectInputStream( - CompressionUtils.getInputStream(typeSystemLocation, r.getInputStream()))) { - is.setObjectInputFilter(SERIALIZED_CAS_INPUT_FILTER); - casMgrSerializer = (CASMgrSerializer) is.readObject(); - } - catch (ClassNotFoundException e) { - throw new IOException(e); - } - - return casMgrSerializer; - } -} diff --git a/dkpro-core-io-bincas-asl/src/main/java/org/dkpro/core/io/bincas/SerializedCasWriter.java b/dkpro-core-io-bincas-asl/src/main/java/org/dkpro/core/io/bincas/SerializedCasWriter.java deleted file mode 100644 index 8b0d24cb54..0000000000 --- a/dkpro-core-io-bincas-asl/src/main/java/org/dkpro/core/io/bincas/SerializedCasWriter.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.io.bincas; - -import static org.apache.commons.io.IOUtils.closeQuietly; -import static org.apache.uima.cas.impl.Serialization.serializeCASComplete; -import static org.apache.uima.cas.impl.Serialization.serializeCASMgr; - -import java.io.File; -import java.io.IOException; -import java.io.ObjectOutputStream; -import java.io.OutputStream; - -import org.apache.uima.analysis_engine.AnalysisEngineProcessException; -import org.apache.uima.cas.impl.CASCompleteSerializer; -import org.apache.uima.cas.impl.CASMgrSerializer; -import org.apache.uima.cas.impl.CASSerializer; -import org.apache.uima.fit.descriptor.ConfigurationParameter; -import org.apache.uima.fit.descriptor.ResourceMetaData; -import org.apache.uima.fit.descriptor.TypeCapability; -import org.apache.uima.jcas.JCas; -import org.dkpro.core.api.io.JCasFileWriter_ImplBase; -import org.dkpro.core.api.parameter.ComponentParameters; -import org.dkpro.core.api.resources.CompressionUtils; - -import eu.openminted.share.annotations.api.DocumentationResource; - -/** - * @deprecated use {@code BinaryCasWriter} with format S instead. - */ -@ResourceMetaData(name = "UIMA Serialized CAS Writer") -@DocumentationResource("${docbase}/format-reference.html#format-${command}") -@TypeCapability(inputs = { "de.tudarmstadt.ukp.dkpro.core.api.metadata.type.DocumentMetaData" }) -@Deprecated -public class SerializedCasWriter - extends JCasFileWriter_ImplBase -{ - /** - * Location to write the type system to. The type system is saved using Java serialization, it - * is not saved as a XML type system description. We recommend to use the name - * {@code typesystem.ser}.
- * The {@link #PARAM_COMPRESSION} parameter has no effect on the type system. Instead, if the - * type system file should be compressed or not is detected from the file name extension (e.g. - * ".gz").
- * If this parameter is set, the type system and index repository are no longer serialized into - * the same file as the test of the CAS. The {@link SerializedCasReader} can currently not read - * such files. Use this only if you really know what you are doing. - */ - public static final String PARAM_TYPE_SYSTEM_LOCATION = "typeSystemLocation"; - @ConfigurationParameter(name = PARAM_TYPE_SYSTEM_LOCATION, mandatory = false) - private String typeSystemLocation; - - /** - * Use this filename extension. - */ - public static final String PARAM_FILENAME_EXTENSION = ComponentParameters.PARAM_FILENAME_EXTENSION; - @ConfigurationParameter(name = PARAM_FILENAME_EXTENSION, mandatory = true, defaultValue = ".ser") - private String filenameExtension; - - private boolean typeSystemWritten; - - @Override - public void process(JCas aJCas) throws AnalysisEngineProcessException - { - // To support writing to ZIPs, the type system must be written before the CAS document - // output stream is obtained. - try { - if (typeSystemLocation != null && !typeSystemWritten) { - writeTypeSystem(aJCas); - typeSystemWritten = true; - } - } - catch (Exception e) { - throw new AnalysisEngineProcessException(e); - } - - ObjectOutputStream docOS = null; - try { - NamedOutputStream os = getOutputStream(aJCas, filenameExtension); - docOS = new ObjectOutputStream(os); - - if (typeSystemLocation == null) { - getLogger().debug("Writing CAS and type system to [" + os + "]"); - CASCompleteSerializer serializer = serializeCASComplete(aJCas.getCasImpl()); - docOS.writeObject(serializer); - } - else { - getLogger().debug("Writing CAS to [" + os + "]"); - CASSerializer serializer = new CASSerializer(); - serializer.addCAS(aJCas.getCasImpl()); - docOS.writeObject(serializer); - } - } - catch (Exception e) { - throw new AnalysisEngineProcessException(e); - } - finally { - closeQuietly(docOS); - } - } - - private void writeTypeSystem(JCas aJCas) throws IOException - { - // If the type system location is an absolute file system location, write it there, - // otherwise use the default storage which places the file relative to the target location - if (!typeSystemLocation.startsWith(JAR_PREFIX) - && new File(typeSystemLocation).isAbsolute()) { - OutputStream typeOS = null; - try { - typeOS = CompressionUtils.getOutputStream(new File(typeSystemLocation)); - getLogger().debug("Writing type system to [" + typeSystemLocation + "]"); - writeTypeSystem(aJCas, typeOS); - } - finally { - closeQuietly(typeOS); - } - } - else { - NamedOutputStream typeOS = null; - try { - typeOS = getOutputStream(typeSystemLocation, ""); - getLogger().debug("Writing type system to [" + typeOS + "]"); - writeTypeSystem(aJCas, typeOS); - } - finally { - closeQuietly(typeOS); - } - } - } - - private void writeTypeSystem(JCas aJCas, OutputStream aOS) throws IOException - { - ObjectOutputStream typeOS = new ObjectOutputStream(aOS); - CASMgrSerializer casMgrSerializer = serializeCASMgr(aJCas.getCasImpl()); - typeOS.writeObject(casMgrSerializer); - typeOS.flush(); - } -} diff --git a/dkpro-core-io-bincas-asl/src/test/java/org/dkpro/core/io/bincas/BinaryCasWriterReaderTest.java b/dkpro-core-io-bincas-asl/src/test/java/org/dkpro/core/io/bincas/BinaryCasWriterReaderTest.java index 6ccaa150b1..34b37e7733 100644 --- a/dkpro-core-io-bincas-asl/src/test/java/org/dkpro/core/io/bincas/BinaryCasWriterReaderTest.java +++ b/dkpro-core-io-bincas-asl/src/test/java/org/dkpro/core/io/bincas/BinaryCasWriterReaderTest.java @@ -476,23 +476,15 @@ public void writeSerialized(String aLocation, boolean aWriteTypeSystem) throws E TextReader.PARAM_PATTERNS, "*.txt", // TextReader.PARAM_LANGUAGE, "latin"); - AnalysisEngine writer; - if (false) { - writer = AnalysisEngineFactory.createEngine( // - SerializedCasWriter.class, // - SerializedCasWriter.PARAM_TARGET_LOCATION, aLocation, // - SerializedCasWriter.PARAM_FILENAME_EXTENSION, ".bin", // - SerializedCasWriter.PARAM_TYPE_SYSTEM_LOCATION, - aWriteTypeSystem ? new File(aLocation, "typesystem.bin") : null); - } - else { - writer = AnalysisEngineFactory.createEngine( // - SerializedCasWriter.class, // - SerializedCasWriter.PARAM_TARGET_LOCATION, testFolder, // - SerializedCasWriter.PARAM_FILENAME_EXTENSION, ".bin", // - SerializedCasWriter.PARAM_TYPE_SYSTEM_LOCATION, - aWriteTypeSystem ? "typesystem.bin" : null); - } + // "S+" embeds the type system in the CAS file (matches the old SerializedCasWriter with no + // separate type system location), "S" writes the type system to a separate location. + AnalysisEngine writer = AnalysisEngineFactory.createEngine( // + BinaryCasWriter.class, // + BinaryCasWriter.PARAM_FORMAT, aWriteTypeSystem ? "S" : "S+", // + BinaryCasWriter.PARAM_TARGET_LOCATION, testFolder, // + BinaryCasWriter.PARAM_FILENAME_EXTENSION, ".bin", // + BinaryCasWriter.PARAM_TYPE_SYSTEM_LOCATION, + aWriteTypeSystem ? "typesystem.bin" : null); runPipeline(reader, writer); diff --git a/dkpro-core-io-bincas-asl/src/test/java/org/dkpro/core/io/bincas/SerializedCasWriterReaderTest.java b/dkpro-core-io-bincas-asl/src/test/java/org/dkpro/core/io/bincas/SerializedCasWriterReaderTest.java deleted file mode 100644 index 556fe8e2d6..0000000000 --- a/dkpro-core-io-bincas-asl/src/test/java/org/dkpro/core/io/bincas/SerializedCasWriterReaderTest.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.io.bincas; - -import static org.apache.commons.io.FileUtils.readFileToString; -import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngine; -import static org.apache.uima.fit.factory.CollectionReaderFactory.createReader; -import static org.apache.uima.fit.factory.TypeSystemDescriptionFactory.createTypeSystemDescription; -import static org.apache.uima.fit.pipeline.SimplePipeline.runPipeline; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.File; - -import org.apache.uima.analysis_engine.AnalysisEngine; -import org.apache.uima.cas.CAS; -import org.apache.uima.collection.CollectionReader; -import org.apache.uima.util.CasCreationUtils; -import org.dkpro.core.io.text.TextReader; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -public class SerializedCasWriterReaderTest -{ - private @TempDir File testFolder; - - @Test - public void testCasWithTypeSystemEmbedded() throws Exception - { - write(true); - read(); - } - - @Test - public void testCasWithTypeSystemSeparate() throws Exception - { - write(false); - read(); - } - - public void write(boolean aIncludeTypeSystem) throws Exception - { - CollectionReader reader = createReader( // - TextReader.class, // - TextReader.PARAM_SOURCE_LOCATION, "src/test/resources/texts", // - TextReader.PARAM_PATTERNS, "*.txt", // - TextReader.PARAM_LANGUAGE, "latin"); - - AnalysisEngine writer = createEngine( // - SerializedCasWriter.class, // - SerializedCasWriter.PARAM_TARGET_LOCATION, testFolder, // - SerializedCasWriter.PARAM_TYPE_SYSTEM_LOCATION, - aIncludeTypeSystem ? null : new File(testFolder, "typesystem.ser")); - - runPipeline(reader, writer); - - assertTrue(new File(testFolder, "example1.txt.ser").exists()); - } - - public void read() throws Exception - { - CollectionReader reader = createReader( // - SerializedCasReader.class, // - SerializedCasReader.PARAM_SOURCE_LOCATION, testFolder, // - SerializedCasReader.PARAM_PATTERNS, "*.ser", // - SerializedCasReader.PARAM_TYPE_SYSTEM_LOCATION, - new File(testFolder, "typesystem.ser")); - - CAS cas = CasCreationUtils.createCas(createTypeSystemDescription(), null, null); - reader.getNext(cas); - - String refText = readFileToString(new File("src/test/resources/texts/example1.txt")); - assertEquals(refText, cas.getDocumentText()); - assertEquals("latin", cas.getDocumentLanguage()); - } - - // @Test - // public void lenientTest() throws Exception - // { - // TypeSystemDescription tsdMeta = TypeSystemDescriptionFactory - // .createTypeSystemDescription("desc.type.metadata"); - // - // - // // Create a CAS initialized with that type system and set the text - // CAS casOut = createCas(tsdMeta, null, null); - // casOut.setDocumentText("This is a test."); - // DocumentMetaData meta = DocumentMetaData.create(casOut); - // meta.setDocumentId("document"); - // - // // Write out - // AnalysisEngine writer = AnalysisEngineFactory.createEngine( - // SerializedCasWriter.class, tsdMeta, - // SerializedCasWriter.PARAM_TARGET_LOCATION, testFolder.getRoot().getPath()); - // writer.process(casOut); - // - // // Create a new type system from scratch - // TypeSystemDescription tsd = new TypeSystemDescription_impl(); - // TypeDescription tokenTypeDesc = tsd.addType("Token", "", CAS.TYPE_NAME_ANNOTATION); - // tokenTypeDesc.addFeature("length", "", CAS.TYPE_NAME_INTEGER); - // tsd = CasCreationUtils.mergeTypeSystems(asList(tsd, tsdMeta)); - // - // // Now read in to CAS with different type system - // CollectionReader reader = CollectionReaderFactory.createReader( - // SerializedCasReader.class, - // SerializedCasReader.PARAM_SOURCE_LOCATION, testFolder.getRoot().getPath(), - // SerializedCasReader.PARAM_PATTERNS, new String [] { - // SerializedCasReader.INCLUDE_PREFIX+"*.ser" - // }); - // - // CAS casIn = CasCreationUtils.createCas(tsd, null, null); - // reader.getNext(casIn); - // - // upgrade(casIn, tsd); - // - // // Try to create an annotation with the extra type - // AnnotationFS fs = casOut.createAnnotation(casIn.getTypeSystem().getType("Token"), 0, 1); - // casOut.addFsToIndexes(fs); - // } - // - // private void upgrade(CAS aCas, TypeSystemDescription aTsd) throws Exception - // { - // // Prepare template for new CAS - // CAS newCas = CasCreationUtils.createCas(aTsd, null, null); - // CASCompleteSerializer serializer = Serialization.serializeCASComplete((CASImpl) newCas); - // - // // Save old type system - // TypeSystem oldTypeSystem = aCas.getTypeSystem(); - // - // // Save old CAS contents - // ByteArrayOutputStream os2 = new ByteArrayOutputStream(); - // Serialization.serializeWithCompression(aCas, os2, oldTypeSystem); - // - // // Prepare CAS with new type system - // Serialization.deserializeCASComplete(serializer, (CASImpl) aCas); - // - // // Restore CAS data to new type system - // Serialization.deserializeCAS(aCas, new ByteArrayInputStream(os2.toByteArray()), - // oldTypeSystem, null); - // } - // - // private void upgrade(CAS aCas) throws Exception - // { - // // Prepare template for new CAS - // CAS newCas = JCasFactory.createJCas().getCas(); - // CASCompleteSerializer serializer = Serialization.serializeCASComplete((CASImpl) newCas); - // - // // Save old type system - // TypeSystem oldTypeSystem = aCas.getTypeSystem(); - // - // // Save old CAS contents - // ByteArrayOutputStream os2 = new ByteArrayOutputStream(); - // Serialization.serializeWithCompression(aCas, os2, oldTypeSystem); - // - // // Prepare CAS with new type system - // Serialization.deserializeCASComplete(serializer, (CASImpl) aCas); - // - // // Restore CAS data to new type system - // Serialization.deserializeCAS(aCas, new ByteArrayInputStream(os2.toByteArray()), - // oldTypeSystem, null); - // } -} diff --git a/dkpro-core-tokit-asl/src/main/java/org/dkpro/core/tokit/LineBasedSentenceSegmenter.java b/dkpro-core-tokit-asl/src/main/java/org/dkpro/core/tokit/LineBasedSentenceSegmenter.java deleted file mode 100644 index 790374d484..0000000000 --- a/dkpro-core-tokit-asl/src/main/java/org/dkpro/core/tokit/LineBasedSentenceSegmenter.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dkpro.core.tokit; - -import org.apache.uima.analysis_engine.AnalysisEngineProcessException; -import org.apache.uima.fit.descriptor.ResourceMetaData; -import org.apache.uima.fit.descriptor.TypeCapability; -import org.apache.uima.jcas.JCas; -import org.dkpro.core.api.segmentation.SegmenterBase; - -/** - * Annotates each line in the source text as a sentence. This segmenter is not capable of creating - * tokens! All respective parameters have no functionality. - * - * @deprecated Use {@link RegexSegmenter} - */ -@ResourceMetaData(name = "Line-based Sentence Segmenter") -@TypeCapability(outputs = { "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence" }) -@Deprecated -public class LineBasedSentenceSegmenter - extends SegmenterBase -{ - @Override - protected void process(JCas aJCas, String aText, int aZoneBegin) - throws AnalysisEngineProcessException - { - int begin = 0; - int cursor = 0; - while (true) { - // Create a new sentence - if (cursor >= aText.length() || aText.charAt(cursor) == '\n') { - cursor = Math.min(cursor, aText.length()); - createSentence(aJCas, aZoneBegin + begin, aZoneBegin + cursor); - begin = cursor + 1; - } - - // Stop at end of text - if (cursor >= aText.length()) { - break; - } - - cursor++; - } - } -} diff --git a/dkpro-core-tokit-asl/src/main/java/org/dkpro/core/tokit/WhitespaceSegmenter.java b/dkpro-core-tokit-asl/src/main/java/org/dkpro/core/tokit/WhitespaceSegmenter.java deleted file mode 100644 index 5177287a46..0000000000 --- a/dkpro-core-tokit-asl/src/main/java/org/dkpro/core/tokit/WhitespaceSegmenter.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dkpro.core.tokit; - -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.uima.analysis_engine.AnalysisEngineProcessException; -import org.apache.uima.fit.descriptor.ResourceMetaData; -import org.apache.uima.fit.descriptor.TypeCapability; -import org.apache.uima.jcas.JCas; -import org.dkpro.core.api.segmentation.SegmenterBase; - -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; -import eu.openminted.share.annotations.api.DocumentationResource; - -/** - * A strict whitespace tokenizer, i.e. tokenizes according to whitespaces and linebreaks only. - *

- * If {@code PARAM_WRITE_SENTENCES} is set to true, one sentence per line is assumed. Otherwise, no - * sentences are created. - * - * @deprecated Use {@link RegexSegmenter} - */ -@ResourceMetaData(name = "Whitespace Segmenter") -@DocumentationResource("${docbase}/component-reference.html#engine-${shortClassName}") -@TypeCapability(outputs = { "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token", - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence" }) -@Deprecated -public class WhitespaceSegmenter - extends SegmenterBase -{ - // FIXME: this does not match Windows linebreaks ("\r\n") - private static final Pattern lineBreak = Pattern.compile("\n"); - private static final Pattern whitespace = Pattern.compile("[ \n]+"); - - @Override - protected void process(JCas aJCas, String text, int zoneBegin) - throws AnalysisEngineProcessException - { - /* append trailing linebreak if necessary */ - text = text.endsWith("\n") ? text : text + "\n"; - - if (isWriteSentence()) { - createSentences(aJCas, text); - } - - createTokens(aJCas, text); - } - - /** - * Create sentences using the boundary pattern defined in {@link #lineBreak}. - * - * @param aJCas - * the {@link JCas} - * @param text - * the text. - */ - private void createSentences(JCas aJCas, String text) - { - Matcher lineMatcher = lineBreak.matcher(text); - int previousStart = 0; - while (lineMatcher.find()) { - int end = lineMatcher.start(); - Sentence sentence = new Sentence(aJCas, previousStart, end); - sentence.addToIndexes(aJCas); - previousStart = lineMatcher.end(); - } - } - - /** - * Create tokens using the boundary pattern defined in {@link #whitespace}. - * - * @param aJCas - * the {@link JCas} - * @param text - * the text - */ - private void createTokens(JCas aJCas, String text) - { - Matcher whitespaceMatcher = whitespace.matcher(text); - int previousStart = 0; - while (whitespaceMatcher.find()) { - int end = whitespaceMatcher.start(); - Token token = new Token(aJCas, previousStart, end); - token.addToIndexes(aJCas); - previousStart = whitespaceMatcher.end(); - } - } -} diff --git a/dkpro-core-tokit-asl/src/test/java/org/dkpro/core/tokit/LineBasedSentenceSegmenterTest.java b/dkpro-core-tokit-asl/src/test/java/org/dkpro/core/tokit/LineBasedSentenceSegmenterTest.java deleted file mode 100644 index 62038e3933..0000000000 --- a/dkpro-core-tokit-asl/src/test/java/org/dkpro/core/tokit/LineBasedSentenceSegmenterTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.tokit; - -import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; -import static org.apache.uima.fit.util.JCasUtil.select; - -import org.apache.uima.fit.factory.JCasFactory; -import org.apache.uima.fit.pipeline.SimplePipeline; -import org.apache.uima.jcas.JCas; -import org.dkpro.core.testing.AssertAnnotations; -import org.junit.jupiter.api.Test; - -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; - -public class LineBasedSentenceSegmenterTest -{ - @Test - public void testUnix() throws Exception - { - String text = "This is \n" + "a test.\n\n" + "Right now!\nEnd"; - - String[] sentences = { "This is", "a test.", "Right now!", "End" }; - - JCas jcas = JCasFactory.createJCas(); - jcas.setDocumentText(text); - SimplePipeline.runPipeline(jcas, createEngineDescription(LineBasedSentenceSegmenter.class)); - - AssertAnnotations.assertSentence(sentences, select(jcas, Sentence.class)); - } - - @Test - public void testWindows() throws Exception - { - String text = "This is \r\n" + "a test.\r\n\r\n" + "Right now!\r\nEnd"; - - String[] sentences = { "This is", "a test.", "Right now!", "End" }; - - JCas jcas = JCasFactory.createJCas(); - jcas.setDocumentText(text); - SimplePipeline.runPipeline(jcas, createEngineDescription(LineBasedSentenceSegmenter.class)); - - AssertAnnotations.assertSentence(sentences, select(jcas, Sentence.class)); - } -} diff --git a/dkpro-core-tokit-asl/src/test/java/org/dkpro/core/tokit/WhitespaceSegmenterTest.java b/dkpro-core-tokit-asl/src/test/java/org/dkpro/core/tokit/WhitespaceSegmenterTest.java deleted file mode 100644 index 995ea9b5ee..0000000000 --- a/dkpro-core-tokit-asl/src/test/java/org/dkpro/core/tokit/WhitespaceSegmenterTest.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.dkpro.core.tokit; - -import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; -import static org.apache.uima.fit.factory.CollectionReaderFactory.createReaderDescription; -import static org.apache.uima.fit.pipeline.SimplePipeline.runPipeline; -import static org.apache.uima.fit.util.JCasUtil.select; -import static org.apache.uima.fit.util.JCasUtil.selectCovered; -import static org.dkpro.core.testing.AssertAnnotations.assertSentence; -import static org.dkpro.core.testing.AssertAnnotations.assertToken; - -import org.apache.uima.analysis_engine.AnalysisEngineDescription; -import org.apache.uima.collection.CollectionReaderDescription; -import org.apache.uima.fit.factory.JCasFactory; -import org.apache.uima.fit.pipeline.SimplePipeline; -import org.apache.uima.jcas.JCas; -import org.apache.uima.resource.ResourceInitializationException; -import org.dkpro.core.io.text.StringReader; -import org.junit.jupiter.api.Test; - -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence; -import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; - -public class WhitespaceSegmenterTest -{ - @Test - public void simpleExample() throws Exception - { - // NOTE: This file contains Asciidoc markers for partial inclusion of this file in the - // documentation. Do not remove these tags! - // tag::example[] - JCas jcas = JCasFactory.createText("This is sentence 1 .\nThis is number 2 .", "en"); - - runPipeline(jcas, createEngineDescription(WhitespaceSegmenter.class)); - - for (Sentence s : select(jcas, Sentence.class)) { - for (Token t : selectCovered(Token.class, s)) { - System.out.printf("[%s] ", t.getCoveredText()); - } - System.out.println(); - } - // end::example[] - - assertToken(new String[] { "This", "is", "sentence", "1", ".", "This", "is", "number", "2", - "." }, select(jcas, Token.class)); - assertSentence(new String[] { "This is sentence 1 .", "This is number 2 ." }, - select(jcas, Sentence.class)); - } - - @Test - public void test() throws ResourceInitializationException - { - String text = "This is a tokenized text ."; - String[] expectedSentences = new String[] { "This is a tokenized text ." }; - String[] expectedTokens = new String[] { "This", "is", "a", "tokenized", "text", "." }; - CollectionReaderDescription reader = createReaderDescription(StringReader.class, - StringReader.PARAM_DOCUMENT_TEXT, text, StringReader.PARAM_LANGUAGE, "en"); - AnalysisEngineDescription segmenter = createEngineDescription(WhitespaceSegmenter.class); - - for (JCas jcas : SimplePipeline.iteratePipeline(reader, segmenter)) { - assertSentence(expectedSentences, select(jcas, Sentence.class)); - assertToken(expectedTokens, select(jcas, Token.class)); - } - } - - @Test - public void testTwoLines() throws ResourceInitializationException - { - String text = "This is a tokenized text .\nAnother line with tokens ."; - String[] expectedTokens = new String[] { "This", "is", "a", "tokenized", "text", ".", - "Another", "line", "with", "tokens", "." }; - String[] expectedSentences = new String[] { "This is a tokenized text .", - "Another line with tokens ." }; - - CollectionReaderDescription reader = createReaderDescription(StringReader.class, - StringReader.PARAM_DOCUMENT_TEXT, text, StringReader.PARAM_LANGUAGE, "en"); - AnalysisEngineDescription segmenter = createEngineDescription(WhitespaceSegmenter.class); - - for (JCas jcas : SimplePipeline.iteratePipeline(reader, segmenter)) { - assertSentence(expectedSentences, select(jcas, Sentence.class)); - assertToken(expectedTokens, select(jcas, Token.class)); - } - } - - @Test - public void testNoPunctuation() throws ResourceInitializationException - { - String text = "This is a tokenized text"; - String[] expectedSentences = new String[] { "This is a tokenized text" }; - String[] expectedTokens = new String[] { "This", "is", "a", "tokenized", "text" }; - CollectionReaderDescription reader = createReaderDescription(StringReader.class, - StringReader.PARAM_DOCUMENT_TEXT, text, StringReader.PARAM_LANGUAGE, "en"); - AnalysisEngineDescription segmenter = createEngineDescription(WhitespaceSegmenter.class); - - for (JCas jcas : SimplePipeline.iteratePipeline(reader, segmenter)) { - assertSentence(expectedSentences, select(jcas, Sentence.class)); - assertToken(expectedTokens, select(jcas, Token.class)); - } - } - - @Test - public void testTrailingWhitespace() throws ResourceInitializationException - { - String text = "This is a tokenized text "; - String[] expectedSentences = new String[] { "This is a tokenized text " }; - String[] expectedTokens = new String[] { "This", "is", "a", "tokenized", "text" }; - CollectionReaderDescription reader = createReaderDescription(StringReader.class, - StringReader.PARAM_DOCUMENT_TEXT, text, StringReader.PARAM_LANGUAGE, "en"); - AnalysisEngineDescription segmenter = createEngineDescription(WhitespaceSegmenter.class); - - for (JCas jcas : SimplePipeline.iteratePipeline(reader, segmenter)) { - assertSentence(expectedSentences, select(jcas, Sentence.class)); - assertToken(expectedTokens, select(jcas, Token.class)); - } - } - - @Test - public void testPunctuation() throws ResourceInitializationException - { - String text = "This , is a tokenized text , with a final period ."; - String[] expectedSentences = new String[] { - "This , is a tokenized text , with a final period ." }; - String[] expectedTokens = new String[] { "This", ",", "is", "a", "tokenized", "text", ",", - "with", "a", "final", "period", "." }; - - CollectionReaderDescription reader = createReaderDescription(StringReader.class, - StringReader.PARAM_DOCUMENT_TEXT, text, StringReader.PARAM_LANGUAGE, "en"); - AnalysisEngineDescription segmenter = createEngineDescription(WhitespaceSegmenter.class); - - for (JCas jcas : SimplePipeline.iteratePipeline(reader, segmenter)) { - assertSentence(expectedSentences, select(jcas, Sentence.class)); - assertToken(expectedTokens, select(jcas, Token.class)); - } - } - - /** - * Test for Windows line breaks; not yet implemented - * - * @throws ResourceInitializationException - * @deprecated Windows linebreaks are not supported - */ - @Deprecated - public void testTwoLinesWindowsLineBreaks() throws ResourceInitializationException - { - String text = "This is a tokenized text .\r\nAnother line with tokens ."; - String[] expectedTokens = new String[] { "This", "is", "a", "tokenized", "text", ".", - "Another", "line", "with", "tokens", "." }; - String[] expectedSentences = new String[] { "This is a tokenized text .", - "Another line with tokens ." }; - - CollectionReaderDescription reader = createReaderDescription(StringReader.class, - StringReader.PARAM_DOCUMENT_TEXT, text, StringReader.PARAM_LANGUAGE, "en"); - AnalysisEngineDescription segmenter = createEngineDescription(WhitespaceSegmenter.class); - - for (JCas jcas : SimplePipeline.iteratePipeline(reader, segmenter)) { - assertSentence(expectedSentences, select(jcas, Sentence.class)); - assertToken(expectedTokens, select(jcas, Token.class)); - } - } -} From 5eee7a56b77e217076043b050c6919f6c8c7f472 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 14 Jun 2026 15:35:57 +0200 Subject: [PATCH 2/3] #1636 - Remove deprecated APIs for 3.0.0 - Remove deprecated ResourceCollectionReaderBase.PARAM_PATH (use PARAM_SOURCE_LOCATION) - Remove deprecated ResourceMetadata.META_ENCODING (use META_MODEL_ENCODING) - Remove deprecated ResourceUtils.resolveCompressedInputStream (use CompressionUtils.getInputStream) - Remove deprecated SegmenterBase.createToken(JCas, int, int, int) overload (use createToken(JCas, int, int)) - Remove deprecated TfIdfWriter.PARAM_OUTPUT_PATH (use PARAM_TARGET_LOCATION) - Remove deprecated AssertAnnotations.assertMorpheme (use assertMorph) and assertSemanticPredicates (use assertSemPred), plus now-unused imports --- .../api/io/ResourceCollectionReaderBase.java | 7 -- .../core/api/resources/ResourceMetadata.java | 6 -- .../core/api/resources/ResourceUtils.java | 17 ----- .../core/api/segmentation/SegmenterBase.java | 10 --- .../core/frequency/tfidf/TfIdfWriter.java | 2 - .../dkpro/core/testing/AssertAnnotations.java | 66 ------------------- 6 files changed, 108 deletions(-) diff --git a/dkpro-core-api-io-asl/src/main/java/org/dkpro/core/api/io/ResourceCollectionReaderBase.java b/dkpro-core-api-io-asl/src/main/java/org/dkpro/core/api/io/ResourceCollectionReaderBase.java index 8d44fd5733..e59000d8b5 100644 --- a/dkpro-core-api-io-asl/src/main/java/org/dkpro/core/api/io/ResourceCollectionReaderBase.java +++ b/dkpro-core-api-io-asl/src/main/java/org/dkpro/core/api/io/ResourceCollectionReaderBase.java @@ -93,13 +93,6 @@ public abstract class ResourceCollectionReaderBase public static final String INCLUDE_PREFIX = "[+]"; public static final String EXCLUDE_PREFIX = "[-]"; - /** - * Location from which the input is read. - * - * @deprecated use {@link #PARAM_SOURCE_LOCATION} - */ - @Deprecated - public static final String PARAM_PATH = ComponentParameters.PARAM_SOURCE_LOCATION; /** * Location from which the input is read. */ diff --git a/dkpro-core-api-resources-asl/src/main/java/org/dkpro/core/api/resources/ResourceMetadata.java b/dkpro-core-api-resources-asl/src/main/java/org/dkpro/core/api/resources/ResourceMetadata.java index 2c558569b4..e77b3f53f1 100644 --- a/dkpro-core-api-resources-asl/src/main/java/org/dkpro/core/api/resources/ResourceMetadata.java +++ b/dkpro-core-api-resources-asl/src/main/java/org/dkpro/core/api/resources/ResourceMetadata.java @@ -64,10 +64,4 @@ public class ResourceMetadata public static final String DC_IDENTIFIER = DC + SEP + "identifier"; public static final String DC_RIGHTS = DC + SEP + "rights"; - /** - * @deprecated Use {@link #META_MODEL_ENCODING} instead. - */ - @Deprecated - public static final String META_ENCODING = "encoding"; - } diff --git a/dkpro-core-api-resources-asl/src/main/java/org/dkpro/core/api/resources/ResourceUtils.java b/dkpro-core-api-resources-asl/src/main/java/org/dkpro/core/api/resources/ResourceUtils.java index b438003621..a38d67c140 100644 --- a/dkpro-core-api-resources-asl/src/main/java/org/dkpro/core/api/resources/ResourceUtils.java +++ b/dkpro-core-api-resources-asl/src/main/java/org/dkpro/core/api/resources/ResourceUtils.java @@ -327,23 +327,6 @@ && checkFolderPermissions(errorMessage, return file; } - /** - * @param is - * An {@link InputStream}. - * @param filename - * The filename this stream was created from. - * @return A resolved {@link InputStream} - * @throws IOException - * if something went wrong during resolving the input stream - * @deprecated Use {@link CompressionUtils#getInputStream(String, InputStream)} - */ - @Deprecated - public static InputStream resolveCompressedInputStream(InputStream is, String filename) - throws IOException - { - return CompressionUtils.getInputStream(filename, is); - } - /** * Resolve a location (which can be many things) to an URL. If the location starts with * {@code classpath:} the location is interpreted as a classpath location. Otherwise it is tried diff --git a/dkpro-core-api-segmentation-asl/src/main/java/org/dkpro/core/api/segmentation/SegmenterBase.java b/dkpro-core-api-segmentation-asl/src/main/java/org/dkpro/core/api/segmentation/SegmenterBase.java index b3dec6cc01..069ba45060 100644 --- a/dkpro-core-api-segmentation-asl/src/main/java/org/dkpro/core/api/segmentation/SegmenterBase.java +++ b/dkpro-core-api-segmentation-asl/src/main/java/org/dkpro/core/api/segmentation/SegmenterBase.java @@ -215,16 +215,6 @@ protected Sentence createSentence(final JCas aJCas, final int aBegin, final int } } - /** - * @deprecated use {@link #createToken(JCas, int, int)} - */ - @Deprecated - protected Token createToken(final JCas aJCas, final int aBegin, final int aEnd, - final int aIndex) - { - return createToken(aJCas, null, aBegin, aEnd); - } - protected Token createToken(final JCas aJCas, final int aBegin, final int aEnd) { return createToken(aJCas, null, aBegin, aEnd); diff --git a/dkpro-core-frequency-asl/src/main/java/org/dkpro/core/frequency/tfidf/TfIdfWriter.java b/dkpro-core-frequency-asl/src/main/java/org/dkpro/core/frequency/tfidf/TfIdfWriter.java index c7aad10a37..f3636a58e0 100644 --- a/dkpro-core-frequency-asl/src/main/java/org/dkpro/core/frequency/tfidf/TfIdfWriter.java +++ b/dkpro-core-frequency-asl/src/main/java/org/dkpro/core/frequency/tfidf/TfIdfWriter.java @@ -43,8 +43,6 @@ public class TfIdfWriter extends JCasAnnotator_ImplBase { - @Deprecated - public static final String PARAM_OUTPUT_PATH = ComponentParameters.PARAM_TARGET_LOCATION; /** * Specifies the path and filename where the model file is written. */ diff --git a/dkpro-core-testing-asl/src/main/java/org/dkpro/core/testing/AssertAnnotations.java b/dkpro-core-testing-asl/src/main/java/org/dkpro/core/testing/AssertAnnotations.java index 61ef83486f..26215871bb 100644 --- a/dkpro-core-testing-asl/src/main/java/org/dkpro/core/testing/AssertAnnotations.java +++ b/dkpro-core-testing-asl/src/main/java/org/dkpro/core/testing/AssertAnnotations.java @@ -70,7 +70,6 @@ import de.tudarmstadt.ukp.dkpro.core.api.anomaly.type.Anomaly; import de.tudarmstadt.ukp.dkpro.core.api.coref.type.CoreferenceChain; -import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.morph.Morpheme; import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.morph.MorphologicalFeatures; import de.tudarmstadt.ukp.dkpro.core.api.lexmorph.type.pos.POS; import de.tudarmstadt.ukp.dkpro.core.api.metadata.type.DocumentMetaData; @@ -83,9 +82,7 @@ import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token; import de.tudarmstadt.ukp.dkpro.core.api.semantics.type.SemArgLink; import de.tudarmstadt.ukp.dkpro.core.api.semantics.type.SemPred; -import de.tudarmstadt.ukp.dkpro.core.api.semantics.type.SemanticArgument; import de.tudarmstadt.ukp.dkpro.core.api.semantics.type.SemanticField; -import de.tudarmstadt.ukp.dkpro.core.api.semantics.type.SemanticPredicate; import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.PennTree; import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.chunk.Chunk; import de.tudarmstadt.ukp.dkpro.core.api.syntax.type.constituent.Constituent; @@ -214,33 +211,6 @@ public static void assertLemma(String[] aExpected, Collection aActual) assertEquals(asCopyableString(expected, true), asCopyableString(actual, true)); } - /** - * @param aExpected - * expected morph tags - * @param aActual - * actual morph tags - * @deprecated Use {@link #assertMorph(String[], Collection)} - */ - @Deprecated - public static void assertMorpheme(String[] aExpected, Collection aActual) - { - if (aExpected == null) { - return; - } - - List expected = asList(aExpected); - List actual = new ArrayList(); - - for (Morpheme a : aActual) { - actual.add(a.getMorphTag()); - } - - System.out.printf("%-20s - Expected: %s%n", "Morphemes", asCopyableString(expected)); - System.out.printf("%-20s - Actual : %s%n", "Morphemes", asCopyableString(actual)); - - assertEquals(asCopyableString(expected, true), asCopyableString(actual, true)); - } - public static void assertMorph(String[] aExpected, Collection aActual) { if (aExpected == null) { @@ -493,42 +463,6 @@ public static void assertPennTree(String aExpected, String aActual) assertEquals(expected, actual); } - /** - * @param aExpected - * expected semantic predicates - * @param aActual - * actual semantic predicates - * @deprecated Use {@link #assertSemPred(String[], Collection)} - */ - @Deprecated - public static void assertSemanticPredicates(String[] aExpected, - Collection aActual) - { - List expected = new ArrayList(asList(aExpected)); - List actual = new ArrayList(); - - for (SemanticPredicate p : aActual) { - StringBuilder sb = new StringBuilder(); - sb.append(p.getCoveredText()).append(" (").append(p.getCategory()).append("): ["); - for (SemanticArgument a : select(p.getArguments(), SemanticArgument.class)) { - sb.append('(').append(a.getRole()).append(':').append(a.getCoveredText()) - .append(')'); - } - sb.append(']'); - actual.add(sb.toString()); - } - - Collections.sort(actual); - Collections.sort(expected); - - System.out.printf("%-20s - Expected: %s%n", "Semantic predicates", - asCopyableString(expected)); - System.out.printf("%-20s - Actual : %s%n", "Semantic predicates", - asCopyableString(actual)); - - assertEquals(asCopyableString(expected, true), asCopyableString(actual, true)); - } - public static void assertSemPred(String[] aExpected, Collection aActual) { List expected = new ArrayList(asList(aExpected)); From c68d48e6014448d8195a52292e8f94da464823e6 Mon Sep 17 00:00:00 2001 From: Richard Eckart de Castilho Date: Sun, 14 Jun 2026 20:30:07 +0200 Subject: [PATCH 3/3] #1636 - Remove deprecated APIs for 3.0.0 - Remove deprecated DataPackage and DatasetLoader (api-datasets), superseded by DatasetFactory; drop the disabled DatasetLoaderTest - Remove deprecated text normalizer framework (textnormalizer): Normalizer_ImplBase, ReplacementFileNormalizer, NormalizerFactory, internal AnnotationComparator, and their tests - Move dkpro-core-tokit-asl and dkpro-core-jazzy-asl to test scope in textnormalizer (only used by tests after the removals) - Move commons-io to test scope in io-bincas (only used by tests after removing SerializedCasReader/Writer) --- .../dkpro/core/api/datasets/DataPackage.java | 120 -------- .../core/api/datasets/DatasetLoader.java | 270 ------------------ .../core/api/datasets/DatasetLoaderTest.java | 60 ---- dkpro-core-io-bincas-asl/pom.xml | 9 +- dkpro-core-textnormalizer-asl/pom.xml | 3 +- .../textnormalizer/Normalizer_ImplBase.java | 102 ------- .../ReplacementFileNormalizer.java | 210 -------------- .../factory/NormalizerFactory.java | 153 ---------- .../internal/AnnotationComparator.java | 33 --- .../ReplacementFileNormalizerTest.java | 104 ------- .../factory/NormalizerFactoryTest.java | 110 ------- 11 files changed, 7 insertions(+), 1167 deletions(-) delete mode 100644 dkpro-core-api-datasets-asl/src/main/java/org/dkpro/core/api/datasets/DataPackage.java delete mode 100644 dkpro-core-api-datasets-asl/src/main/java/org/dkpro/core/api/datasets/DatasetLoader.java delete mode 100644 dkpro-core-api-datasets-asl/src/test/java/org/dkpro/core/api/datasets/DatasetLoaderTest.java delete mode 100644 dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/Normalizer_ImplBase.java delete mode 100644 dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/ReplacementFileNormalizer.java delete mode 100644 dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/factory/NormalizerFactory.java delete mode 100644 dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/internal/AnnotationComparator.java delete mode 100644 dkpro-core-textnormalizer-asl/src/test/java/org/dkpro/core/textnormalizer/ReplacementFileNormalizerTest.java delete mode 100644 dkpro-core-textnormalizer-asl/src/test/java/org/dkpro/core/textnormalizer/factory/NormalizerFactoryTest.java diff --git a/dkpro-core-api-datasets-asl/src/main/java/org/dkpro/core/api/datasets/DataPackage.java b/dkpro-core-api-datasets-asl/src/main/java/org/dkpro/core/api/datasets/DataPackage.java deleted file mode 100644 index 638186bdaf..0000000000 --- a/dkpro-core-api-datasets-asl/src/main/java/org/dkpro/core/api/datasets/DataPackage.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.api.datasets; - -@Deprecated -public class DataPackage -{ - public static final DataPackage LICENSE_CC_BY_4_0 = new DataPackage.Builder() - .url("https://creativecommons.org/licenses/by/4.0/legalcode.txt") - .sha1("1167f0e28fe2db01e38e883aaf1e749fb09f9ceb").target("LICENSE.txt").build(); - - public static final DataPackage LICENSE_CC_BY_SA_3_0 = new DataPackage.Builder() - .url("https://creativecommons.org/licenses/by-sa/3.0/legalcode.txt") - .sha1("fb41626a3005c2b6e14b8b3f5d9d0b19b5faaa51").target("LICENSE.txt").build(); - - private String url; - private String target; - private String sha1; - private String md5; - private Callback action; - - public DataPackage(Builder aBuilder) - { - url = aBuilder.url; - target = aBuilder.target; - sha1 = aBuilder.sha1; - md5 = aBuilder.md5; - action = aBuilder.action; - } - - public String getUrl() - { - return url; - } - - public String getTarget() - { - return target; - } - - public String getSha1() - { - return sha1; - } - - public String getMd5() - { - return md5; - } - - public Callback getPostAction() - { - return action; - } - - public static class Builder - { - private String url; - private String target; - private String sha1; - private String md5; - private Callback action; - - public Builder url(String aUrl) - { - url = aUrl; - return this; - } - - public Builder target(String aTarget) - { - target = aTarget; - return this; - } - - public Builder sha1(String aSha1) - { - sha1 = aSha1; - return this; - } - - public Builder md5(String aMd5) - { - md5 = aMd5; - return this; - } - - public Builder postAction(Callback aAction) - { - action = aAction; - return this; - } - - public DataPackage build() - { - return new DataPackage(this); - } - } - - @FunctionalInterface - public static interface Callback - { - void run(DataPackage aPackage) throws Exception; - } -} diff --git a/dkpro-core-api-datasets-asl/src/main/java/org/dkpro/core/api/datasets/DatasetLoader.java b/dkpro-core-api-datasets-asl/src/main/java/org/dkpro/core/api/datasets/DatasetLoader.java deleted file mode 100644 index 5efdd677d6..0000000000 --- a/dkpro-core-api-datasets-asl/src/main/java/org/dkpro/core/api/datasets/DatasetLoader.java +++ /dev/null @@ -1,270 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.api.datasets; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.net.URLConnection; -import java.nio.file.Path; -import java.security.DigestInputStream; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.ArchiveInputStream; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.input.CloseShieldInputStream; -import org.apache.commons.io.output.NullOutputStream; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.dkpro.core.api.datasets.internal.ud.UDDataset; - -@Deprecated -public class DatasetLoader -{ - private final Log LOG = LogFactory.getLog(getClass()); - - private File cacheRoot; - - public DatasetLoader() - { - } - - public DatasetLoader(File aCacheRoot) - { - setCacheRoot(aCacheRoot); - } - - public void setCacheRoot(File aCacheRoot) - { - cacheRoot = aCacheRoot; - } - - public File getCacheRoot() - { - return cacheRoot; - } - - public List loadUniversalDependencyTreebankV1_3() throws IOException - { - File dataDir = new File(cacheRoot, "ud-treebanks-v1.3"); - - DataPackage data = new DataPackage.Builder() - .url("https://lindat.mff.cuni.cz/repository/xmlui/bitstream/handle/11234/" - + "1-1699/ud-treebanks-v1.3.tgz?sequence=1&isAllowed=y") - .sha1("44367112880cf0af3f293cb3f0cc6ce50c0e65c0").target("ud-treebanks-v1.3.tgz") - .postAction((d) -> { - untgz(new File(dataDir, d.getTarget()), dataDir); - }).build(); - - fetch(dataDir, data); - - List sets = new ArrayList<>(); - for (File f : new File(dataDir, "ud-treebanks-v1.3").listFiles()) { - sets.add(new UDDataset(f)); - } - return sets; - } - - private void fetch(File aTarget, DataPackage... aPackages) throws IOException - { - // First validate if local copies are still up-to-date - boolean reload = false; - packageValidationLoop: for (DataPackage pack : aPackages) { - File cachedFile = new File(aTarget, pack.getTarget()); - if (!cachedFile.exists()) { - continue; - } - - if (pack.getSha1() != null) { - String actual = getDigest(cachedFile, "SHA1"); - if (!pack.getSha1().equals(actual)) { - LOG.info("Local SHA1 hash mismatch on [" + cachedFile + "] - expected [" - + pack.getSha1() + "] - actual [" + actual + "]"); - reload = true; - break packageValidationLoop; - } - else { - LOG.info("Local SHA1 hash verified on [" + cachedFile + "] - [" + actual + "]"); - } - } - - if (pack.getMd5() != null) { - String actual = getDigest(cachedFile, "MD5"); - if (!pack.getMd5().equals(actual)) { - LOG.info("Local MD5 hash mismatch on [" + cachedFile + "] - expected [" - + pack.getMd5() + "] - actual [" + actual + "]"); - reload = true; - break packageValidationLoop; - } - else { - LOG.info("Local MD5 hash verified on [" + cachedFile + "] - [" + actual + "]"); - } - } - - } - - // If any of the packages are outdated, clear the cache and download again - if (reload) { - LOG.info("Clearing local cache for [" + aTarget + "]"); - FileUtils.deleteQuietly(aTarget); - } - - for (DataPackage pack : aPackages) { - File cachedFile = new File(aTarget, pack.getTarget()); - - if (cachedFile.exists()) { - continue; - } - - MessageDigest md5; - try { - md5 = MessageDigest.getInstance("MD5"); - } - catch (NoSuchAlgorithmException e) { - throw new IOException(e); - } - - MessageDigest sha1; - try { - sha1 = MessageDigest.getInstance("SHA1"); - } - catch (NoSuchAlgorithmException e) { - throw new IOException(e); - } - - cachedFile.getParentFile().mkdirs(); - URL source = new URL(pack.getUrl()); - - LOG.info("Fetching [" + cachedFile + "]"); - - URLConnection connection = source.openConnection(); - connection.setRequestProperty("User-Agent", "Java"); - - try (InputStream is = connection.getInputStream()) { - DigestInputStream md5Filter = new DigestInputStream(is, md5); - DigestInputStream sha1Filter = new DigestInputStream(md5Filter, sha1); - FileUtils.copyInputStreamToFile(sha1Filter, cachedFile); - - if (pack.getMd5() != null) { - String md5Hex = new String( - Hex.encodeHex(md5Filter.getMessageDigest().digest())); - if (!pack.getMd5().equals(md5Hex)) { - String message = "MD5 mismatch. Expected [" + pack.getMd5() + "] but got [" - + md5Hex + "]."; - LOG.error(message); - throw new IOException(message); - } - } - - if (pack.getSha1() != null) { - String sha1Hex = new String( - Hex.encodeHex(sha1Filter.getMessageDigest().digest())); - if (!pack.getSha1().equals(sha1Hex)) { - String message = "SHA1 mismatch. Expected [" + pack.getSha1() - + "] but got [" + sha1Hex + "]."; - LOG.error(message); - throw new IOException(message); - } - } - } - } - - // Perform a post-fetch action such as unpacking - for (DataPackage pack : aPackages) { - File cachedFile = new File(aTarget, pack.getTarget()); - File postActionCompleteMarker = new File(cachedFile.getPath() + ".postComplete"); - if (pack.getPostAction() != null && !postActionCompleteMarker.exists()) { - try { - pack.getPostAction().run(pack); - FileUtils.touch(postActionCompleteMarker); - } - catch (IOException e) { - throw e; - } - catch (Exception e) { - throw new IllegalStateException(e); - } - } - } - } - - private String getDigest(File aFile, String aDigest) throws IOException - { - MessageDigest digest; - try { - digest = MessageDigest.getInstance(aDigest); - } - catch (NoSuchAlgorithmException e) { - throw new IOException(e); - } - try (InputStream is = new FileInputStream(aFile)) { - DigestInputStream digestFilter = new DigestInputStream(is, digest); - IOUtils.copy(digestFilter, new NullOutputStream()); - return new String(Hex.encodeHex(digestFilter.getMessageDigest().digest())); - } - } - - private void untgz(File aArchive, File aTarget) throws IOException - { - try (ArchiveInputStream archive = new TarArchiveInputStream(new GzipCompressorInputStream( - new BufferedInputStream(new FileInputStream(aArchive))))) { - extract(aArchive, archive, aTarget); - } - } - - private void extract(File aArchive, ArchiveInputStream aArchiveStream, File aTarget) - throws IOException - { - ArchiveEntry entry = null; - while ((entry = aArchiveStream.getNextEntry()) != null) { - String name = entry.getName(); - - // Ensure that the filename will not break the manifest - if (name.contains("\n")) { - throw new IllegalStateException("Filename must not contain line break"); - } - - Path base = aTarget.toPath().toAbsolutePath().normalize(); - Path out = base.resolve(name).toAbsolutePath().normalize(); - - if (!out.startsWith(base)) { - // Ignore attempts to write outside the base - continue; - } - - if (entry.isDirectory()) { - FileUtils.forceMkdir(out.toFile()); - } - else { - FileUtils.copyInputStreamToFile(new CloseShieldInputStream(aArchiveStream), - out.toFile()); - } - } - } -} diff --git a/dkpro-core-api-datasets-asl/src/test/java/org/dkpro/core/api/datasets/DatasetLoaderTest.java b/dkpro-core-api-datasets-asl/src/test/java/org/dkpro/core/api/datasets/DatasetLoaderTest.java deleted file mode 100644 index 8a283beeed..0000000000 --- a/dkpro-core-api-datasets-asl/src/test/java/org/dkpro/core/api/datasets/DatasetLoaderTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.api.datasets; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.io.File; -import java.util.List; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -@Disabled("Normally we do not run this") -public class DatasetLoaderTest -{ - @Test - public void testUniversalDependencyTreebankV1_3(@TempDir File tempDir) throws Exception - { - List dss = new DatasetLoader(tempDir).loadUniversalDependencyTreebankV1_3(); - for (Dataset ds : dss) { - assertDatasetOk(ds); - } - } - - private void assertDatasetOk(Dataset ds) - { - assertThat(ds.getName()).isNotNull(); - assertThat(ds.getLanguage()).isNotNull(); - Split split = ds.getDefaultSplit(); - assertNullOrExists(split.getTestFiles()); - assertNullOrExists(split.getTestFiles()); - assertNullOrExists(split.getDevelopmentFiles()); - assertNullOrExists(ds.getLicenseFiles()); - } - - private void assertNullOrExists(File... aFiles) - { - if (aFiles != null) { - for (File f : aFiles) { - assertThat(f).exists(); - } - } - } -} diff --git a/dkpro-core-io-bincas-asl/pom.xml b/dkpro-core-io-bincas-asl/pom.xml index 80f24d1057..6b2256c649 100644 --- a/dkpro-core-io-bincas-asl/pom.xml +++ b/dkpro-core-io-bincas-asl/pom.xml @@ -39,14 +39,15 @@ org.apache.uima uimafit-core - - commons-io - commons-io - org.springframework spring-core + + commons-io + commons-io + test + org.dkpro.core dkpro-core-api-io-asl diff --git a/dkpro-core-textnormalizer-asl/pom.xml b/dkpro-core-textnormalizer-asl/pom.xml index 15e5dd54f0..8d2125a9a7 100644 --- a/dkpro-core-textnormalizer-asl/pom.xml +++ b/dkpro-core-textnormalizer-asl/pom.xml @@ -101,11 +101,13 @@ org.dkpro.core dkpro-core-tokit-asl ${project.version} + test org.dkpro.core dkpro-core-jazzy-asl ${project.version} + test org.dkpro.core @@ -196,7 +198,6 @@ The following components require model files which cannot be provided on the OpenMinTeD platform. --> - **/ReplacementFileNormalizer.xml **/DictionaryBasedTokenTransformer.xml **/FileBasedTokenTransformer.xml **/HyphenationRemover.xml diff --git a/dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/Normalizer_ImplBase.java b/dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/Normalizer_ImplBase.java deleted file mode 100644 index 31478725b5..0000000000 --- a/dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/Normalizer_ImplBase.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.textnormalizer; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -import org.apache.uima.analysis_engine.AnalysisEngineProcessException; -import org.apache.uima.fit.component.JCasAnnotator_ImplBase; -import org.apache.uima.fit.descriptor.TypeCapability; -import org.apache.uima.jcas.JCas; -import org.dkpro.core.api.transform.alignment.AlignedString; -import org.dkpro.core.textnormalizer.internal.AnnotationComparator; -import org.dkpro.core.textnormalizer.util.NormalizationUtils; - -import de.tudarmstadt.ukp.dkpro.core.api.transform.type.SofaChangeAnnotation; - -/** - * Base class for normalizers - * - */ -@TypeCapability(inputs = { - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token" }, outputs = { - "de.tudarmstadt.ukp.dkpro.core.api.transform.type.SofaChangeAnnotation" }) -@Deprecated -public abstract class Normalizer_ImplBase - extends JCasAnnotator_ImplBase -{ - - /** - * A map, where a token position maps to a list of SofaChangeAnnotations that should be applied - * for that token - * - * @param jcas - * a CAS. - * @return the mapping. - */ - protected abstract Map> createSofaChangesMap(JCas jcas); - - /** - * A map showing which token should be kept and which should be replaced. "true" indicates: - * "replace with changed version" - * - * @param jcas - * a CAS. - * @param as - * an aligned string. - * @return the mapping. - * @throws AnalysisEngineProcessException - * if there is an error. - */ - protected abstract Map createTokenReplaceMap(JCas jcas, AlignedString as) - throws AnalysisEngineProcessException; - - @Override - public void process(JCas jcas) throws AnalysisEngineProcessException - { - // Put all SofaChangeAnnotations in a map, - // where a token position maps to a list of SFCs that should be applied for that token - Map> changesMap = createSofaChangesMap(jcas); - - // create an AlignedString with all the changes applied and sort by offset - List allChanges = new ArrayList(); - for (Map.Entry> changesEntry : changesMap.entrySet()) { - allChanges.addAll(changesEntry.getValue()); - } - Collections.sort(allChanges, new AnnotationComparator()); - - AlignedString as = new AlignedString(jcas.getDocumentText()); - NormalizationUtils.applyChanges(as, allChanges); - - // create a map showing which token should be kept and which should be replaced - // "true" means replace with changed version - Map tokenReplaceMap = createTokenReplaceMap(jcas, as); - - // add SofaChangeAnnotation to indexes if replace is valid - for (int key : tokenReplaceMap.keySet()) { - if (tokenReplaceMap.get(key)) { - for (SofaChangeAnnotation c : changesMap.get(key)) { - c.addToIndexes(); - } - } - } - } -} diff --git a/dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/ReplacementFileNormalizer.java b/dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/ReplacementFileNormalizer.java deleted file mode 100644 index 33fde7cc2f..0000000000 --- a/dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/ReplacementFileNormalizer.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.textnormalizer; - -import static org.dkpro.core.castransformation.ApplyChangesAnnotator.OP_REPLACE; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.commons.io.FileUtils; -import org.apache.uima.UimaContext; -import org.apache.uima.analysis_engine.AnalysisEngineProcessException; -import org.apache.uima.fit.descriptor.ConfigurationParameter; -import org.apache.uima.fit.descriptor.ResourceMetaData; -import org.apache.uima.fit.descriptor.TypeCapability; -import org.apache.uima.jcas.JCas; -import org.apache.uima.resource.ResourceInitializationException; -import org.dkpro.core.api.parameter.ComponentParameters; -import org.dkpro.core.api.transform.alignment.AlignedString; - -import de.tudarmstadt.ukp.dkpro.core.api.transform.type.SofaChangeAnnotation; -import eu.openminted.share.annotations.api.Component; -import eu.openminted.share.annotations.api.DocumentationResource; -import eu.openminted.share.annotations.api.constants.OperationType; - -/** - * Takes a text and replaces desired expressions. This class should not work on tokens as some - * expressions might span several tokens. - */ -@Component(OperationType.NORMALIZER) -@ResourceMetaData(name = "Replacement File Normalizer") -@DocumentationResource("${docbase}/component-reference.html#engine-${shortClassName}") -@TypeCapability(inputs = { - "de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Token" }, outputs = { - "de.tudarmstadt.ukp.dkpro.core.api.transform.type.SofaChangeAnnotation" }) -@Deprecated -public class ReplacementFileNormalizer - extends Normalizer_ImplBase -{ - /** - * Location of a file which contains all replacing characters - */ - public static final String PARAM_MODEL_LOCATION = ComponentParameters.PARAM_MODEL_LOCATION; - @ConfigurationParameter(name = PARAM_MODEL_LOCATION, mandatory = true) - private String replacePath; - - /** - * The character encoding used by the model. - */ - public static final String PARAM_MODEL_ENCODING = ComponentParameters.PARAM_MODEL_ENCODING; - @ConfigurationParameter(name = PARAM_MODEL_ENCODING, mandatory = true, defaultValue = "UTF-8") - protected String modelEncoding; - - /** - * Pattern describing valid left/right context of the source expression. - */ - public static final String PARAM_SRC_SURROUNDINGS = "srcExpressionSurroundings"; - @ConfigurationParameter(name = PARAM_SRC_SURROUNDINGS, mandatory = true, defaultValue = "IRRELEVANT") - private SrcSurroundings srcExpressionSurroundings; - - /** - * Left/right context of the replacement. - */ - public static final String PARAM_TARGET_SURROUNDINGS = "targetExpressionSurroundings"; - @ConfigurationParameter(name = PARAM_TARGET_SURROUNDINGS, mandatory = true, defaultValue = "NOTHING") - private TargetSurroundings targetExpressionSurroundings; - - private String srcSurroundingsStart; - private String srcSurroundingsEnd; - private String targetSurroundings; - - public enum SrcSurroundings - { - ONLY_ALPHANIMERIC, IRRELEVANT - } - - public enum TargetSurroundings - { - WHITESPACE, NOTHING - } - - protected Map replacementMap; - - @Override - public void initialize(UimaContext context) throws ResourceInitializationException - { - super.initialize(context); - - replacementMap = getReplacementMap(); - - switch (srcExpressionSurroundings) { - case ONLY_ALPHANIMERIC: - srcSurroundingsStart = "([^a-zA-Z0-9äöüß]|^)"; - srcSurroundingsEnd = "([^a-zA-Z0-9äöüß]|$)"; - break; - case IRRELEVANT: - srcSurroundingsStart = ""; - srcSurroundingsEnd = ""; - break; - } - - switch (targetExpressionSurroundings) { - case WHITESPACE: - targetSurroundings = " "; - break; - case NOTHING: - targetSurroundings = ""; - break; - } - - } - - @Override - protected Map> createSofaChangesMap(JCas jcas) - { - Map> changesMap = new TreeMap>(); - int mapKey = 1; - - String coveredText = jcas.getDocumentText().toLowerCase(); - - List scaChangesList = new ArrayList(); - for (Map.Entry entry : replacementMap.entrySet()) { - String replacementKey = entry.getKey().toLowerCase(); - String replacementValue = targetSurroundings + entry.getValue() + targetSurroundings; - - String regex = srcSurroundingsStart + "(" + Pattern.quote(replacementKey) + ")" - + srcSurroundingsEnd; - Pattern pattern = Pattern.compile(regex); - Matcher matcher = pattern.matcher(coveredText); - - int groupNumberOfKey = (matcher.groupCount() == 1) ? 1 : 2; - - while (matcher.find()) { - int start = matcher.start(groupNumberOfKey); - int end = matcher.end(groupNumberOfKey); - - SofaChangeAnnotation sca = new SofaChangeAnnotation(jcas); - sca.setBegin(start); - sca.setEnd(end); - sca.setOperation(OP_REPLACE); - sca.setValue(replacementValue); - scaChangesList.add(sca); - - System.out.println(matcher.group(0)); - } - - } - changesMap.put(mapKey++, scaChangesList); - - return changesMap; - } - - @SuppressWarnings("serial") - @Override - protected Map createTokenReplaceMap(JCas jcas, AlignedString as) - throws AnalysisEngineProcessException - { - return new TreeMap() - { - { - put(1, true); - } - }; - } - - private Map getReplacementMap() throws ResourceInitializationException - { - Map replacementMap = new HashMap(); - try { - // Reads in all mappings of expressions(to be replaced expression, target expression) - // and fills replacement map - for (String line : FileUtils.readLines(new File(replacePath), modelEncoding)) { - if (!line.isEmpty()) { - // Each line of source file contains mapping of "to replaced expression" and the - // "target expressions" - // those expressions are separated by tabs - String[] entry = line.split("\t"); - replacementMap.put(entry[0], entry[1]); - } - } - } - catch (IOException e) { - throw new ResourceInitializationException(e); - } - - return replacementMap; - } -} diff --git a/dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/factory/NormalizerFactory.java b/dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/factory/NormalizerFactory.java deleted file mode 100644 index c9c4440980..0000000000 --- a/dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/factory/NormalizerFactory.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.textnormalizer.factory; - -import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; - -import org.apache.uima.analysis_engine.AnalysisEngineDescription; -import org.apache.uima.cas.CAS; -import org.apache.uima.fit.factory.AggregateBuilder; -import org.apache.uima.resource.ExternalResourceDescription; -import org.apache.uima.resource.ResourceInitializationException; -import org.dkpro.core.castransformation.ApplyChangesAnnotator; -import org.dkpro.core.jazzy.JazzyChecker; -import org.dkpro.core.textnormalizer.ReplacementFileNormalizer; -import org.dkpro.core.textnormalizer.ReplacementFileNormalizer.SrcSurroundings; -import org.dkpro.core.textnormalizer.ReplacementFileNormalizer.TargetSurroundings; -import org.dkpro.core.textnormalizer.SpellingNormalizer; -import org.dkpro.core.textnormalizer.frequency.CapitalizationNormalizer; -import org.dkpro.core.textnormalizer.frequency.ExpressiveLengtheningNormalizer; -import org.dkpro.core.textnormalizer.frequency.SharpSNormalizer; -import org.dkpro.core.tokit.BreakIteratorSegmenter; - -@Deprecated -public class NormalizerFactory -{ - private int view_counter = 0; - - public AnalysisEngineDescription getSpellcorrection(String aModelLocation) - throws ResourceInitializationException - { - AggregateBuilder ab = new AggregateBuilder(); - ab.add(createEngineDescription(BreakIteratorSegmenter.class), CAS.NAME_DEFAULT_SOFA, - getSourceView()); - ab.add(createEngineDescription(JazzyChecker.class, JazzyChecker.PARAM_MODEL_LOCATION, - aModelLocation), CAS.NAME_DEFAULT_SOFA, getSourceView()); - ab.add(createEngineDescription(SpellingNormalizer.class), CAS.NAME_DEFAULT_SOFA, - getSourceView()); - ab.add(createEngineDescription(ApplyChangesAnnotator.class), - ApplyChangesAnnotator.VIEW_SOURCE, getSourceView(), - ApplyChangesAnnotator.VIEW_TARGET, getTargetView()); - AnalysisEngineDescription aed = ab.createAggregateDescription(); - aed.setAnnotatorImplementationName("Spell"); - - return aed; - } - - public AnalysisEngineDescription getUmlautSharpSNormalization( - ExternalResourceDescription aFrequencyProvider, int aMinFrequency) - throws ResourceInitializationException - { - AggregateBuilder ab = new AggregateBuilder(); - ab.add(createEngineDescription(BreakIteratorSegmenter.class), CAS.NAME_DEFAULT_SOFA, - getSourceView()); - ab.add(createEngineDescription(SharpSNormalizer.class, - SharpSNormalizer.RES_FREQUENCY_PROVIDER, aFrequencyProvider, - SharpSNormalizer.PARAM_MIN_FREQUENCY_THRESHOLD, aMinFrequency), - CAS.NAME_DEFAULT_SOFA, getSourceView()); - ab.add(createEngineDescription(ApplyChangesAnnotator.class), - ApplyChangesAnnotator.VIEW_SOURCE, getSourceView(), - ApplyChangesAnnotator.VIEW_TARGET, getTargetView()); - AnalysisEngineDescription aed = ab.createAggregateDescription(); - aed.setAnnotatorImplementationName("Umlaute"); - - return aed; - } - - public AnalysisEngineDescription getReplacementNormalization(String aModelLocation, - SrcSurroundings aSrc, TargetSurroundings aTarget) - throws ResourceInitializationException - { - AggregateBuilder ab = new AggregateBuilder(); - ab.add(createEngineDescription(ReplacementFileNormalizer.class, - ReplacementFileNormalizer.PARAM_MODEL_LOCATION, aModelLocation, - ReplacementFileNormalizer.PARAM_SRC_SURROUNDINGS, aSrc, - ReplacementFileNormalizer.PARAM_TARGET_SURROUNDINGS, aTarget), - CAS.NAME_DEFAULT_SOFA, getSourceView()); - ab.add(createEngineDescription(ApplyChangesAnnotator.class), - ApplyChangesAnnotator.VIEW_SOURCE, getSourceView(), - ApplyChangesAnnotator.VIEW_TARGET, getTargetView()); - AnalysisEngineDescription aed = ab.createAggregateDescription(); - // aed.setAnnotatorImplementationName(new File(filepath).getName().split("\\")[0]); - - return aed; - } - - public AnalysisEngineDescription getExpressiveLengtheningNormalization( - ExternalResourceDescription aFrequencyProvider) - throws ResourceInitializationException - { - AggregateBuilder ab = new AggregateBuilder(); - ab.add(createEngineDescription(BreakIteratorSegmenter.class), CAS.NAME_DEFAULT_SOFA, - getSourceView()); - ab.add(createEngineDescription(ExpressiveLengtheningNormalizer.class, - ExpressiveLengtheningNormalizer.RES_FREQUENCY_PROVIDER, aFrequencyProvider), - CAS.NAME_DEFAULT_SOFA, getSourceView()); - ab.add(createEngineDescription(ApplyChangesAnnotator.class), - ApplyChangesAnnotator.VIEW_SOURCE, getSourceView(), - ApplyChangesAnnotator.VIEW_TARGET, getTargetView()); - AnalysisEngineDescription aed = ab.createAggregateDescription(); - aed.setAnnotatorImplementationName("Lengthening"); - - return aed; - } - - public AnalysisEngineDescription getCapitalizationNormalization( - ExternalResourceDescription aFrequencyProvider) - throws ResourceInitializationException - { - AggregateBuilder ab = new AggregateBuilder(); - ab.add(createEngineDescription(BreakIteratorSegmenter.class), CAS.NAME_DEFAULT_SOFA, - getSourceView()); - ab.add(createEngineDescription(CapitalizationNormalizer.class, - CapitalizationNormalizer.RES_FREQUENCY_PROVIDER, aFrequencyProvider), - CAS.NAME_DEFAULT_SOFA, getSourceView()); - ab.add(createEngineDescription(ApplyChangesAnnotator.class), - ApplyChangesAnnotator.VIEW_SOURCE, getSourceView(), - ApplyChangesAnnotator.VIEW_TARGET, getTargetView()); - AnalysisEngineDescription aed = ab.createAggregateDescription(); - aed.setAnnotatorImplementationName("Capitalization"); - - return aed; - } - - protected String getSourceView() - { - return (view_counter > 0) ? "view" + view_counter : CAS.NAME_DEFAULT_SOFA; - } - - protected String getTargetView() - { - return "view" + ++view_counter; - } - - public String getOutputView() - { - return "view" + view_counter; - } -} diff --git a/dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/internal/AnnotationComparator.java b/dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/internal/AnnotationComparator.java deleted file mode 100644 index 010a7f7002..0000000000 --- a/dkpro-core-textnormalizer-asl/src/main/java/org/dkpro/core/textnormalizer/internal/AnnotationComparator.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.textnormalizer.internal; - -import java.util.Comparator; - -import org.apache.uima.cas.text.AnnotationFS; - -@Deprecated -public class AnnotationComparator - implements Comparator -{ - @Override - public int compare(AnnotationFS arg0, AnnotationFS arg1) - { - return arg0.getBegin() - arg1.getBegin(); - } -} diff --git a/dkpro-core-textnormalizer-asl/src/test/java/org/dkpro/core/textnormalizer/ReplacementFileNormalizerTest.java b/dkpro-core-textnormalizer-asl/src/test/java/org/dkpro/core/textnormalizer/ReplacementFileNormalizerTest.java deleted file mode 100644 index ae7c38f926..0000000000 --- a/dkpro-core-textnormalizer-asl/src/test/java/org/dkpro/core/textnormalizer/ReplacementFileNormalizerTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.textnormalizer; - -import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.apache.uima.analysis_engine.AnalysisEngine; -import org.apache.uima.analysis_engine.AnalysisEngineDescription; -import org.apache.uima.cas.CAS; -import org.apache.uima.fit.factory.AggregateBuilder; -import org.apache.uima.jcas.JCas; -import org.dkpro.core.castransformation.ApplyChangesAnnotator; -import org.dkpro.core.textnormalizer.ReplacementFileNormalizer.SrcSurroundings; -import org.dkpro.core.textnormalizer.ReplacementFileNormalizer.TargetSurroundings; -import org.dkpro.core.tokit.BreakIteratorSegmenter; -import org.junit.jupiter.api.Test; - -import de.tudarmstadt.ukp.dkpro.core.api.metadata.type.DocumentMetaData; - -public class ReplacementFileNormalizerTest -{ - @Test - public void testReplacementNormalizer() throws Exception - { - testEmoticonReplacement(":-):-(", " lächeln traurig "); - // testInternetslangReplacement("AKA hdl AKA.", "Also known as Hab' dich lieb Also known - // as."); - } - - public void testEmoticonReplacement(String input, String output) throws Exception - { - AnalysisEngineDescription replace = createEngineDescription(ReplacementFileNormalizer.class, - ReplacementFileNormalizer.PARAM_MODEL_LOCATION, - "src/main/resources/replaceLists/emoticons_de.txt", - ReplacementFileNormalizer.PARAM_TARGET_SURROUNDINGS, TargetSurroundings.WHITESPACE); - - AggregateBuilder ab = new AggregateBuilder(); - ab.add(replace); - ab.add(createEngineDescription(ApplyChangesAnnotator.class), - ApplyChangesAnnotator.VIEW_SOURCE, CAS.NAME_DEFAULT_SOFA, - ApplyChangesAnnotator.VIEW_TARGET, "view1"); - - AnalysisEngine engine = ab.createAggregate(); - JCas jcas = engine.newJCas(); - jcas.setDocumentText(input); - DocumentMetaData.create(jcas); - engine.process(jcas); - - JCas view0 = jcas.getView(CAS.NAME_DEFAULT_SOFA); - JCas view1 = jcas.getView("view1"); - - System.out.println(view0.getDocumentText()); - System.out.println(view1.getDocumentText()); - - assertEquals(output, view1.getDocumentText()); - } - - public void testInternetslangReplacement(String input, String output) throws Exception - { - AnalysisEngineDescription replace = createEngineDescription(ReplacementFileNormalizer.class, - ReplacementFileNormalizer.PARAM_MODEL_LOCATION, - "src/main/resources/replaceLists/internetslang.txt", - ReplacementFileNormalizer.PARAM_SRC_SURROUNDINGS, - SrcSurroundings.ONLY_ALPHANIMERIC); - - AggregateBuilder ab = new AggregateBuilder(); - ab.add(createEngineDescription(BreakIteratorSegmenter.class), CAS.NAME_DEFAULT_SOFA, - CAS.NAME_DEFAULT_SOFA); - ab.add(replace); - ab.add(createEngineDescription(ApplyChangesAnnotator.class), - ApplyChangesAnnotator.VIEW_SOURCE, CAS.NAME_DEFAULT_SOFA, - ApplyChangesAnnotator.VIEW_TARGET, "view1"); - - AnalysisEngine engine = ab.createAggregate(); - JCas jcas = engine.newJCas(); - jcas.setDocumentText(input); - DocumentMetaData.create(jcas); - engine.process(jcas); - - JCas view0 = jcas.getView(CAS.NAME_DEFAULT_SOFA); - JCas view1 = jcas.getView("view1"); - - System.out.println(view0.getDocumentText()); - System.out.println(view1.getDocumentText()); - - assertEquals(output, view1.getDocumentText()); - } -} diff --git a/dkpro-core-textnormalizer-asl/src/test/java/org/dkpro/core/textnormalizer/factory/NormalizerFactoryTest.java b/dkpro-core-textnormalizer-asl/src/test/java/org/dkpro/core/textnormalizer/factory/NormalizerFactoryTest.java deleted file mode 100644 index 234de9daa0..0000000000 --- a/dkpro-core-textnormalizer-asl/src/test/java/org/dkpro/core/textnormalizer/factory/NormalizerFactoryTest.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2017 - * Ubiquitous Knowledge Processing (UKP) Lab - * Technische Universität Darmstadt - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.dkpro.core.textnormalizer.factory; - -import static org.apache.uima.fit.factory.ExternalResourceFactory.createResourceDescription; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.apache.uima.analysis_engine.AnalysisEngine; -import org.apache.uima.analysis_engine.AnalysisEngineDescription; -import org.apache.uima.fit.factory.AggregateBuilder; -import org.apache.uima.jcas.JCas; -import org.apache.uima.resource.ExternalResourceDescription; -import org.dkpro.core.frequency.resources.Web1TFrequencyCountResource; -import org.dkpro.core.textnormalizer.ReplacementFileNormalizer.SrcSurroundings; -import org.dkpro.core.textnormalizer.ReplacementFileNormalizer.TargetSurroundings; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; - -import de.tudarmstadt.ukp.dkpro.core.api.metadata.type.DocumentMetaData; - -public class NormalizerFactoryTest -{ - private ExternalResourceDescription frequencyProvider; - - @BeforeEach - public void init() - { - frequencyProvider = createResourceDescription(Web1TFrequencyCountResource.class, - Web1TFrequencyCountResource.PARAM_LANGUAGE, "de", - Web1TFrequencyCountResource.PARAM_MIN_NGRAM_LEVEL, "1", - Web1TFrequencyCountResource.PARAM_MAX_NGRAM_LEVEL, "1", - Web1TFrequencyCountResource.PARAM_INDEX_PATH, "src/test/resources/jweb1t"); - } - - @Disabled("This test will not work while the module is in transition - after we need to fix it") - @Test - public void testNormalizerFactory() throws Exception - { - test("GMBH +++ Gewerkschaftb +++ HDL +++ :-)", - "GmbH +++ Gewerkschaft +++ Hab' dich lieb +++ lächeln "); - } - - public void test(String input, String output) throws Exception - { - NormalizerFactory nf = new NormalizerFactory(); - AnalysisEngineDescription normalizeSharpSUmlaute = nf - .getUmlautSharpSNormalization(frequencyProvider, 0); - AnalysisEngineDescription normalizeRepitions = nf - .getExpressiveLengtheningNormalization(frequencyProvider); - AnalysisEngineDescription normalizeCapitalization = nf - .getCapitalizationNormalization(frequencyProvider); - AnalysisEngineDescription normalizeInternetslang = nf.getReplacementNormalization( - "src/main/resources/replaceLists/internetslang.txt", - SrcSurroundings.ONLY_ALPHANIMERIC, TargetSurroundings.NOTHING); - AnalysisEngineDescription normalizeSpelling = nf - .getSpellcorrection("src/test/resources/dictionary/ngerman"); - AnalysisEngineDescription normalizeEmoticons = nf.getReplacementNormalization( - "src/main/resources/replaceLists/emoticons_de.txt", SrcSurroundings.IRRELEVANT, - TargetSurroundings.WHITESPACE); - - AggregateBuilder ab = new AggregateBuilder(); - ab.add(normalizeSharpSUmlaute); - ab.add(normalizeRepitions); - ab.add(normalizeCapitalization); - ab.add(normalizeInternetslang); - ab.add(normalizeSpelling); - ab.add(normalizeEmoticons); - - AnalysisEngine engine = ab.createAggregate(); - JCas jcas = engine.newJCas(); - jcas.setDocumentText(input); - DocumentMetaData.create(jcas); - engine.process(jcas); - - JCas view0 = jcas.getView("_InitialView"); - JCas view1 = jcas.getView("view1"); - JCas view2 = jcas.getView("view2"); - JCas view3 = jcas.getView("view3"); - JCas view4 = jcas.getView("view4"); - JCas view5 = jcas.getView("view5"); - JCas view6 = jcas.getView("view6"); - - System.out.println("Original :" + view0.getDocumentText()); - System.out.println("Umlaute :" + view1.getDocumentText()); - System.out.println("Repitions :" + view2.getDocumentText()); - System.out.println("Capitalization :" + view3.getDocumentText()); - System.out.println("Internetslang :" + view4.getDocumentText()); - System.out.println("Spelling :" + view5.getDocumentText()); - System.out.println("Emoticons :" + view6.getDocumentText()); - System.out.println("Perfect :" + output); - - assertEquals(output, view6.getDocumentText()); - } -}