From dd5edc7f6c0753d3716909c79b3479841a93d609 Mon Sep 17 00:00:00 2001 From: Uchman21 Date: Mon, 27 Mar 2017 15:59:49 +0300 Subject: [PATCH 1/6] Add files via upload --- .../icm/cermine/CommandLineOptionsParser.java | 55 +++- .../pl/edu/icm/cermine/ContentExtractor.java | 260 ++++++++++++------ 2 files changed, 223 insertions(+), 92 deletions(-) diff --git a/cermine-impl/src/main/java/pl/edu/icm/cermine/CommandLineOptionsParser.java b/cermine-impl/src/main/java/pl/edu/icm/cermine/CommandLineOptionsParser.java index 6f363ab5..d03d890d 100644 --- a/cermine-impl/src/main/java/pl/edu/icm/cermine/CommandLineOptionsParser.java +++ b/cermine-impl/src/main/java/pl/edu/icm/cermine/CommandLineOptionsParser.java @@ -38,6 +38,10 @@ public class CommandLineOptionsParser { public CommandLineOptionsParser() { options = new Options(); options.addOption("path", true, "file or directory path"); + options.addOption("start", true, "start point"); + options.addOption("stop", true, "stop point"); + options.addOption("workers", true, "number of workers"); + options.addOption("outpath", true, "file or directory path for output"); options.addOption("outputs", true, "types of the output"); options.addOption("exts", true, "extensions of the output files"); options.addOption("ext", true, "metadata file extension"); @@ -55,7 +59,7 @@ public String parse(String[] args) throws ParseException { if (commandLine.getOptionValue("path") == null) { return "\"path\" parameter not specified"; } - + String output = commandLine.getOptionValue("outputs"); String exts = commandLine.getOptionValue("exts"); if (output != null) { @@ -75,6 +79,53 @@ public String parse(String[] args) throws ParseException { public String getPath() { return commandLine.getOptionValue("path"); } + + public String getOutPath() { + if (!commandLine.hasOption("outpath")) { + return commandLine.getOptionValue("path"); + } else { + return commandLine.getOptionValue("outpath"); + } + } + + public Long getStart() { + if (!commandLine.hasOption("start")) { + return 0L; + } else { + Long value = Long.parseLong(commandLine.getOptionValue("start")); + if (value < 0) { + throw new RuntimeException("The 'start' value given as a " + + "command line parameter has to be nonnegative."); + } + return value; + } + } + + public Long getStop() { + if (!commandLine.hasOption("stop")) { + return null; + } else { + Long value = Long.parseLong(commandLine.getOptionValue("stop")); + if (value < 0) { + throw new RuntimeException("The 'stop' value given as a " + + "command line parameter has to be nonnegative."); + } + return value; + } + } + + public Long getWorkers() { + if (!commandLine.hasOption("workers")) { + return 1L; + } else { + Long value = Long.parseLong(commandLine.getOptionValue("workers")); + if (value <= 0) { + throw new RuntimeException("The 'workers' value given as a " + + "command line parameter has to be nonnegative."); + } + return value; + } + } public Map getTypesAndExtensions() { Map typesAndExts = new HashMap(); @@ -84,7 +135,7 @@ public Map getTypesAndExtensions() { typesAndExts.put("trueviz", "cermstr"); typesAndExts.put("images", "images"); - String[] types = getStringOptionValue("jats,images", "outputs").split(","); + String[] types = getStringOptionValue("jats", "outputs").split(","); for (String type: Lists.newArrayList(typesAndExts.keySet())) { if (!ArrayUtils.contains(types, type)) { typesAndExts.remove(type); diff --git a/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java b/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java index 5e110c91..197e7d7f 100644 --- a/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java +++ b/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java @@ -21,6 +21,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.file.Paths; import java.util.Collection; import java.util.List; import org.apache.commons.cli.ParseException; @@ -726,6 +727,154 @@ private Timeout combineWithMainTimeout(long timeoutSeconds) { return Timeout.min(mainTimeout, local); } + private static void my_parallel_parser(ExtractionConfigBuilder builder, Collection files, Map extensions,Long start, Long stop, boolean override, Long timeoutSeconds, CommandLineOptionsParser parser, Long workers, String outpath) throws ParseException, AnalysisException, IOException, TransformationException{ + class paraTask implements Runnable { + ExtractionConfigBuilder builder; + Collection files; + Map extensions; + Long start_point; + Long stop_point; + boolean override; + Long timeoutSeconds; + CommandLineOptionsParser parser; + String outpath; + + paraTask(ExtractionConfigBuilder m_builder, Collection m_files, Map m_extensions,Long m_start, Long m_stop, boolean m_override, Long m_timeoutSeconds, CommandLineOptionsParser m_parser, String m_outpath) { + start_point = m_start; + stop_point = m_stop; + builder = m_builder; + files = m_files; + extensions = m_extensions; + override = m_override; + timeoutSeconds = m_timeoutSeconds; + parser = m_parser; + outpath = m_outpath; + } + + public void run() { + try{ + runParallel(); + }catch(IOException e) { + printException(e); + }catch(ParseException e) { + printException(e); + }catch(AnalysisException e) { + printException(e); + }catch(TransformationException e) { + printException(e); + } + + } + public void runParallel() throws ParseException, AnalysisException, IOException, TransformationException{ + int iter = 0; + for (File pdf : files) { + if (iter >= start_point && iter < stop_point){ + Map outputs = new HashMap(); + for (Map.Entry entry : extensions.entrySet()) { + File outputFile = getOutputFile(outpath, pdf, entry.getValue()); + if (override || !outputFile.exists()) { + outputs.put(entry.getKey(), outputFile); + } + } + if (outputs.isEmpty()) { + continue; + } + + long start = System.currentTimeMillis(); + float elapsed; + + System.out.println("File processed: " + pdf.getPath()); + + ContentExtractor extractor = null; + try { + extractor = createContentExtractor(timeoutSeconds); + + InputStream in = new FileInputStream(pdf); + extractor.setPDF(in); + + if (outputs.containsKey("images")) { + List images = extractor.getImages(outputs.get("images").getPath()); + FileUtils.forceMkdir(outputs.get("images")); + for (BxImage image : images) { + ImageIO.write(image.getImage(), "png", new File(image.getPath())); + } + } + + if (outputs.containsKey("jats")) { + Element jats; + if (outputs.containsKey("images")) { + jats = extractor.getContentAsNLM(outputs.get("images").getPath()); + } else { + jats = extractor.getContentAsNLM(null); + } + XMLOutputter outputter = new XMLOutputter(Format.getPrettyFormat()); + DocType dt = new DocType("article", "-//NLM//DTD JATS (Z39.96) Journal Archiving and Interchange DTD v1.0 20120330//EN", "JATS-archivearticle1.dtd"); + FileUtils.writeStringToFile(outputs.get("jats"), outputter.outputString(dt), "UTF-8"); + FileUtils.writeStringToFile(outputs.get("jats"), "\n", "UTF-8", true); + FileUtils.writeStringToFile(outputs.get("jats"), outputter.outputString(jats), "UTF-8", true); + } + + if (outputs.containsKey("trueviz")) { + BxDocument doc = extractor.getBxDocumentWithSpecificLabels(); + BxDocumentToTrueVizWriter writer = new BxDocumentToTrueVizWriter(); + Writer fw = new OutputStreamWriter(new FileOutputStream(outputs.get("trueviz")), "UTF-8"); + writer.write(fw, Lists.newArrayList(doc), "UTF-8"); + } + + if (outputs.containsKey("zones")) { + Element text = extractor.getLabelledFullText(); + XMLOutputter outputter = new XMLOutputter(Format.getPrettyFormat()); + FileUtils.writeStringToFile(outputs.get("zones"), outputter.outputString(text), "UTF-8"); + } + + if (outputs.containsKey("text")) { + String text = extractor.getRawFullText(); + FileUtils.writeStringToFile(outputs.get("text"), text, "UTF-8"); + } + + } catch (AnalysisException ex) { + printException(ex); + } catch (TransformationException ex) { + printException(ex); + } catch (TimeoutException ex) { + printException(ex); + }catch (IOException ex) { + printException(ex); + }catch (Exception ex) { + System.out.println("OOps.........."); + } finally { + if (extractor != null) { + extractor.removeTimeout(); + } + long end = System.currentTimeMillis(); + elapsed = (end - start) / 1000F; + } + + } else if (iter >= stop_point){ + break; + } + iter++; + } + } + } + Thread myThreads[] = new Thread[workers.intValue()]; + Long chunk = (stop - start)/workers; + for(int i=0; i < workers; i++){ + stop = start + chunk; + myThreads[i] = new Thread(new paraTask(builder, files, extensions,start, stop, override, timeoutSeconds, parser, outpath)); + myThreads[i].start(); + start = stop; + } + + for (int j = 0; j < workers; j++) { + try{ + myThreads[j].join(); //todo add catch exception + } catch (InterruptedException e){ + printException(e); + } + } + } + public static void main(String[] args) throws ParseException, AnalysisException, IOException, TransformationException { CommandLineOptionsParser parser = new CommandLineOptionsParser(); String error = parser.parse(args); @@ -736,13 +885,17 @@ public static void main(String[] args) throws ParseException, AnalysisException, + "Tool for extracting metadata and content from PDF files.\n\n" + "Arguments:\n" + " -path path to a directory containing PDF files\n" + + " -start which point to start parsing the PDF files\n" + + " -stop which point to stop parsing the PDF files\n" + + " -workers how many workers to run in parallel\n" + + " -outpath path to a directory to write the resulting files\n" + " -outputs (optional) comma-separated list of extraction\n" + " output(s); possible values: \"jats\" (document\n" + " metadata and content in NLM JATS format), \"text\"\n" + " (raw document text), \"zones\" (text zones with\n" + " their labels), \"trueviz\" (geometric structure in\n" + " TrueViz format), \"images\" (images from the\n" - + " document); default: \"jats,images\"\n" + + " document); default: \"jats\"\n" + " -exts (optional) comma-separated list of extensions of the\n" + " resulting files; the list has to have the same\n" + " length as output list; default: \"cermxml,images\"\n" @@ -763,104 +916,31 @@ public static void main(String[] args) throws ParseException, AnalysisException, Long timeoutSeconds = parser.getTimeout(); String path = parser.getPath(); + String outpath = parser.getOutPath()+'/'; Map extensions = parser.getTypesAndExtensions(); File file = new File(path); Collection files = FileUtils.listFiles(file, new String[]{"pdf"}, true); + Long start_point = parser.getStart(); + Long stop_point = parser.getStop(); + if (stop_point == null){ + stop_point = new Long(files.size()); + } + + Long workers = parser.getWorkers(); + if(workers > (stop_point - start_point)){ + workers = stop_point - start_point; + } + ExtractionConfigBuilder builder = new ExtractionConfigBuilder(); if (parser.getConfigurationPath() != null) { builder.addConfiguration(parser.getConfigurationPath()); } builder.setProperty(ExtractionConfigProperty.IMAGES_EXTRACTION, extensions.containsKey("images")); ExtractionConfigRegister.set(builder.buildConfiguration()); - - int i = 0; - for (File pdf : files) { - Map outputs = new HashMap(); - for (Map.Entry entry : extensions.entrySet()) { - File outputFile = getOutputFile(pdf, entry.getValue()); - if (override || !outputFile.exists()) { - outputs.put(entry.getKey(), outputFile); - } - } - if (outputs.isEmpty()) { - i++; - continue; - } - - long start = System.currentTimeMillis(); - float elapsed; - - System.out.println("File processed: " + pdf.getPath()); - - ContentExtractor extractor = null; - try { - extractor = createContentExtractor(timeoutSeconds); - - InputStream in = new FileInputStream(pdf); - extractor.setPDF(in); - - if (outputs.containsKey("images")) { - List images = extractor.getImages(outputs.get("images").getPath()); - FileUtils.forceMkdir(outputs.get("images")); - for (BxImage image : images) { - ImageIO.write(image.getImage(), "png", new File(image.getPath())); - } - } - - if (outputs.containsKey("jats")) { - Element jats; - if (outputs.containsKey("images")) { - jats = extractor.getContentAsNLM(outputs.get("images").getPath()); - } else { - jats = extractor.getContentAsNLM(null); - } - XMLOutputter outputter = new XMLOutputter(Format.getPrettyFormat()); - DocType dt = new DocType("article", "-//NLM//DTD JATS (Z39.96) Journal Archiving and Interchange DTD v1.0 20120330//EN", "JATS-archivearticle1.dtd"); - FileUtils.writeStringToFile(outputs.get("jats"), outputter.outputString(dt), "UTF-8"); - FileUtils.writeStringToFile(outputs.get("jats"), "\n", "UTF-8", true); - FileUtils.writeStringToFile(outputs.get("jats"), outputter.outputString(jats), "UTF-8", true); - } - - if (outputs.containsKey("trueviz")) { - BxDocument doc = extractor.getBxDocumentWithSpecificLabels(); - BxDocumentToTrueVizWriter writer = new BxDocumentToTrueVizWriter(); - Writer fw = new OutputStreamWriter(new FileOutputStream(outputs.get("trueviz")), "UTF-8"); - writer.write(fw, Lists.newArrayList(doc), "UTF-8"); - } - - if (outputs.containsKey("zones")) { - Element text = extractor.getLabelledFullText(); - XMLOutputter outputter = new XMLOutputter(Format.getPrettyFormat()); - FileUtils.writeStringToFile(outputs.get("zones"), outputter.outputString(text), "UTF-8"); - } - - if (outputs.containsKey("text")) { - String text = extractor.getRawFullText(); - FileUtils.writeStringToFile(outputs.get("text"), text, "UTF-8"); - } - - } catch (AnalysisException ex) { - printException(ex); - } catch (TransformationException ex) { - printException(ex); - } catch (TimeoutException ex) { - printException(ex); - } finally { - if (extractor != null) { - extractor.removeTimeout(); - } - long end = System.currentTimeMillis(); - elapsed = (end - start) / 1000F; - } - - i++; - int percentage = i * 100 / files.size(); - System.out.println("Extraction time: " + Math.round(elapsed) + "s"); - System.out.println("Progress: " + percentage + "% done (" + i + " out of " + files.size() + ")"); - System.out.println(""); - } + my_parallel_parser(builder, files, extensions,start_point, stop_point, override, timeoutSeconds, parser, workers, outpath); + } private static ContentExtractor createContentExtractor(Long timeoutSeconds) @@ -875,8 +955,8 @@ private static ContentExtractor createContentExtractor(Long timeoutSeconds) return extractor; } - private static File getOutputFile(File pdf, String ext) { - return new File(pdf.getPath().replaceFirst("pdf$", ext)); + private static File getOutputFile(String outfile, File pdf, String ext) { + return new File(Paths.get(outfile+pdf.getName().replaceFirst("pdf$", ext)).toString()); } private static void printException(Exception ex) { From b7e5408a6cf91060e7679865984e5c7f4c037bc6 Mon Sep 17 00:00:00 2001 From: Uchman21 Date: Mon, 27 Mar 2017 16:24:18 +0300 Subject: [PATCH 2/6] Added some new options and parallelization --- .../main/java/pl/edu/icm/cermine/ContentExtractor.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java b/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java index 197e7d7f..1719707c 100644 --- a/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java +++ b/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java @@ -841,7 +841,7 @@ public void runParallel() throws ParseException, AnalysisException, IOException, }catch (IOException ex) { printException(ex); }catch (Exception ex) { - System.out.println("OOps.........."); + printException(ex); } finally { if (extractor != null) { extractor.removeTimeout(); @@ -885,10 +885,10 @@ public static void main(String[] args) throws ParseException, AnalysisException, + "Tool for extracting metadata and content from PDF files.\n\n" + "Arguments:\n" + " -path path to a directory containing PDF files\n" - + " -start which point to start parsing the PDF files\n" - + " -stop which point to stop parsing the PDF files\n" - + " -workers how many workers to run in parallel\n" - + " -outpath path to a directory to write the resulting files\n" + + " -start (optional) which point to start parsing the PDF files default: \"beginning\"\n" + + " -stop (optional) which point to stop parsing the PDF files default: \"end\"\n" + + " -workers (optional) how many workers to run in parallel default: \"1\"\n" + + " -outpath (optional) path to a directory to write the resulting files default: \"-path\"\n" + " -outputs (optional) comma-separated list of extraction\n" + " output(s); possible values: \"jats\" (document\n" + " metadata and content in NLM JATS format), \"text\"\n" From 145b4484afb96cbd6bda8b8b7149ddf95667d5e3 Mon Sep 17 00:00:00 2001 From: Uchman21 Date: Thu, 13 Apr 2017 19:48:52 +0300 Subject: [PATCH 3/6] Better implementation for performance Improved performance --- .../icm/cermine/CommandLineOptionsParser.java | 30 +- .../pl/edu/icm/cermine/ContentExtractor.java | 335 ++++++++++-------- 2 files changed, 186 insertions(+), 179 deletions(-) diff --git a/cermine-impl/src/main/java/pl/edu/icm/cermine/CommandLineOptionsParser.java b/cermine-impl/src/main/java/pl/edu/icm/cermine/CommandLineOptionsParser.java index d03d890d..8f4aa76b 100644 --- a/cermine-impl/src/main/java/pl/edu/icm/cermine/CommandLineOptionsParser.java +++ b/cermine-impl/src/main/java/pl/edu/icm/cermine/CommandLineOptionsParser.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.Map; import org.apache.commons.cli.*; -import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.ArrayUtils; /** * @author Dominika Tkaczyk (d.tkaczyk@icm.edu.pl) @@ -38,8 +38,7 @@ public class CommandLineOptionsParser { public CommandLineOptionsParser() { options = new Options(); options.addOption("path", true, "file or directory path"); - options.addOption("start", true, "start point"); - options.addOption("stop", true, "stop point"); + options.addOption("limit", true, "number of pdfs to parse starting from the top"); options.addOption("workers", true, "number of workers"); options.addOption("outpath", true, "file or directory path for output"); options.addOption("outputs", true, "types of the output"); @@ -76,11 +75,11 @@ public String parse(String[] args) throws ParseException { return null; } - public String getPath() { + public String getPath () { return commandLine.getOptionValue("path"); } - public String getOutPath() { + public String getOutPath () { if (!commandLine.hasOption("outpath")) { return commandLine.getOptionValue("path"); } else { @@ -88,26 +87,13 @@ public String getOutPath() { } } - public Long getStart() { - if (!commandLine.hasOption("start")) { - return 0L; - } else { - Long value = Long.parseLong(commandLine.getOptionValue("start")); - if (value < 0) { - throw new RuntimeException("The 'start' value given as a " - + "command line parameter has to be nonnegative."); - } - return value; - } - } - - public Long getStop() { - if (!commandLine.hasOption("stop")) { + public Long getLimit() { + if (!commandLine.hasOption("limit")) { return null; } else { - Long value = Long.parseLong(commandLine.getOptionValue("stop")); + Long value = Long.parseLong(commandLine.getOptionValue("limit")); if (value < 0) { - throw new RuntimeException("The 'stop' value given as a " + throw new RuntimeException("The 'start' value given as a " + "command line parameter has to be nonnegative."); } return value; diff --git a/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java b/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java index 1719707c..602336c7 100644 --- a/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java +++ b/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java @@ -22,8 +22,17 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Paths; +import java.nio.file.Path; +import java.nio.file.Files; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.FileVisitResult; +import java.nio.file.attribute.BasicFileAttributes; import java.util.Collection; import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.net.URI; +import java.net.URISyntaxException; import org.apache.commons.cli.ParseException; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.exception.ExceptionUtils; @@ -51,7 +60,7 @@ import pl.edu.icm.cermine.structure.transformers.BxDocumentToTrueVizWriter; import pl.edu.icm.cermine.tools.timeout.Timeout; import pl.edu.icm.cermine.tools.timeout.TimeoutException; -import pl.edu.icm.cermine.tools.timeout.TimeoutRegister; +import pl.edu.icm.cermine.tools.timeout.TimeoutRegister; /** * Extracts content from PDF files. @@ -73,6 +82,8 @@ public class ContentExtractor { private Timeout mainTimeout = new Timeout(); + private static int progressCount = 0; + /** * Creates the object with overridden default configuration. * @@ -727,155 +738,126 @@ private Timeout combineWithMainTimeout(long timeoutSeconds) { return Timeout.min(mainTimeout, local); } - private static void my_parallel_parser(ExtractionConfigBuilder builder, Collection files, Map extensions,Long start, Long stop, boolean override, Long timeoutSeconds, CommandLineOptionsParser parser, Long workers, String outpath) throws ParseException, AnalysisException, IOException, TransformationException{ - class paraTask implements Runnable { - ExtractionConfigBuilder builder; - Collection files; - Map extensions; - Long start_point; - Long stop_point; - boolean override; - Long timeoutSeconds; - CommandLineOptionsParser parser; - String outpath; - - paraTask(ExtractionConfigBuilder m_builder, Collection m_files, Map m_extensions,Long m_start, Long m_stop, boolean m_override, Long m_timeoutSeconds, CommandLineOptionsParser m_parser, String m_outpath) { - start_point = m_start; - stop_point = m_stop; - builder = m_builder; - files = m_files; - extensions = m_extensions; - override = m_override; - timeoutSeconds = m_timeoutSeconds; - parser = m_parser; - outpath = m_outpath; - } - public void run() { - try{ - runParallel(); - }catch(IOException e) { - printException(e); - }catch(ParseException e) { - printException(e); - }catch(AnalysisException e) { - printException(e); - }catch(TransformationException e) { - printException(e); + synchronized void printProgress(int total) { + progressCount++; + int percentage = progressCount * 100 / total; + System.out.println("Progress: " + percentage + "% done (" + progressCount + " out of " + total + ")"); + System.out.println(""); + } + + private static class parallelTask implements Runnable { + File file; + Map extensions; + boolean override; + Long timeoutSeconds; + String outpath; + Long total; + + parallelTask ( File files, Map extensions, boolean override, Long timeoutSeconds, String outpath, Long limit) { + this.file = files; + this.extensions = extensions; + this.override = override; + this.timeoutSeconds = timeoutSeconds; + this.outpath = outpath; + this.total = limit; + } + + public void run() { + try { + runParallel(); + }catch(IOException e) { + printException(e); + }catch(ParseException e) { + printException(e); + }catch(AnalysisException e) { + printException(e); + }catch(TransformationException e) { + printException(e); + } + + } + + public void runParallel () throws ParseException, AnalysisException, IOException, TransformationException { + Map outputs = new HashMap(); + for (Map.Entry entry : extensions.entrySet()) { + File outputFile = getOutputFile(outpath, file, entry.getValue()); + if (override || !outputFile.exists()) { + outputs.put(entry.getKey(), outputFile); } - } - public void runParallel() throws ParseException, AnalysisException, IOException, TransformationException{ - int iter = 0; - for (File pdf : files) { - if (iter >= start_point && iter < stop_point){ - Map outputs = new HashMap(); - for (Map.Entry entry : extensions.entrySet()) { - File outputFile = getOutputFile(outpath, pdf, entry.getValue()); - if (override || !outputFile.exists()) { - outputs.put(entry.getKey(), outputFile); - } - } - if (outputs.isEmpty()) { - continue; - } - - long start = System.currentTimeMillis(); - float elapsed; - - System.out.println("File processed: " + pdf.getPath()); - - ContentExtractor extractor = null; - try { - extractor = createContentExtractor(timeoutSeconds); - - InputStream in = new FileInputStream(pdf); - extractor.setPDF(in); - - if (outputs.containsKey("images")) { - List images = extractor.getImages(outputs.get("images").getPath()); - FileUtils.forceMkdir(outputs.get("images")); - for (BxImage image : images) { - ImageIO.write(image.getImage(), "png", new File(image.getPath())); - } - } - - if (outputs.containsKey("jats")) { - Element jats; - if (outputs.containsKey("images")) { - jats = extractor.getContentAsNLM(outputs.get("images").getPath()); - } else { - jats = extractor.getContentAsNLM(null); - } - XMLOutputter outputter = new XMLOutputter(Format.getPrettyFormat()); - DocType dt = new DocType("article", "-//NLM//DTD JATS (Z39.96) Journal Archiving and Interchange DTD v1.0 20120330//EN", "JATS-archivearticle1.dtd"); - FileUtils.writeStringToFile(outputs.get("jats"), outputter.outputString(dt), "UTF-8"); - FileUtils.writeStringToFile(outputs.get("jats"), "\n", "UTF-8", true); - FileUtils.writeStringToFile(outputs.get("jats"), outputter.outputString(jats), "UTF-8", true); - } - - if (outputs.containsKey("trueviz")) { - BxDocument doc = extractor.getBxDocumentWithSpecificLabels(); - BxDocumentToTrueVizWriter writer = new BxDocumentToTrueVizWriter(); - Writer fw = new OutputStreamWriter(new FileOutputStream(outputs.get("trueviz")), "UTF-8"); - writer.write(fw, Lists.newArrayList(doc), "UTF-8"); - } - - if (outputs.containsKey("zones")) { - Element text = extractor.getLabelledFullText(); - XMLOutputter outputter = new XMLOutputter(Format.getPrettyFormat()); - FileUtils.writeStringToFile(outputs.get("zones"), outputter.outputString(text), "UTF-8"); - } - - if (outputs.containsKey("text")) { - String text = extractor.getRawFullText(); - FileUtils.writeStringToFile(outputs.get("text"), text, "UTF-8"); - } - - } catch (AnalysisException ex) { - printException(ex); - } catch (TransformationException ex) { - printException(ex); - } catch (TimeoutException ex) { - printException(ex); - }catch (IOException ex) { - printException(ex); - }catch (Exception ex) { - printException(ex); - } finally { - if (extractor != null) { - extractor.removeTimeout(); - } - long end = System.currentTimeMillis(); - elapsed = (end - start) / 1000F; - } - - } else if (iter >= stop_point){ - break; + if (outputs.isEmpty()) { + return; + } + + + ContentExtractor extractor = null; + try { + extractor = createContentExtractor(timeoutSeconds); + + InputStream in = new FileInputStream(file); + extractor.setPDF(in); + + if (outputs.containsKey("images")) { + List images = extractor.getImages(outputs.get("images").getPath()); + FileUtils.forceMkdir(outputs.get("images")); + for (BxImage image : images) { + ImageIO.write(image.getImage(), "png", new File(image.getPath())); } - iter++; } - } - } - Thread myThreads[] = new Thread[workers.intValue()]; - Long chunk = (stop - start)/workers; - for(int i=0; i < workers; i++){ - stop = start + chunk; - myThreads[i] = new Thread(new paraTask(builder, files, extensions,start, stop, override, timeoutSeconds, parser, outpath)); - myThreads[i].start(); - start = stop; - } - - for (int j = 0; j < workers; j++) { - try{ - myThreads[j].join(); //todo add catch exception - } catch (InterruptedException e){ - printException(e); + + if (outputs.containsKey("jats")) { + Element jats; + if (outputs.containsKey("images")) { + jats = extractor.getContentAsNLM(outputs.get("images").getPath()); + } else { + jats = extractor.getContentAsNLM(null); + } + XMLOutputter outputter = new XMLOutputter(Format.getPrettyFormat()); + DocType dt = new DocType("article", "-//NLM//DTD JATS (Z39.96) Journal Archiving and Interchange DTD v1.0 20120330//EN", "JATS-archivearticle1.dtd"); + FileUtils.writeStringToFile(outputs.get("jats"), outputter.outputString(dt), "UTF-8"); + FileUtils.writeStringToFile(outputs.get("jats"), "\n", "UTF-8", true); + FileUtils.writeStringToFile(outputs.get("jats"), outputter.outputString(jats), "UTF-8", true); + } + + if (outputs.containsKey("trueviz")) { + BxDocument doc = extractor.getBxDocumentWithSpecificLabels(); + BxDocumentToTrueVizWriter writer = new BxDocumentToTrueVizWriter(); + Writer fw = new OutputStreamWriter(new FileOutputStream(outputs.get("trueviz")), "UTF-8"); + writer.write(fw, Lists.newArrayList(doc), "UTF-8"); + } + + if (outputs.containsKey("zones")) { + Element text = extractor.getLabelledFullText(); + XMLOutputter outputter = new XMLOutputter(Format.getPrettyFormat()); + FileUtils.writeStringToFile(outputs.get("zones"), outputter.outputString(text), "UTF-8"); + } + + if (outputs.containsKey("text")) { + String text = extractor.getRawFullText(); + FileUtils.writeStringToFile(outputs.get("text"), text, "UTF-8"); + } + + } catch (AnalysisException ex) { + printException(ex); + } catch (TransformationException ex) { + printException(ex); + } catch (TimeoutException ex) { + printException(ex); + }catch (IOException ex) { + printException(ex); + }catch (Exception ex) { + printException(ex); + } finally { + if (extractor != null) { + extractor.printProgress(total.intValue()); + extractor.removeTimeout(); + } } } } - public static void main(String[] args) throws ParseException, AnalysisException, IOException, TransformationException { + public static void main(String[] args) throws ParseException, AnalysisException, URISyntaxException, IOException, TransformationException { CommandLineOptionsParser parser = new CommandLineOptionsParser(); String error = parser.parse(args); if (error != null) { @@ -885,8 +867,7 @@ public static void main(String[] args) throws ParseException, AnalysisException, + "Tool for extracting metadata and content from PDF files.\n\n" + "Arguments:\n" + " -path path to a directory containing PDF files\n" - + " -start (optional) which point to start parsing the PDF files default: \"beginning\"\n" - + " -stop (optional) which point to stop parsing the PDF files default: \"end\"\n" + + " -limit (optional) the number of PDF files to parse starting from the top default: \"all\"\n" + " -workers (optional) how many workers to run in parallel default: \"1\"\n" + " -outpath (optional) path to a directory to write the resulting files default: \"-path\"\n" + " -outputs (optional) comma-separated list of extraction\n" @@ -912,34 +893,74 @@ public static void main(String[] args) throws ParseException, AnalysisException, System.exit(1); } - boolean override = parser.override(); - Long timeoutSeconds = parser.getTimeout(); + final boolean override = parser.override(); + final Long timeoutSeconds = parser.getTimeout(); String path = parser.getPath(); - String outpath = parser.getOutPath()+'/'; - Map extensions = parser.getTypesAndExtensions(); + final String outpath = parser.getOutPath()+'/'; + final Map extensions = parser.getTypesAndExtensions(); File file = new File(path); Collection files = FileUtils.listFiles(file, new String[]{"pdf"}, true); - Long start_point = parser.getStart(); - Long stop_point = parser.getStop(); - if (stop_point == null){ - stop_point = new Long(files.size()); + Long getlimit = parser.getLimit(); + Long workers = parser.getWorkers(); + + if (getlimit == null || getlimit > files.size()) { + getlimit = new Long(files.size()); } - Long workers = parser.getWorkers(); - if(workers > (stop_point - start_point)){ - workers = stop_point - start_point; + if (workers > getlimit) { + workers = getlimit; } ExtractionConfigBuilder builder = new ExtractionConfigBuilder(); if (parser.getConfigurationPath() != null) { builder.addConfiguration(parser.getConfigurationPath()); } + + final Long limit = getlimit; builder.setProperty(ExtractionConfigProperty.IMAGES_EXTRACTION, extensions.containsKey("images")); ExtractionConfigRegister.set(builder.buildConfiguration()); - my_parallel_parser(builder, files, extensions,start_point, stop_point, override, timeoutSeconds, parser, workers, outpath); + final ExecutorService executor = Executors.newFixedThreadPool(workers.intValue()); + + final Path startFilePath = Paths.get(new URI("file://"+path)); + + long start = System.currentTimeMillis(); + float elapsed; + + Files.walkFileTree(startFilePath, new SimpleFileVisitor() { + int counter = 0; + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (file.toString().toLowerCase().endsWith(".pdf") && counter < limit) { + Runnable worker = new parallelTask(file.toFile(), extensions, override, timeoutSeconds, outpath, limit); + executor.execute(worker); + counter++; + } else if (counter >= limit) { + return FileVisitResult.SKIP_SIBLINGS; + } + return FileVisitResult.CONTINUE; + } + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { + if (e == null) { + if (dir == startFilePath) { + executor.shutdown(); + while (!executor.isTerminated()) { } + } + return FileVisitResult.CONTINUE; + } else { + // directory iteration failed + executor.shutdown(); + throw e; + } + } + }); + + long end = System.currentTimeMillis(); + elapsed = (end - start) / 1000F; + System.out.println("Total extraction time: " + Math.round(elapsed) + "s"); } @@ -955,8 +976,8 @@ private static ContentExtractor createContentExtractor(Long timeoutSeconds) return extractor; } - private static File getOutputFile(String outfile, File pdf, String ext) { - return new File(Paths.get(outfile+pdf.getName().replaceFirst("pdf$", ext)).toString()); + private static File getOutputFile(String outfile, File file, String ext) { + return new File(Paths.get(outfile+file.getName().replaceFirst("pdf$", ext)).toString()); } private static void printException(Exception ex) { From 0fabc9aa1e22df21afa43b2ffc0da0c7d1a05e3d Mon Sep 17 00:00:00 2001 From: Uchman21 Date: Thu, 13 Apr 2017 21:02:13 +0300 Subject: [PATCH 4/6] Corrected a performance bug --- .../main/java/pl/edu/icm/cermine/ContentExtractor.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java b/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java index 602336c7..0ca1e823 100644 --- a/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java +++ b/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java @@ -900,14 +900,12 @@ public static void main(String[] args) throws ParseException, AnalysisException, final String outpath = parser.getOutPath()+'/'; final Map extensions = parser.getTypesAndExtensions(); - File file = new File(path); - Collection files = FileUtils.listFiles(file, new String[]{"pdf"}, true); - + Long fileLength = Files.list(Paths.get(path)).count(); Long getlimit = parser.getLimit(); Long workers = parser.getWorkers(); - if (getlimit == null || getlimit > files.size()) { - getlimit = new Long(files.size()); + if (getlimit == null || getlimit > fileLength) { + getlimit = fileLength; } if (workers > getlimit) { From 99b2b8e655af150242c7a334365934ffa782248b Mon Sep 17 00:00:00 2001 From: Uchman21 Date: Tue, 20 Jun 2017 19:21:17 +0300 Subject: [PATCH 5/6] Parallel extraction fix Addressed all the issues raised from the previous versions. Passed all unit test. Fixed some bugs. Should be better now. --- .../icm/cermine/CommandLineOptionsParser.java | 26 ++-- .../pl/edu/icm/cermine/ContentExtractor.java | 137 +++++++++--------- 2 files changed, 85 insertions(+), 78 deletions(-) diff --git a/cermine-impl/src/main/java/pl/edu/icm/cermine/CommandLineOptionsParser.java b/cermine-impl/src/main/java/pl/edu/icm/cermine/CommandLineOptionsParser.java index 8f4aa76b..e79c53c1 100644 --- a/cermine-impl/src/main/java/pl/edu/icm/cermine/CommandLineOptionsParser.java +++ b/cermine-impl/src/main/java/pl/edu/icm/cermine/CommandLineOptionsParser.java @@ -38,7 +38,7 @@ public class CommandLineOptionsParser { public CommandLineOptionsParser() { options = new Options(); options.addOption("path", true, "file or directory path"); - options.addOption("limit", true, "number of pdfs to parse starting from the top"); + options.addOption("limit", true, "maximum number of files to process"); options.addOption("workers", true, "number of workers"); options.addOption("outpath", true, "file or directory path for output"); options.addOption("outputs", true, "types of the output"); @@ -75,40 +75,44 @@ public String parse(String[] args) throws ParseException { return null; } - public String getPath () { + public String getPath() { return commandLine.getOptionValue("path"); } - public String getOutPath () { + public String getOutPath() { if (!commandLine.hasOption("outpath")) { - return commandLine.getOptionValue("path"); + return null; } else { return commandLine.getOptionValue("outpath"); } } - public Long getLimit() { + public int getLimit() { if (!commandLine.hasOption("limit")) { - return null; + return -1; } else { - Long value = Long.parseLong(commandLine.getOptionValue("limit")); + int value = Integer.parseInt(commandLine.getOptionValue("limit")); if (value < 0) { - throw new RuntimeException("The 'start' value given as a " + throw new RuntimeException("The 'limit' value given as a " + "command line parameter has to be nonnegative."); } return value; } } - public Long getWorkers() { + public int getWorkers(int limit) { if (!commandLine.hasOption("workers")) { - return 1L; + return 1; } else { - Long value = Long.parseLong(commandLine.getOptionValue("workers")); + int value = Integer.parseInt(commandLine.getOptionValue("workers")); if (value <= 0) { throw new RuntimeException("The 'workers' value given as a " + "command line parameter has to be nonnegative."); } + + if (value > limit) { + value = limit; + } return value; } } diff --git a/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java b/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java index 0ca1e823..0850058a 100644 --- a/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java +++ b/cermine-impl/src/main/java/pl/edu/icm/cermine/ContentExtractor.java @@ -27,12 +27,10 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.FileVisitResult; import java.nio.file.attribute.BasicFileAttributes; -import java.util.Collection; import java.util.List; -import java.util.concurrent.ExecutorService; +import java.util.ArrayList; +import java.util.concurrent.CompletionService; import java.util.concurrent.Executors; -import java.net.URI; -import java.net.URISyntaxException; import org.apache.commons.cli.ParseException; import org.apache.commons.io.FileUtils; import org.apache.commons.lang.exception.ExceptionUtils; @@ -62,6 +60,12 @@ import pl.edu.icm.cermine.tools.timeout.TimeoutException; import pl.edu.icm.cermine.tools.timeout.TimeoutRegister; + +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; /** * Extracts content from PDF files. *

@@ -82,7 +86,8 @@ public class ContentExtractor { private Timeout mainTimeout = new Timeout(); - private static int progressCount = 0; + + private static ArrayList filePaths = new ArrayList(); /** * Creates the object with overridden default configuration. @@ -738,47 +743,32 @@ private Timeout combineWithMainTimeout(long timeoutSeconds) { return Timeout.min(mainTimeout, local); } - - synchronized void printProgress(int total) { - progressCount++; - int percentage = progressCount * 100 / total; - System.out.println("Progress: " + percentage + "% done (" + progressCount + " out of " + total + ")"); - System.out.println(""); + private static String parseOutputPath(String outpath, String currentPath) { + if (outpath == null) { + return currentPath + "/"; + } + return outpath + "/"; } - private static class parallelTask implements Runnable { + + private static class ParallelTask implements Callable { File file; Map extensions; boolean override; Long timeoutSeconds; String outpath; - Long total; - parallelTask ( File files, Map extensions, boolean override, Long timeoutSeconds, String outpath, Long limit) { + ParallelTask ( File files, Map extensions, boolean override, Long timeoutSeconds, String outpath) { this.file = files; this.extensions = extensions; this.override = override; this.timeoutSeconds = timeoutSeconds; this.outpath = outpath; - this.total = limit; - } - - public void run() { - try { - runParallel(); - }catch(IOException e) { - printException(e); - }catch(ParseException e) { - printException(e); - }catch(AnalysisException e) { - printException(e); - }catch(TransformationException e) { - printException(e); - } - } - public void runParallel () throws ParseException, AnalysisException, IOException, TransformationException { + public String call () throws ParseException, AnalysisException, IOException, TransformationException { + long start = System.currentTimeMillis(); + float elapsed; Map outputs = new HashMap(); for (Map.Entry entry : extensions.entrySet()) { File outputFile = getOutputFile(outpath, file, entry.getValue()); @@ -787,7 +777,9 @@ public void runParallel () throws ParseException, AnalysisException, IOException } } if (outputs.isEmpty()) { - return; + long end = System.currentTimeMillis(); + elapsed = (end - start) / 1000F; + return file.getAbsolutePath() + "; extraction time: " + Math.round(elapsed) + "s"; } @@ -850,14 +842,16 @@ public void runParallel () throws ParseException, AnalysisException, IOException printException(ex); } finally { if (extractor != null) { - extractor.printProgress(total.intValue()); extractor.removeTimeout(); } } + long end = System.currentTimeMillis(); + elapsed = (end - start) / 1000F; + return file.getAbsolutePath() + "; extraction time: " + Math.round(elapsed) + "s"; } } - public static void main(String[] args) throws ParseException, AnalysisException, URISyntaxException, IOException, TransformationException { + public static void main(String[] args) throws ParseException, AnalysisException, IOException, TransformationException { CommandLineOptionsParser parser = new CommandLineOptionsParser(); String error = parser.parse(args); if (error != null) { @@ -867,9 +861,9 @@ public static void main(String[] args) throws ParseException, AnalysisException, + "Tool for extracting metadata and content from PDF files.\n\n" + "Arguments:\n" + " -path path to a directory containing PDF files\n" - + " -limit (optional) the number of PDF files to parse starting from the top default: \"all\"\n" - + " -workers (optional) how many workers to run in parallel default: \"1\"\n" - + " -outpath (optional) path to a directory to write the resulting files default: \"-path\"\n" + + " -limit (optional) max number of files to process; default: \"all\"\n" + + " -workers (optional) number of workers to do task; default: \"1\"\n" + + " -outpath (optional) output file location; default: \"-path\"\n" + " -outputs (optional) comma-separated list of extraction\n" + " output(s); possible values: \"jats\" (document\n" + " metadata and content in NLM JATS format), \"text\"\n" @@ -897,45 +891,27 @@ public static void main(String[] args) throws ParseException, AnalysisException, final Long timeoutSeconds = parser.getTimeout(); String path = parser.getPath(); - final String outpath = parser.getOutPath()+'/'; + final String outpath = parser.getOutPath(); final Map extensions = parser.getTypesAndExtensions(); - - Long fileLength = Files.list(Paths.get(path)).count(); - Long getlimit = parser.getLimit(); - Long workers = parser.getWorkers(); - - if (getlimit == null || getlimit > fileLength) { - getlimit = fileLength; - } - - if (workers > getlimit) { - workers = getlimit; - } + final int limit = parser.getLimit(); ExtractionConfigBuilder builder = new ExtractionConfigBuilder(); if (parser.getConfigurationPath() != null) { builder.addConfiguration(parser.getConfigurationPath()); } - final Long limit = getlimit; builder.setProperty(ExtractionConfigProperty.IMAGES_EXTRACTION, extensions.containsKey("images")); ExtractionConfigRegister.set(builder.buildConfiguration()); - final ExecutorService executor = Executors.newFixedThreadPool(workers.intValue()); - - final Path startFilePath = Paths.get(new URI("file://"+path)); - - long start = System.currentTimeMillis(); - float elapsed; + final Path startFilePath = Paths.get(path); Files.walkFileTree(startFilePath, new SimpleFileVisitor() { int counter = 0; @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (file.toString().toLowerCase().endsWith(".pdf") && counter < limit) { - Runnable worker = new parallelTask(file.toFile(), extensions, override, timeoutSeconds, outpath, limit); - executor.execute(worker); + if (file.toString().toLowerCase().endsWith(".pdf") && ((limit != -1 && counter < limit) || limit == -1)) { + filePaths.add(file); counter++; - } else if (counter >= limit) { + } else if (limit != -1 && counter >= limit) { return FileVisitResult.SKIP_SIBLINGS; } return FileVisitResult.CONTINUE; @@ -943,19 +919,46 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO @Override public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException { if (e == null) { - if (dir == startFilePath) { - executor.shutdown(); - while (!executor.isTerminated()) { } - } return FileVisitResult.CONTINUE; } else { - // directory iteration failed - executor.shutdown(); throw e; } } }); + + int progressCount = 0; + int fileLength = filePaths.size(); + int workers = parser.getWorkers(fileLength); + final ExecutorService executor = Executors.newFixedThreadPool(workers); + final CompletionService completionService = new ExecutorCompletionService(executor); + + for (Path file : filePaths) { + completionService.submit(new ParallelTask(file.toFile(), extensions, override, timeoutSeconds, parseOutputPath(outpath,file.getParent().toString()))); + } + + long start = System.currentTimeMillis(); + float elapsed; + + try { + for (int t = 0, n = fileLength; t < n; t++) { + Future processedFile = completionService.take(); + System.out.println("File processed : " + processedFile.get()); + progressCount++; + int percentage = progressCount * 100 / fileLength; + System.out.println("Progress: " + percentage + "% done (" + progressCount + " out of " + fileLength + ")"); + System.out.println(""); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } catch (ExecutionException e) { + Thread.currentThread().interrupt(); + } finally { + if (executor != null) { + executor.shutdownNow(); + } + } + long end = System.currentTimeMillis(); elapsed = (end - start) / 1000F; System.out.println("Total extraction time: " + Math.round(elapsed) + "s"); From 92bfde6e5fba3b924307b75d4896cf57a3054bc7 Mon Sep 17 00:00:00 2001 From: Uchman21 Date: Tue, 20 Jun 2017 19:25:14 +0300 Subject: [PATCH 6/6] New option tests Added test for the new options. --- .../icm/cermine/CommandLineOptionsParserTest.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/cermine-impl/src/test/java/pl/edu/icm/cermine/CommandLineOptionsParserTest.java b/cermine-impl/src/test/java/pl/edu/icm/cermine/CommandLineOptionsParserTest.java index 024cae73..601e513e 100644 --- a/cermine-impl/src/test/java/pl/edu/icm/cermine/CommandLineOptionsParserTest.java +++ b/cermine-impl/src/test/java/pl/edu/icm/cermine/CommandLineOptionsParserTest.java @@ -58,9 +58,8 @@ public void parse_WITH_PATH() throws ParseException { assertEquals("/path/to/pdfs/folder", cmdLineOptionsParser.getPath()); Map typesAndExtensions = cmdLineOptionsParser.getTypesAndExtensions(); - assertEquals(2, typesAndExtensions.size()); + assertEquals(1, typesAndExtensions.size()); assertEquals("cermxml", typesAndExtensions.get("jats")); - assertEquals("images", typesAndExtensions.get("images")); assertFalse(cmdLineOptionsParser.override()); assertNull(cmdLineOptionsParser.getTimeout()); @@ -84,7 +83,10 @@ public void parse_OVERRIDE_DEFAULTS() throws ParseException { "-configuration", "config.properties", "-ext", "xml2", "-str", - "-strext", "xml3",}); + "-strext", "xml3", + "-limit", "20", + "-outpath", "/path/to/pdfs/folder", + "-workers", "1"}); // assert assertNull(error); @@ -104,6 +106,9 @@ public void parse_OVERRIDE_DEFAULTS() throws ParseException { assertEquals("xml2", cmdLineOptionsParser.getTextExtension()); assertTrue(cmdLineOptionsParser.extractStructure()); assertEquals("xml3", cmdLineOptionsParser.getBxExtension()); + assertEquals(20, cmdLineOptionsParser.getLimit()); + assertEquals("/path/to/pdfs/folder", cmdLineOptionsParser.getOutPath()); + assertEquals(1, cmdLineOptionsParser.getWorkers(20)); } @Test @@ -111,6 +116,7 @@ public void parse_INVALID_EXTENSIONS_LIST_SIZE() throws ParseException { // execute String error = cmdLineOptionsParser.parse(new String[] { "-path", "/path/to/pdfs/folder", + "-outpath", "/path/to/pdfs/folder", "-outputs", "jats,zones,trueviz", "-exts", "xml,xml2"});