fix dependencies from pillow branch and fix processing of pdf files

Author: kreuzberger

libpdf/catalog.py

@@ -8,7 +8,7 @@
 from libpdf.progress import bar_format_lvl2, tqdm
 from libpdf.utils import decode_title, to_pdfplumber_bbox
 
-from pdfminer.pdftypes import PDFObjRef
+from pdfminer.pdftypes import PDFObjRef, resolve1
 from pdfminer.psparser import PSLiteral
 
 
@@ -48,7 +48,9 @@ def get_named_destination(pdf):  # pylint: disable=too-many-branches
         if isinstance(pdf_catalog['Names'], PDFObjRef) and 'Dests' in pdf_catalog['Names'].resolve():
             name_tree = pdf_catalog['Names'].resolve()['Dests'].resolve()
         elif isinstance(pdf_catalog['Names'], dict) and 'Dests' in pdf_catalog['Names']:
-            name_tree = pdf_catalog['Names']['Dests'].resolve()
+            name_tree = resolve1(pdf_catalog['Names']['Dests'])
+            #name_tree = pdf_catalog['Names']['Dests'].resolve()
+            #LOG.debug(f"{name_tree}")
         # check if name tree not empty
         if name_tree:
             # map page id to page number
@@ -405,7 +407,7 @@ def update_ann_info(annotation_page_map, ann_resolved, page, idx_page, pdf):  #
         page.height,
     )
     page_crop = page.within_bbox(ann_bbox)
-    ann_text = page_crop.extract_text(x_tolerance=1, y_tolerance=4)
+    ann_text = page_crop.extract_text(x_tolerance=float(1), y_tolerance=float(4))
 
     if 'A' in ann_resolved:
         # make sure ann_resolved['A'] is resolved

libpdf/extract.py

@@ -536,7 +536,7 @@ def extract_figures(
         lt_page = page._layout  # pylint: disable=protected-access  # easiest way to obtain LTPage
 
         # check and filter figures
-        figures = check_and_filter_figures(page_crop.figures)
+        figures = check_and_filter_figures(page_crop.objects['figure']) if 'figure' in page_crop.objects else []
 
         if len(figures) != 0:
             for idx_figure, figure in enumerate(figures):

libpdf/process.py

@@ -40,9 +40,9 @@ def remove_page_header_footer(single_page):
     page_crop = single_page.within_bbox(
         (
             0,
-            decimal.Decimal(parameters.PAGE_CROP_MARGINS['top']),
+            parameters.PAGE_CROP_MARGINS['top'],
             single_page.width,
-            single_page.height - decimal.Decimal(parameters.PAGE_CROP_MARGINS['bottom']),
+            single_page.height - parameters.PAGE_CROP_MARGINS['bottom'],
         ),
     )
 

libpdf/tables.py

@@ -16,18 +16,15 @@
 pdfminer's layout is used.
 """
 import logging
-from decimal import Decimal
 from typing import List, Union
 
-from libpdf import textbox
-from libpdf import utils
+from libpdf import textbox, utils
 from libpdf.catalog import catalog
 from libpdf.log import logging_needed
 from libpdf.models.figure import Figure
 from libpdf.models.page import Page
 from libpdf.models.position import Position
-from libpdf.models.table import Cell
-from libpdf.models.table import Table
+from libpdf.models.table import Cell, Table
 from libpdf.parameters import LA_PARAMS
 from libpdf.progress import bar_format_lvl2, tqdm
 from libpdf.utils import from_pdfplumber_bbox, lt_to_libpdf_hbox_converter
@@ -65,17 +62,21 @@ def extract_pdf_table(pdf, pages_list: List[Page], figure_list: List[Figure]):
         'explicit_vertical_lines': [],
         'explicit_horizontal_lines': [],
         'snap_tolerance': 3,
+        "snap_x_tolerance": 3,
+        "snap_y_tolerance": 3,
         'join_tolerance': 3,
+        "join_x_tolerance": 3,
+        "join_y_tolerance": 3,
         'edge_min_length': 3,
         'min_words_vertical': 3,
         'min_words_horizontal': 1,
-        'keep_blank_chars': False,
+        #'keep_blank_chars': False,
         'text_tolerance': 3,
         'text_x_tolerance': 2,
         'text_y_tolerance': 2,
         'intersection_tolerance': 3,
-        'intersection_x_tolerance': None,
-        'intersection_y_tolerance': None,
+        'intersection_x_tolerance': 3,
+        'intersection_y_tolerance': 3,
     }
 
     table_dict = {'page': {}}
@@ -157,7 +158,7 @@ def extract_cells(lt_page: LTPage, rows: List, list_cell: List[Cell], page: Page
                     row_cell[1],
                     row_cell[2],
                     row_cell[3],
-                    Decimal(lt_page.height),
+                    lt_page.height,
                 )
                 pos_cell = Position(pos_cell_bbox[0], pos_cell_bbox[1], pos_cell_bbox[2], pos_cell_bbox[3], page)
                 # extract cell text

libpdf/textbox.py

@@ -27,9 +27,9 @@
 pdfminer sees y0 and y1 from the bottom of the page, so y0 is smaller than y1.
 All coordinates are given in points where 72 points are 1 inch.
 """
+from difflib import SequenceMatcher
 import logging
 import re
-from difflib import SequenceMatcher
 from typing import Dict, List, Tuple, Union
 
 from libpdf import parameters
@@ -51,14 +51,7 @@
 )
 from libpdf.progress import bar_format_lvl2, tqdm
 from libpdf.utils import lt_page_crop, lt_to_libpdf_hbox_converter, textbox_crop
-
-from pdfminer.layout import (
-    LTAnno,
-    LTChar,
-    LTText,
-    LTTextBox,
-    LTTextLineHorizontal,
-)
+from pdfminer.layout import LTAnno, LTChar, LTText, LTTextBox, LTTextLineHorizontal
 
 
 LOG = logging.getLogger(__name__)
@@ -877,9 +870,9 @@ def pdfminer_get_lt_textboxes(pdf) -> Dict[int, List[LTTextBox]]:
         if logging_needed(idx_page, len(pdf.pages)):
             LOG.debug('Extracting layout page %s of %s', idx_page + 1, len(pdf.pages))
 
-        pdf.interpreter.process_page(page.page_obj)
-        result = pdf.device.get_result()
-        lt_textboxes = [obj for obj in result if isinstance(obj, LTTextBox)]
+        # pdf.interpreter.process_page(page.page_obj)
+        layout_objects = page.layout._objs
+        lt_textboxes = [obj for obj in layout_objects if isinstance(obj, LTTextBox)]
         # remove detected header and footer lt_textboxes based on given page crop margin parameter
         filter_lt_textboxes = list(
             filter(

libpdf/utils.py

@@ -164,10 +164,10 @@ def to_pdfplumber_bbox(x0, y0, x1, y1, page_height):
     :return: [x0, top, x1, bottom]
     """
     # pylint: disable=invalid-name  # short is better here
-    ret_x0 = Decimal(x0)
-    ret_y0 = Decimal(Decimal(page_height) - Decimal(y1))
-    ret_x1 = Decimal(x1)
-    ret_y1 = Decimal(Decimal(page_height) - Decimal(y0))
+    ret_x0 = x0
+    ret_y0 = page_height - y1
+    ret_x1 = x1
+    ret_y1 = page_height - y0
     return [ret_x0, ret_y0, ret_x1, ret_y1]
 
 

pyproject.toml

@@ -38,24 +38,26 @@ include = [
 
 [tool.poetry.dependencies]
 python = "^3.7"
-chardet = "^4"
-click = "^8"
+chardet = "*"
+click = "*"
 importlib-metadata = { version = "^1.6.0", python = "~3.7" }
-PyYAML = "^6"
-"ruamel.yaml" = "^0.17"
+PyYAML = "*"
+"ruamel.yaml" = "*"
+pdfplumber = "*"
+"pdfminer.six" = "*"
 
 # optional deps for progress bars
-tqdm = { version = "^4.50.0", optional = true }
-colorama = { version = "^0.4.4", optional = true }
+tqdm = { version = "*", optional = true }
+colorama = { version = "*", optional = true }
 
 # dependencies needed by pdfminer.six and pdfplumber which are deliverd as wheels in the deps folder
 # the libs were patched and no upstream PR has been raised yet
 # see [tool.poetry.dev-dependencies] for the forked libraries
-pycryptodome = "^3.9.9"
+pycryptodome = "*"
 sortedcontainers = "^2.3.0"
-pillow = "~9.0.1"  # cannot go higher because of this bug https://github.com/jsvine/pdfplumber/discussions/637
-unicodecsv = "^0.14.1"
-wand = "^0.6.5"
+pillow = "*"
+unicodecsv = "*"
+wand = "*"
 
 # optional deps for docs, needed to make RTD work with pyproject.toml
 # see https://github.com/readthedocs/readthedocs.org/issues/4912#issuecomment-664002569
@@ -73,8 +75,8 @@ docs = ["sphinx", "sphinx_rtd_theme", "sphinxcontrib-plantuml"]
 # they must be dev deps so make them invisible to PyPI and the egg/wheel requires section
 # 'poetry install' installs those by default, so the Git deps are actually used instead of the wheels in the deps
 # folder; see also libpdf/_import_forks.py
-pdfplumber = { git = "https://github.com/useblocks/pdfplumber.git" }
-"pdfminer.six" = {git = "https://github.com/useblocks/pdfminer.six", rev = "develop"}
+# pdfplumber = { git = "https://github.com/useblocks/pdfplumber.git" }
+# "pdfminer.six" = {git = "https://github.com/useblocks/pdfminer.six", rev = "develop"}
 
 # testing
 pytest = "^7"