Extracting Text from PDF table

Preparation

Download and install python from https://www.python.org/downloads

open command prompt and run:

pip install pdfminer

This will install PDFMiner python library for working with PDF files

PDFMiner is a tool for extracting information from PDF documents. Unlike other PDF-related tools, it focuses entirely on getting and analyzing text data. PDFMiner allows obtaining the exact location of texts on a page, as well as other information such as fonts or lines. It includes a PDF converter that can transform PDF files into other text formats (such as HTML). It has an extensible PDF parser that can be used for other purposes instead of text analysis.

https://pypi.python.org/pypi/pdfminer/

Extracting data from PDF tables

With the help of stackoverflow we created pdftable2csv.py python script which can be downloaded from here

This command will convert PDF tables into csv file:

python C:\Python27\Scripts\pdftable2csv.py 2 CDE-Report1.pdf CDE-Report1.csv

Where “2” is the distance multiplier after which a character is considered part of a new word/column/block. Usually, 1.5 works quite well

Running python scripts from package

Is done by using “External application” Package Action:

Dealing with Encrypted PDF files

QPDF.exe is here to help

qpdf -password= --decrypt test.pdf test1.pdf

Note: Some pdf files are actually encrypted with empty passwords

Python 2 code

import sys

def pdf_to_csv(filename, separator, threshold):
    from cStringIO import StringIO
    from pdfminer.converter import LTChar, TextConverter
    from pdfminer.layout import LAParams
    from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
    from pdfminer.pdfpage import PDFPage

    class CsvConverter(TextConverter):
        def __init__(self, *args, **kwargs):
            TextConverter.__init__(self, *args, **kwargs)
            self.separator = separator
            self.threshold = threshold

        def end_page(self, i):
            from collections import defaultdict
            lines = defaultdict(lambda: {})
            for child in self.cur_item._objs:  # <-- changed
                if isinstance(child, LTChar):
                    (_, _, x, y) = child.bbox
                    line = lines[int(-y)]
                    line[x] = child._text.encode(self.codec)  # <-- changed
            for y in sorted(lines.keys()):
                line = lines[y]
                self.line_creator(line)
                self.outfp.write(self.line_creator(line))
                self.outfp.write("\n")

        def line_creator(self, line):
            keys = sorted(line.keys())
            # calculate the average distance between each character on this row
            average_distance = sum([keys[i] - keys[i - 1] for i in range(1, len(keys))]) / len(keys)
            # append the first character to the result
            result = [line[keys[0]]]
            for i in range(1, len(keys)):
                # if the distance between this character and the last character is greater than the average*threshold
                if (keys[i] - keys[i - 1]) > average_distance * self.threshold:
                    # append the separator into that position
                    result.append(self.separator)
                # append the character
                result.append(line[keys[i]])
            printable_line = ''.join(result)
            return printable_line

    # ... the following part of the code is a remix of the
    # convert() function in the pdfminer/tools/pdf2text module
    rsrc = PDFResourceManager()
    outfp = StringIO()
    device = CsvConverter(rsrc, outfp, codec="utf-8", laparams=LAParams())
    # because my test documents are utf-8 (note: utf-8 is the default codec)

    fp = open(filename, 'rb')

    interpreter = PDFPageInterpreter(rsrc, device)
    for i, page in enumerate(PDFPage.get_pages(fp)):
        outfp.write("START PAGE %d\n" % i)
        if page is not None:
            print 'none'
            interpreter.process_page(page)
        outfp.write("END PAGE %d\n" % i)

    device.close()
    fp.close()

    return outfp.getvalue()


if __name__ == '__main__':
    # the separator to use with the CSV
    separator = ';'
    # the distance multiplier after which a character is considered part of a new word/column/block. Usually 1.5 works quite well
    threshold = float(sys.argv[1])
    file = open(sys.argv[3], "wb")
    file.write(pdf_to_csv(sys.argv[2], separator, threshold))
    file.close()

Python 3 code

import sys
from io import StringIO
from pdfminer.converter import LTChar, TextConverter
from pdfminer.layout import LAParams
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.pdfpage import PDFPage
from collections import defaultdict

def pdf_to_csv(filename, separator, threshold):
    class CsvConverter(TextConverter):
        def __init__(self, *args, **kwargs):
            TextConverter.__init__(self, *args, **kwargs)
            self.separator = separator
            self.threshold = threshold

        def end_page(self, i):
            lines = defaultdict(lambda: {})
            for child in self.cur_item._objs:
                if isinstance(child, LTChar):
                    _, _, x, y = child.bbox
                    line = lines[int(-y)]
                    line[x] = child._text.encode(self.codec).decode(self.codec)  # Decode after encoding

            for y in sorted(lines.keys()):
                line = lines[y]
                self.outfp.write(self.line_creator(line))
                self.outfp.write("\n")

        def line_creator(self, line):
            keys = sorted(line.keys())
            if not keys:
                return ""  # Handle empty lines

            average_distance = sum([keys[i] - keys[i - 1] for i in range(1, len(keys))]) / (len(keys) - 1) if len(keys) > 1 else 0

            result = [line[keys[0]]]
            for i in range(1, len(keys)):
                if average_distance and (keys[i] - keys[i - 1]) > average_distance * self.threshold:
                    result.append(self.separator)
                result.append(line[keys[i]])
            printable_line = ''.join(result)
            return printable_line

    rsrc = PDFResourceManager()
    outfp = StringIO()
    device = CsvConverter(rsrc, outfp, codec="utf-8", laparams=LAParams())

    with open(filename, 'rb') as fp:
        interpreter = PDFPageInterpreter(rsrc, device)
        for i, page in enumerate(PDFPage.get_pages(fp)):
            outfp.write(f"START PAGE {i}\n")
            if page is not None:
                interpreter.process_page(page)
            outfp.write(f"END PAGE {i}\n")

    device.close()
    return outfp.getvalue()

if __name__ == '__main__':
    separator = ';'
    threshold = float(sys.argv[1])
    with open(sys.argv[3], "wb") as file:
        file.write(pdf_to_csv(sys.argv[2], separator, threshold).encode('utf-8')) #encode the string before writing in binary mode.