PDF 解析新纪元：用 OpenDataLoader-PDF 让 AI 真正“读懂”文档 📄🤖

// 示例：提取的表格可能被结构化为如下 JSON
{
  "type": "table",
  "bbox": [x0, y0, x1, y1],
  "data": [
    ["季度", "产品 A 销量", "产品 B 销量"],
    ["Q1", "15,000", "8,200"],
    ["Q2", "18,500", "9,100"]
  ],
  "headers": [{"row": 0, "col": 0, "text": "季度"}, ...]
}

pip install opendataloader-pdf
# 或者从源码安装最新版
git clone https://github.com/opendataloader-project/opendataloader-pdf.git
cd opendataloader-pdf
pip install -e .

from opendataloader_pdf import PDFParser

# 初始化解析器
parser = PDFParser()

# 解析 PDF 文件
document = parser.parse("your_document.pdf")

# 访问解析后的内容
# 1. 获取所有文本块（带位置和样式信息）
for block in document.text_blocks:
    print(f"文本: {block.text}")
    print(f"  字体: {block.font_name}, 大小: {block.font_size}")
    print(f"  位置: {block.bbox}")

# 2. 获取所有提取的表格
for table in document.tables:
    # 转换为 pandas DataFrame 进行数据分析
    df = table.to_pandas()
    print(df.head())

# 3. 导出为结构化的 JSON（AI-ready!）
structured_output = document.to_json()
with open("output.json", "w") as f:
    import json
    json.dump(structured_output, f, indent=2, ensure_ascii=False)

# 4. 导出为可访问的 HTML
html_output = document.to_html()
with open("output.html", "w") as f:
    f.write(html_output)

# 利用解析出的标题层级进行智能切分
chunks = []
current_chunk = ""
current_section = ""

for element in document.elements: # elements 是包含语义类型的流
    if element.type == "heading":
        # 保存上一个块
        if current_chunk:
            chunks.append({"section": current_section, "content": current_chunk})
        # 开始新块
        current_section = element.text
        current_chunk = element.text + "\n"
    else:
        current_chunk += element.text + "\n"

# 这样的 chunk 包含完整上下文，极大提升检索准确率。

from langchain.schema import Document
from opendataloader_pdf import PDFParser

class OpenDataPDFLoader:
    def __init__(self, file_path):
        self.file_path = file_path

    def load(self):
        parser = PDFParser()
        od_document = parser.parse(self.file_path)
        # 将每个语义块或页面转换为一个 LangChain Document 对象
        langchain_docs = []
        for i, page in enumerate(od_document.pages):
            doc = Document(
                page_content=page.text,
                metadata={
                    "source": self.file_path,
                    "page": i+1,
                    "tables": json.dumps([t.to_dict() for t in page.tables])
                }
            )
            langchain_docs.append(doc)
        return langchain_docs