{"type":"doc","content":[{"type":"customImage","attrs":{"src":"https://67dc5f63b8b6a6f0ea39b99a.storage.fermion.app/public-cached/markdown-editor-file/file-unique-id/h2hfb96lhtvfgstui5oybdnyvwda7bonscz7lnng4ivxddop3krphnjeffp8twsfppilmnfeqwmi_kba5d-26rmq4pav0ih4c7vh","alt":null,"width":600,"height":327}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"A deep-dive into building a production-ready multi-modal RAG system — covering metadata-based document separation, HTML-to-Markdown table conversion, Base64 image encoding with GPT-4o, concurrent LangChain summarization chains, RAG evaluation, and containerized deployment with Docker and Kubernetes."}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"Table of Contents"}]},{"type":"orderedList","attrs":{"start":1,"type":null},"content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#rag-fundamentals--evaluation-matrix","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"RAG Fundamentals & Evaluation Matrix"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#multi-model-systems--document-handling","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Multi-Model Systems & Document Handling"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#metadata-filtering--separating-tables-text--images","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Metadata Filtering — Separating Tables, Text & Images"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#html-to-markdown-table-conversion","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"HTML-to-Markdown Table Conversion"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#chunking-strategies-for-mixed-content-pdfs","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Chunking Strategies for Mixed-Content PDFs"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#image-extraction--verification","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Image Extraction & Verification"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#base64-image-encoding-for-llm-vision-models","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Base64 Image Encoding for LLM Vision Models"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#image-summarization-with-gpt-4o","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Image Summarization with GPT-4o"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#langchain-summarization-chain-tables--text","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"LangChain Summarization Chain (Tables + Text)"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#integrating-all-summaries-into-a-unified-vector-store","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Integrating All Summaries into a Unified Vector Store"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#the-complete-multi-model-pipeline","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"The Complete Multi-Model Pipeline"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#version-management--debugging","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Version Management & Debugging"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#containerization-with-docker--kubernetes","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Containerization with Docker & Kubernetes"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#common-challenges--mitigation-strategies","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Common Challenges & Mitigation Strategies"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/f5eb5bae-30c3-477c-97e4-8b536a72b06f#key-takeaways","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Key Takeaways"}]}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"RAG Fundamentals & Evaluation Matrix"}]},{"type":"customImage","attrs":{"src":"https://67dc5f63b8b6a6f0ea39b99a.storage.fermion.app/public-cached/markdown-editor-file/file-unique-id/879wm0hljkye8egvhlsntoga5klny4sn9xy9jza6j9es9r0lshzi3cybwumwvzdjxeuvlxl8la0vcwpwtawaslnbuxil78yklr-9","alt":null,"width":600,"height":327}},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"italic"}],"text":"Figure: An evaluation matrix scores both retrieval quality and generation accuracy across the pipeline."}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Retrieval-Augmented Generation (RAG)"},{"type":"text","text":" combines a retrieval system that fetches relevant documents with an LLM that generates answers based on the retrieved context."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Why RAG?"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Organizations prefer to work with "},{"type":"text","marks":[{"type":"bold"}],"text":"their own proprietary data"},{"type":"text","text":" rather than relying on the LLM's static training knowledge."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Grounds the LLM in a specific context, significantly improving "},{"type":"text","marks":[{"type":"bold"}],"text":"answer relevance and accuracy"},{"type":"text","text":"."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Allows knowledge updates without retraining — simply update the vector store."}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"The Evaluation Matrix"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"An "},{"type":"text","marks":[{"type":"bold"}],"text":"evaluation matrix"},{"type":"text","text":" is a structured checklist that quantitatively scores both retrieval quality and generation accuracy — enabling systematic assessment without manually reviewing thousands of documents."}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Evaluation Step"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"What to Check"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Metric"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Validate Retrieval"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Are the correct documents fetched from the vector store?"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Precision@K, Recall@K"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Validate LLM Output"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Does the LLM's response align with the retrieved context?"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Faithfulness, Answer Relevance"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"End-to-End"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Does the final answer correctly address the user's question?"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Correctness Score"}]}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":4},"content":[{"type":"text","text":"Implementing a Simple Evaluation Loop"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"def evaluate_rag(query: str, expected_answer: str, retriever, llm, prompt) -> dict:\n \"\"\"Evaluate retrieval and generation quality for a single query.\"\"\"\n\n # Step 1: Retrieve\n docs = retriever.invoke(query)\n context = \"\\n\\n\".join([d.page_content for d in docs])\n\n # Step 2: Generate\n formatted = prompt.format(context=context, question=query)\n generated_answer = llm.invoke(formatted).content\n\n # Step 3: Score (simple keyword overlap as proxy)\n expected_tokens = set(expected_answer.lower().split())\n generated_tokens = set(generated_answer.lower().split())\n overlap = expected_tokens & generated_tokens\n precision = len(overlap) / len(generated_tokens) if generated_tokens else 0\n recall = len(overlap) / len(expected_tokens) if expected_tokens else 0\n\n return {\n \"query\": query,\n \"retrieved_docs\": len(docs),\n \"generated_answer\": generated_answer,\n \"precision\": round(precision, 3),\n \"recall\": round(recall, 3),\n \"retrieved_context_preview\": context[:300]\n }\n"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Best practice:"},{"type":"text","text":" Run the evaluation matrix after each major pipeline change (loader update, chunk-size change, model swap) to catch quality regressions early."}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"Multi-Model Systems & Document Handling"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Real-world RAG projects routinely involve "},{"type":"text","marks":[{"type":"bold"}],"text":"15–20 different document types"},{"type":"text","text":". Simple pipelines break on complex formats; each type needs targeted handling."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Document Type Handling Reference"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"File Type"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Recommended Approach"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Notes"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"PDF"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"UnstructuredPDFLoader"},{"type":"text","text":" — extract text, tables, and images separately"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Set "},{"type":"text","marks":[{"type":"code"}],"text":"high_resolution=True"},{"type":"text","text":" for better OCR accuracy"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"CSV / Excel"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"pandas"},{"type":"text","text":" — treat each column as a separate field"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Ensure correct encoding (UTF-8) to avoid parse errors"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Images (PNG/JPG)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"OCR after converting to high-resolution format"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Verify the image extraction module is up-to-date"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"HTML"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"html2text"},{"type":"text","text":" or BeautifulSoup"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Strip markup before embedding"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"XML / JSON / YAML"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Language-specific parsers; map fields to a unified schema"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Process one file type at a time to isolate errors"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Scanned / Handwritten"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Tesseract OCR; fallback to LLM visual description"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Validate OCR quality manually before bulk processing"}]}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Loading PDFs with Unstructured"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"from langchain_community.document_loaders import UnstructuredPDFLoader\n\nloader = UnstructuredPDFLoader(\n file_path=\"./data/report.pdf\",\n mode=\"elements\", # one Document per logical element\n high_resolution=True, # enables high-quality OCR\n extract_images=True,\n image_output_dir=\"./figures/\",\n infer_table_structure=True\n)\n\ndocuments = loader.load()\nprint(f\"Total elements loaded: {len(documents)}\")\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":4},"content":[{"type":"text","text":"PDF Element Types"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Element Type"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Description"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"Text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Plain narrative paragraphs"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"Title"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Section headings or chapter titles"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"Table"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Structured data — often returned as HTML"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"Image"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Raster graphics (chart, figure, diagram)"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"Header"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Page headers"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"Code"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Inline code snippets"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"Link"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"URLs that can be scraped with BeautifulSoup"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"Scan/Handwritten"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Requires OCR to extract text"}]}]}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"Metadata Filtering — Separating Tables, Text & Images"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"After loading, each "},{"type":"text","marks":[{"type":"code"}],"text":"Document"},{"type":"text","text":" object carries a "},{"type":"text","marks":[{"type":"code"}],"text":"metadata.category"},{"type":"text","text":" field that identifies the element type. "},{"type":"text","marks":[{"type":"bold"}],"text":"Metadata filtering"},{"type":"text","text":" is the reliable, scalable way to route each element to the right processing pipeline."}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"tables = []\ntexts = []\nimages = []\n\nfor doc in documents:\n category = doc.metadata.get(\"category\", \"\").lower()\n if category == \"table\":\n tables.append(doc)\n elif category == \"image\":\n images.append(doc)\n else:\n texts.append(doc)\n\nprint(f\"Text elements : {len(texts)}\")\nprint(f\"Table elements: {len(tables)}\")\nprint(f\"Image elements: {len(images)}\")\n"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Important:"},{"type":"text","text":" The loader may label a block as "},{"type":"text","marks":[{"type":"code"}],"text":"\"table\""},{"type":"text","text":" even when the original PDF markup says "},{"type":"text","marks":[{"type":"code"}],"text":"\"title\""},{"type":"text","text":". Always inspect a sample of "},{"type":"text","marks":[{"type":"code"}],"text":"doc.metadata"},{"type":"text","text":" values before writing your filter logic."}]}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"# Inspect metadata for the first 5 elements\nfor doc in documents[:5]:\n print(doc.metadata)\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"HTML-to-Markdown Table Conversion"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Tables extracted from PDFs are often returned as "},{"type":"text","marks":[{"type":"bold"}],"text":"raw HTML strings"},{"type":"text","text":" in "},{"type":"text","marks":[{"type":"code"}],"text":"doc.page_content"},{"type":"text","text":". Raw HTML is noisy and token-inefficient for LLMs. Converting to Markdown produces cleaner, more readable content."}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"from html2markdown import convert\n\ndef convert_table_to_markdown(table_doc):\n \"\"\"Convert an HTML table element to clean Markdown text.\"\"\"\n html_content = table_doc.page_content\n if html_content.strip().startswith(\"<\"):\n table_doc.page_content = convert(html_content)\n return table_doc\n\n# Apply to all table elements\ntables = [convert_table_to_markdown(tbl) for tbl in tables]\n\n# Preview the first converted table\nprint(tables[0].page_content[:500])\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Before vs. After"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Before (HTML):"}]},{"type":"codeBlock","attrs":{"language":"html"},"content":[{"type":"text","text":"\n

Year	Fire Count	Acreage Burned
2022	68,988	7,577,183

\n"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"After (Markdown):"}]},{"type":"codeBlock","attrs":{"language":"javascript"},"content":[{"type":"text","text":"| Year | Fire Count | Acreage Burned |\n|------|-----------|----------------|\n| 2022 | 68,988 | 7,577,183 |\n"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"The Markdown version is directly embeddable and readable by LLMs without any additional cleaning."}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"Chunking Strategies for Mixed-Content PDFs"}]},{"type":"customImage","attrs":{"src":"https://67dc5f63b8b6a6f0ea39b99a.storage.fermion.app/public-cached/markdown-editor-file/file-unique-id/y_hpvxhwiaq6pr0texznbxpn9_lyuzbvoopvcgjj3qbutabcd6otukzuet6l5zukwixbckcl1x19qfnrd0f6mlsqivbp4wrublid","alt":null,"width":600,"height":327}},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"italic"}],"text":"Figure: Different chunking strategies suit different document structures — from title-based for hierarchical PDFs to token-based for large corpora."}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Chunking"},{"type":"text","text":" breaks a document into smaller, self-contained pieces so LLMs can process them within token limits."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Strategy Comparison"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Strategy"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"How It Works"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Best For"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Title-based (Section)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Split at headings (title, header, section)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"PDFs with clear hierarchical structure (≤ 3 pages)"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Paragraph"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Split after each paragraph or fixed sentence count"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Articles, reports with natural paragraph breaks"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Token-based"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Define max token count (e.g., 4 000) per chunk"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Large corpora (thousands of pages)"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Hybrid (Title + Token)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Split by title first; apply token limit within each section"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Research papers, technical manuals"}]}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Chunk Overlap"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"All strategies benefit from a "},{"type":"text","marks":[{"type":"bold"}],"text":"~10% overlap"},{"type":"text","text":" between adjacent chunks to preserve context across boundaries:"}]},{"type":"codeBlock","attrs":{"language":"javascript"},"content":[{"type":"text","text":"chunk_size = 4000 tokens\nchunk_overlap = 400 tokens (10% of chunk_size)\n\nChunk 1: tokens 1 – 4000\nChunk 2: tokens 3600 – 7600 (starts 400 tokens before the end of Chunk 1)\nChunk 3: tokens 7200 – 11200\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Implementation"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"from langchain.text_splitter import RecursiveCharacterTextSplitter\n\ndef chunk_by_title(text: str, max_tokens: int = 4000, overlap_ratio: float = 0.1) -> list:\n \"\"\"Title-aware chunking with token overlap.\"\"\"\n splitter = RecursiveCharacterTextSplitter(\n chunk_size=max_tokens,\n chunk_overlap=int(max_tokens * overlap_ratio),\n separators=[\"\\n## \", \"\\n### \", \"\\n\\n\", \"\\n\", \" \", \"\"]\n )\n return splitter.split_text(text)\n\n# Apply to text elements\nchunks = []\nfor txt in texts:\n chunks.extend(chunk_by_title(txt.page_content, max_tokens=4000, overlap_ratio=0.1))\n\nprint(f\"Total chunks after splitting: {len(chunks)}\")\n"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Rule of thumb:"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Small PDFs (≤ 3 pages) → title-based chunking is sufficient."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Large corpora (thousands of pages) → token-based chunking with overlap prevents truncation."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Dense tables → use smaller chunks (~500 tokens) for higher retrieval precision."}]}]}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"Image Extraction & Verification"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"After extraction, always verify that all expected images were saved correctly before feeding them into the summarization pipeline."}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"import os\nfrom IPython.display import Image, display\n\n# List extracted images\nimage_dir = \"./figures/\"\nimage_files = [f for f in os.listdir(image_dir) if f.lower().endswith((\".jpg\", \".png\"))]\n\nprint(f\"Extracted {len(image_files)} images:\")\nfor f in image_files:\n print(f\" {f}\")\n\n# Display first image for quality check\nif image_files:\n display(Image(os.path.join(image_dir, image_files[0])))\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Why Verification Matters"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Check"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Purpose"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"File count"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Confirms all figures were captured (no missing pages)"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"File size"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Zero-byte files indicate a failed extraction"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Visual inspection"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Confirms sufficient resolution for OCR / LLM input"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Naming pattern"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"figure_1.1.jpg"},{"type":"text","text":", "},{"type":"text","marks":[{"type":"code"}],"text":"figure_1.2.jpg"},{"type":"text","text":" — consistent naming aids traceability"}]}]}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"Base64 Image Encoding for LLM Vision Models"}]},{"type":"customImage","attrs":{"src":"https://67dc5f63b8b6a6f0ea39b99a.storage.fermion.app/public-cached/markdown-editor-file/file-unique-id/gsatgmhqthdebqkprg0jngapvzmc3rafp30kzkpl7g1vedtkgurhdbahlyk84xeqzrrad_b1nikj42fwl4-3l_mq0i7fdjiviava","alt":null,"width":600,"height":327}},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"italic"}],"text":"Figure: Images are Base64-encoded into ASCII strings, enabling them to be passed directly in an LLM prompt."}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Base64 encoding"},{"type":"text","text":" transforms binary image data into an ASCII string, allowing it to be embedded directly in a JSON payload sent to the LLM API."}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"import base64\n\ndef encode_image(image_path: str) -> str:\n \"\"\"Read an image file and return a Base64-encoded string.\"\"\"\n with open(image_path, \"rb\") as f:\n return base64.b64encode(f.read()).decode(\"utf-8\")\n\n# Test\nb64_str = encode_image(\"./figures/figure_1.1.jpg\")\nprint(f\"Base64 string length: {len(b64_str)} characters\")\nprint(f\"Preview: {b64_str[:80]}...\")\n"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"The function works for any image format ("},{"type":"text","marks":[{"type":"code"}],"text":".jpg"},{"type":"text","text":", "},{"type":"text","marks":[{"type":"code"}],"text":".png"},{"type":"text","text":", "},{"type":"text","marks":[{"type":"code"}],"text":".gif"},{"type":"text","text":", "},{"type":"text","marks":[{"type":"code"}],"text":".webp"},{"type":"text","text":")."}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"Image Summarization with GPT-4o"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Model selection for images:"},{"type":"text","text":" Only "},{"type":"text","marks":[{"type":"bold"}],"text":"omni models"},{"type":"text","text":" ("},{"type":"text","marks":[{"type":"code"}],"text":"gpt-4o"},{"type":"text","text":", "},{"type":"text","marks":[{"type":"code"}],"text":"gpt-4o-mini"},{"type":"text","text":") can accept image inputs. Standard text-only models ("},{"type":"text","marks":[{"type":"code"}],"text":"gpt-4"},{"type":"text","text":") will reject image payloads."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Model Capability Matrix"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Model"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Modalities"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Cost (relative)"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Best For"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"gpt-4"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Text only"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"High"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Pure text pipelines"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"gpt-4o"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Text + Image"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Higher"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Full multi-modal RAG"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"gpt-4o-mini"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Text + Image"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Lower"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Cost-efficient image summarization"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"gpt-5-pro"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Next-gen"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Highest"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Cutting-edge quality"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"LLaMA / Anthropic Claude"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Variable"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Variable"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Open-source / alternative"}]}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Single Image Summarization"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"from langchain_openai import ChatOpenAI\nfrom langchain_core.prompts import ChatPromptTemplate\n\nIMAGE_SUMMARY_PROMPT = \"\"\"You are an assistant tasked with summarizing images for retrieval.\nThe image may contain graphs, charts, tables, or diagrams.\nProvide a concise, non-redundant description suitable for embedding.\nFocus on: axis labels, trends, key values, and any text visible in the image.\"\"\"\n\ndef summarize_image(image_path: str) -> str:\n \"\"\"Summarize a single image using GPT-4o-mini.\"\"\"\n b64_str = encode_image(image_path)\n\n model = ChatOpenAI(model=\"gpt-4o-mini\") # omni model required for image input\n\n human_message = {\n \"type\": \"image_url\",\n \"image_url\": {\"url\": f\"data:image/jpeg;base64,{b64_str}\"}\n }\n text_message = {\"type\": \"text\", \"text\": IMAGE_SUMMARY_PROMPT}\n\n response = model.invoke([{\"role\": \"user\", \"content\": [human_message, text_message]}])\n return response.content\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Batch Image Summarization"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"def batch_summarize_images(folder: str) -> list[str]:\n \"\"\"Summarize all images in a directory.\"\"\"\n summaries = []\n image_files = [\n f for f in os.listdir(folder)\n if f.lower().endswith((\".jpg\", \".jpeg\", \".png\"))\n ]\n\n print(f\"Summarizing {len(image_files)} images...\")\n for i, fname in enumerate(image_files):\n path = os.path.join(folder, fname)\n summary = summarize_image(path)\n summaries.append(summary)\n print(f\" [{i+1}/{len(image_files)}] {fname}: {summary[:80]}...\")\n\n return summaries\n\nimage_summaries = batch_summarize_images(\"./figures/\")\nprint(f\"Generated {len(image_summaries)} image summaries.\")\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"LangChain Summarization Chain (Tables + Text)"}]},{"type":"customImage","attrs":{"src":"https://67dc5f63b8b6a6f0ea39b99a.storage.fermion.app/public-cached/markdown-editor-file/file-unique-id/fyh6imtxbc18wdvyenzve3zoawqtjlc3qujfkgq6zpnvnl_klbgy9-xqjxkkitnfyql1nnb7zbvz8xg8dcocyne6paudwethpnqj","alt":null,"width":600,"height":327}},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"italic"}],"text":"Figure: LangChain runnables compose into a chain — element → prompt → LLM → parser — enabling concurrent batch processing."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Prompt Template for Table + Text Summarization"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"from langchain_core.prompts import ChatPromptTemplate\n\nSUMMARY_PROMPT_TEXT = \"\"\"You are an assistant tasked with summarizing tables and text for semantic retrieval.\nProvide a concise summary optimized for embedding.\nFor tables, add a one-line description of the table's purpose.\nDo not prepend the word \"summary\"; output only the content.\"\"\"\n\nprompt = ChatPromptTemplate.from_template(SUMMARY_PROMPT_TEXT + \"\\n\\n{element}\")\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"RunnablePassThrough — Batch Processing"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"RunnablePassThrough"},{"type":"text","text":" is a LangChain construct that streams inputs through a chain without altering them, enabling batch processing of heterogeneous elements:"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"from langchain_core.runnables import RunnablePassthrough\nfrom langchain_openai import ChatOpenAI\nfrom langchain_core.output_parsers import StrOutputParser\n\n# Build the summarization chain\nchat_model = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\n\nsummary_chain = (\n {\"element\": RunnablePassthrough()}\n | prompt\n | chat_model\n | StrOutputParser()\n)\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Running the Chain with Concurrency"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"# Summarize all table elements (up to 5 in parallel)\ntable_texts = [tbl.page_content for tbl in tables]\ntable_summaries = summary_chain.batch(table_texts, config={\"max_concurrency\": 5})\n\n# Summarize all text elements\ntext_texts = [txt.page_content for txt in texts]\ntext_summaries = summary_chain.batch(text_texts, config={\"max_concurrency\": 5})\n\nprint(f\"Table summaries : {len(table_summaries)}\")\nprint(f\"Text summaries : {len(text_summaries)}\")\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"How "},{"type":"text","marks":[{"type":"code"}],"text":"max_concurrency=5"},{"type":"text","text":" Works"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Setting "},{"type":"text","marks":[{"type":"code"}],"text":"max_concurrency=5"},{"type":"text","text":" means up to "},{"type":"text","marks":[{"type":"bold"}],"text":"5 elements are sent to the LLM simultaneously"},{"type":"text","text":", dramatically reducing total wall-clock time for large corpora:"}]},{"type":"codeBlock","attrs":{"language":"javascript"},"content":[{"type":"text","text":"Without concurrency (serial): N elements × T seconds each = N×T seconds total\nWith max_concurrency=5: ceil(N/5) batches × T seconds each = (N/5)×T seconds total\n → ~5× speedup\n"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Internally, each element goes through "},{"type":"text","marks":[{"type":"bold"}],"text":"~5 refinement passes"},{"type":"text","text":" (similar to training epochs in deep learning) before the final summary is returned — iteratively improving output quality before convergence."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Structured Output Parsing"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"For downstream JSON-formatted storage or validation:"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"from langchain_core.output_parsers import JsonOutputParser\nfrom pydantic import BaseModel, Field\n\nclass ElementSummary(BaseModel):\n summary: str = Field(description=\"Concise summary of the element\")\n element_type: str = Field(description=\"text, table, or image\")\n\nparser = JsonOutputParser(pydantic_object=ElementSummary)\n\nstructured_chain = (\n {\"element\": RunnablePassthrough()}\n | prompt\n | chat_model\n | parser\n)\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"Integrating All Summaries into a Unified Vector Store"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"After generating summaries for all three modalities, merge them and embed into a single ChromaDB collection:"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"import chromadb\nfrom langchain_openai import OpenAIEmbeddings\n\n# Step 1: Combine all summaries\nall_summaries = text_summaries + table_summaries + image_summaries\nall_metadata = (\n [{\"type\": \"text\", \"source\": \"pdf_text\"}] * len(text_summaries) +\n [{\"type\": \"table\", \"source\": \"pdf_table\"}] * len(table_summaries) +\n [{\"type\": \"image\", \"source\": \"pdf_image\"}] * len(image_summaries)\n)\n\nprint(f\"Total summaries to embed: {len(all_summaries)}\")\n\n# Step 2: Embed\nembeddings_model = OpenAIEmbeddings(model=\"text-embedding-3-small\")\nvectors = embeddings_model.embed_documents(all_summaries)\n\n# Step 3: Store in ChromaDB\nclient = chromadb.Client()\ncollection = client.create_collection(name=\"multimodal_rag_docs\")\n\nids = [f\"doc_{i}\" for i in range(len(all_summaries))]\n\ncollection.add(\n documents=all_summaries,\n embeddings=vectors,\n metadatas=all_metadata,\n ids=ids\n)\n\nprint(f\"Stored {collection.count()} vectors in ChromaDB.\")\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Querying Across All Modalities"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"def query_multimodal_store(query: str, n_results: int = 5) -> list:\n \"\"\"Retrieve the most relevant summaries across text, table, and image types.\"\"\"\n query_embedding = embeddings_model.embed_query(query)\n\n results = collection.query(\n query_embeddings=[query_embedding],\n n_results=n_results,\n include=[\"documents\", \"metadatas\", \"distances\"]\n )\n\n for i, (doc, meta, dist) in enumerate(zip(\n results[\"documents\"][0],\n results[\"metadatas\"][0],\n results[\"distances\"][0]\n )):\n print(f\"\\n[{i+1}] Type: {meta['type']} | Distance: {dist:.4f}\")\n print(f\" {doc[:200]}...\")\n\n return results\n\n# Example\nquery_multimodal_store(\"What percentage of acreage was burned in 2022?\")\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"The Complete Multi-Model Pipeline"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Here is the full pipeline from raw PDF to a queryable multi-modal vector store:"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"import os, base64\nfrom langchain_community.document_loaders import UnstructuredPDFLoader\nfrom langchain.text_splitter import RecursiveCharacterTextSplitter\nfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings\nfrom langchain_core.prompts import ChatPromptTemplate\nfrom langchain_core.runnables import RunnablePassthrough\nfrom langchain_core.output_parsers import StrOutputParser\nfrom html2markdown import convert\nimport chromadb\n\n# ── 1. Load PDF ────────────────────────────────────────────\nloader = UnstructuredPDFLoader(\n \"./data/report.pdf\",\n mode=\"elements\",\n high_resolution=True,\n extract_images=True,\n image_output_dir=\"./figures/\",\n infer_table_structure=True\n)\ndocuments = loader.load()\n\n# ── 2. Filter by metadata category ────────────────────────\ntables, texts, image_paths = [], [], []\nfor doc in documents:\n cat = doc.metadata.get(\"category\", \"\").lower()\n if cat == \"table\":\n tables.append(doc)\n elif cat == \"image\":\n image_paths.append(doc.metadata.get(\"image_path\", \"\"))\n else:\n texts.append(doc)\n\n# ── 3. Convert HTML tables → Markdown ─────────────────────\nfor tbl in tables:\n if tbl.page_content.strip().startswith(\"<\"):\n tbl.page_content = convert(tbl.page_content)\n\n# ── 4. Chunk text elements ─────────────────────────────────\nsplitter = RecursiveCharacterTextSplitter(chunk_size=4000, chunk_overlap=400)\ntext_chunks = splitter.split_documents(texts)\n\n# ── 5. Summarization chain ─────────────────────────────────\nPROMPT = \"\"\"Summarize the following for semantic retrieval.\nFor tables, describe the table's purpose in one line.\nOutput only the summary content.\\n\\n{element}\"\"\"\n\nprompt = ChatPromptTemplate.from_template(PROMPT)\nmodel = ChatOpenAI(model=\"gpt-4o-mini\", temperature=0)\nchain = {\"element\": RunnablePassthrough()} | prompt | model | StrOutputParser()\n\ntable_summaries = chain.batch([t.page_content for t in tables], config={\"max_concurrency\": 5})\ntext_summaries = chain.batch([t.page_content for t in text_chunks], config={\"max_concurrency\": 5})\n\n# ── 6. Summarize images ────────────────────────────────────\ndef summarize_image(path: str) -> str:\n with open(path, \"rb\") as f:\n b64 = base64.b64encode(f.read()).decode(\"utf-8\")\n img_model = ChatOpenAI(model=\"gpt-4o-mini\")\n msg = {\"role\": \"user\", \"content\": [\n {\"type\": \"image_url\", \"image_url\": {\"url\": f\"data:image/jpeg;base64,{b64}\"}},\n {\"type\": \"text\", \"text\": \"Describe this image concisely for retrieval.\"}\n ]}\n return img_model.invoke([msg]).content\n\nimage_dir = \"./figures/\"\nimage_summaries = [\n summarize_image(os.path.join(image_dir, f))\n for f in os.listdir(image_dir)\n if f.lower().endswith((\".jpg\", \".png\"))\n]\n\n# ── 7. Embed & store ───────────────────────────────────────\nall_summaries = text_summaries + table_summaries + image_summaries\nall_meta = (\n [{\"type\": \"text\"}] * len(text_summaries) +\n [{\"type\": \"table\"}] * len(table_summaries) +\n [{\"type\": \"image\"}] * len(image_summaries)\n)\n\nembed_model = OpenAIEmbeddings(model=\"text-embedding-3-small\")\nvectors = embed_model.embed_documents(all_summaries)\n\nclient = chromadb.Client()\ncollection = client.create_collection(\"multimodal_rag\")\ncollection.add(\n documents = all_summaries,\n embeddings = vectors,\n metadatas = all_meta,\n ids = [f\"doc_{i}\" for i in range(len(all_summaries))]\n)\n\nprint(f\"Pipeline complete. {collection.count()} vectors stored.\")\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"Version Management & Debugging"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"The Versioning Problem"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"LangChain and ChromaDB release frequently. A new version may introduce "},{"type":"text","marks":[{"type":"bold"}],"text":"breaking changes"},{"type":"text","text":" that silently corrupt your pipeline — wrong output formats, missing methods, or incompatible type signatures."}]},{"type":"codeBlock","attrs":{"language":"javascript"},"content":[{"type":"text","text":"Example breaking versions:\n LangChain 0.3.10 (released 24 July 2025) — introduced breaking changes\n ChromaDB 0.2.6 (released 12 Sept 2025) — caused vector store incompatibilities\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Solution: Pin to Stable Versions"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Create a "},{"type":"text","marks":[{"type":"code"}],"text":"requirements.txt"},{"type":"text","text":" and pin every package to a tested version:"}]},{"type":"codeBlock","attrs":{"language":"text"},"content":[{"type":"text","text":"# requirements.txt\nlangchain==0.2.7\nlangchain-openai==0.1.8\nlangchain-community==0.2.7\nchromadb==0.4.24\nunstructured==0.14.0\nhtml2markdown==0.1.7\nredis==5.0.1\npdfminer.six==20221105\nml-dtypes==0.1.0\n"}]},{"type":"codeBlock","attrs":{"language":"bash"},"content":[{"type":"text","text":"# Install from pinned requirements\npip install -r requirements.txt\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Steps to Downgrade a Package"}]},{"type":"orderedList","attrs":{"start":1,"type":null},"content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Locate the package in "},{"type":"text","marks":[{"type":"code"}],"text":"requirements.txt"},{"type":"text","text":"."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Replace the version number with the desired stable version."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Re-install: "},{"type":"text","marks":[{"type":"code"}],"text":"pip install -r requirements.txt"},{"type":"text","text":"."}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Debugging Checklist"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Step"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Action"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Error after upgrade?"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Pin to an older stable version immediately"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Dependency conflict?"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Use "},{"type":"text","marks":[{"type":"code"}],"text":"pip install --upgrade pip"},{"type":"text","text":" then reinstall"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Start small"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Test with a single small PDF before scaling to the full corpus"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Check logs"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Monitor for "},{"type":"text","marks":[{"type":"code"}],"text":"403"},{"type":"text","text":" errors (auth), "},{"type":"text","marks":[{"type":"code"}],"text":"NLTK"},{"type":"text","text":" errors (missing corpora), or "},{"type":"text","marks":[{"type":"code"}],"text":"AttributeError"},{"type":"text","text":" (version mismatch)"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Install missing NLP data"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"import nltk; nltk.download('all')"},{"type":"text","text":" for NLTK-related failures"}]}]}]}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"# Check installed versions for reproducibility\nimport langchain, chromadb\nprint(f\"LangChain: {langchain.__version__}\")\nprint(f\"ChromaDB : {chromadb.__version__}\")\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"Containerization with Docker & Kubernetes"}]},{"type":"customImage","attrs":{"src":"https://67dc5f63b8b6a6f0ea39b99a.storage.fermion.app/public-cached/markdown-editor-file/file-unique-id/uqu_vnul_492wredw1tm4mya8p5yebwk7sf1xyh4d3ig9qscznnxg9ii1lpqklqfvydx7fu7fc0njfgsrml6gy43dqonr_oawuf1","alt":null,"width":600,"height":327}},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"italic"}],"text":"Figure: Docker packages the exact library versions; Kubernetes orchestrates multiple containers for scaling."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Why Containerization?"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Without a Docker image, production teams may install "},{"type":"text","marks":[{"type":"bold"}],"text":"newer library versions"},{"type":"text","text":", recreating the exact bugs you fixed during development. Docker freezes the environment."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Definitions"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Docker:"},{"type":"text","text":" Creates a reproducible image containing your exact library versions, OS dependencies, and application code."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Kubernetes:"},{"type":"text","text":" Orchestrates multiple Docker containers for "},{"type":"text","marks":[{"type":"bold"}],"text":"scaling, load balancing, and high availability"},{"type":"text","text":"."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Containerization:"},{"type":"text","text":" Packaging an application with its runtime environment, dependencies, and configuration into a portable, isolated container."}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Minimal Dockerfile for a RAG Service"}]},{"type":"codeBlock","attrs":{"language":"dockerfile"},"content":[{"type":"text","text":"# Dockerfile\nFROM python:3.10-slim\n\n# Install system dependencies\nRUN apt-get update && apt-get install -y \\\n tesseract-ocr \\\n poppler-utils \\\n && rm -rf /var/lib/apt/lists/*\n\n# Set working directory\nWORKDIR /app\n\n# Copy and install Python dependencies (pinned versions)\nCOPY requirements.txt .\nRUN pip install --no-cache-dir -r requirements.txt\n\n# Copy application code\nCOPY . .\n\n# Expose API port\nEXPOSE 8000\n\n# Start the RAG service\nCMD [\"uvicorn\", \"main:app\", \"--host\", \"0.0.0.0\", \"--port\", \"8000\"]\n"}]},{"type":"codeBlock","attrs":{"language":"bash"},"content":[{"type":"text","text":"# Build the Docker image\ndocker build -t multimodal-rag:v1.0 .\n\n# Run locally\ndocker run -p 8000:8000 -e OPENAI_API_KEY=your_key multimodal-rag:v1.0\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Kubernetes Deployment (Minimal)"}]},{"type":"codeBlock","attrs":{"language":"yaml"},"content":[{"type":"text","text":"# deployment.yaml\napiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: multimodal-rag\nspec:\n replicas: 3 # run 3 instances for load distribution\n selector:\n matchLabels:\n app: multimodal-rag\n template:\n metadata:\n labels:\n app: multimodal-rag\n spec:\n containers:\n - name: rag-container\n image: multimodal-rag:v1.0\n ports:\n - containerPort: 8000\n env:\n - name: OPENAI_API_KEY\n valueFrom:\n secretKeyRef:\n name: openai-secret\n key: api-key\n"}]},{"type":"codeBlock","attrs":{"language":"bash"},"content":[{"type":"text","text":"kubectl apply -f deployment.yaml\nkubectl get pods # verify 3 replicas are running\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"Common Challenges & Mitigation Strategies"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Challenge"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Impact"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Mitigation"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Complex multi-modal pipelines"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Longer development cycles (2–3 weeks)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Break work into focused sub-tasks: image handling → table conversion → text chunking"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Version incompatibilities"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Pipeline breaks after library upgrades"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Pin stable versions in "},{"type":"text","marks":[{"type":"code"}],"text":"requirements.txt"},{"type":"text","text":" ("},{"type":"text","marks":[{"type":"code"}],"text":"langchain==0.2.7"},{"type":"text","text":", "},{"type":"text","marks":[{"type":"code"}],"text":"chromadb==0.4.24"},{"type":"text","text":")"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"OCR failures on handwritten scans"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Missing content in image summaries"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Validate OCR quality manually; fallback to LLM visual description ("},{"type":"text","marks":[{"type":"code"}],"text":"gpt-4o-mini"},{"type":"text","text":") if OCR fails"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Free-tier quota limits"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"API calls stop after N requests"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Consolidate questions; switch to a paid account; rotate API keys"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Colab session resets"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"All packages lost"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Script all installs at notebook top; persist ChromaDB to Google Drive"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"GPU downgraded to CPU"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Out-of-memory on heavy packages"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Monitor "},{"type":"text","marks":[{"type":"code"}],"text":"!df -h"},{"type":"text","text":"; clear "},{"type":"text","marks":[{"type":"code"}],"text":"/tmp/*"},{"type":"text","text":" before installing Unstructured"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Low-code platform OCR broken"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Images return empty summaries"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Fall back to custom Python code for edge cases"}]}]}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"Key Takeaways"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Metadata filtering"},{"type":"text","text":" ("},{"type":"text","marks":[{"type":"code"}],"text":"doc.metadata.category"},{"type":"text","text":") is the scalable, reliable way to separate text, tables, and images after PDF loading — never rely on file position or manual inspection."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"HTML-to-Markdown conversion"},{"type":"text","text":" is a mandatory step for table elements before embedding — raw HTML adds noise and wastes tokens."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Chunk overlap (~10%)"},{"type":"text","text":" is essential to preserve context across chunk boundaries, especially for long technical documents."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Only omni models"},{"type":"text","text":" ("},{"type":"text","marks":[{"type":"code"}],"text":"gpt-4o"},{"type":"text","text":", "},{"type":"text","marks":[{"type":"code"}],"text":"gpt-4o-mini"},{"type":"text","text":") can accept Base64-encoded image inputs — standard text models will reject them."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"RunnablePassThrough"},{"type":"text","marks":[{"type":"bold"}],"text":" with "},{"type":"text","marks":[{"type":"code"}],"text":"max_concurrency=5"},{"type":"text","text":" enables parallel summarization across all modalities, reducing processing time by ~5×."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Pin all library versions"},{"type":"text","text":" in "},{"type":"text","marks":[{"type":"code"}],"text":"requirements.txt"},{"type":"text","text":" — LangChain and ChromaDB change rapidly and breaking changes are common."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Docker + Kubernetes"},{"type":"text","text":" is the production-grade solution for reproducible, scalable RAG deployments. Without containerization, environment drift between development and production is inevitable."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Iterative validation"},{"type":"text","text":" (small sample → full corpus) catches errors early and prevents wasted API calls on a broken pipeline."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"The "},{"type":"text","marks":[{"type":"bold"}],"text":"evaluation matrix"},{"type":"text","text":" (retrieval accuracy + generation correctness) should be re-run after every major pipeline change."}]}]}]},{"type":"horizontalRule"},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"italic"}],"text":"Last updated: October 2025 | Based on session notes from the Generative AI deep-dive series."}]}]}