Chunking

from ragrails import RagRails

rag = RagRails()

docs = rag.parse(files="files/refund-policy.pdf")
chunks = rag.chunk(
    markdown=docs.outputs,
    chunk_size=1200,
    chunk_overlap=150,
    min_chunk_length=80,
)

chunks = rag.chunk(
    markdown="# Refund policy\n\nCustomers can request a refund within 30 days of purchase.",
    title="Refund policy",
    source="policy.md",
)

chunks.inputs
chunks.chunks
chunks.items
chunks.failed
chunks.errors

{
  "id": "refund-policy-0",
  "text": "Customers can request a refund within 30 days of purchase.",
  "source": "files/refund-policy.pdf",
  "metadata": {"title": "Refund policy", "chunk_index": 0}
}

# String
rag.chunk(markdown="# Guide\n\nContent here.")

# List of strings
rag.chunk(markdown=["# Doc 1\n\nContent.", "# Doc 2\n\nMore content."])

# Document dicts from parse(), scrape(), fetch(), or ingest source results
rag.chunk(markdown=extraction_result.outputs)

embedded = rag.embed(chunks=chunks.items)