import re
from typing import List, Tuple
from dataclasses import dataclass
class TextChunker:
"""
A smart text chunker that analyzes text structure and automatically
determines the best splitting strategy based on detected patterns.
Always splits on sentence boundaries.
"""
def __init__(self, chunk_limit: int = 500, chunk_overlap: int = 0):
"""
Initialize the smart chunker.
Args:
chunk_limit: Target maximum characters per chunk (may be exceeded to preserve sentences)
chunk_overlap: Number of characters to overlap between speech_chunks
"""
self.chunk_limit = chunk_limit
self.chunk_overlap = chunk_overlap
@dataclass
class SeparatorInfo:
"""Information about a detected separator in the text."""
pattern: str
count: int
priority: int
description: str
keep_separator: bool = True
def _detect_separators(self, text: str) -> List["TextChunker.SeparatorInfo"]:
"""
Analyze the text and detect available separators with their priority.
Returns a list of separators ordered by priority (best to worst).
"""
separators = []
# Markdown headers (# Header, ## Header, etc.)
md_headers = re.findall(r"^#{1,6}\s+.+$", text, re.MULTILINE)
if md_headers:
separators.append(
self.SeparatorInfo(
pattern=r"\n(?=#{1,6}\s+)",
count=len(md_headers),
priority=1,
description=f"Markdown headers ({len(md_headers)} found)",
)
)
# HTML headers (
, , etc.)
html_headers = re.findall(
r"]*>.*?", text, re.IGNORECASE | re.DOTALL
)
if html_headers:
separators.append(
self.SeparatorInfo(
pattern=r"\n(?=]*>", text, re.IGNORECASE)
if html_divs:
separators.append(
self.SeparatorInfo(
pattern=r"\n(?=<(?:div|section))",
count=len(html_divs),
priority=3,
description=f"HTML divs/sections ({len(html_divs)} found)",
)
)
# Horizontal rules (---, ***, ___)
hr_count = len(re.findall(r"^(?:---+|\*\*\*+|___+)\s*$", text, re.MULTILINE))
if hr_count:
separators.append(
self.SeparatorInfo(
pattern=r"\n(?:---+|\*\*\*+|___+)\s*\n",
count=hr_count,
priority=4,
description=f"Horizontal rules ({hr_count} found)",
)
)
# Bullet points or numbered lists
list_items = re.findall(r"^[\s]*(?:[-*+]|\d+\.)\s+", text, re.MULTILINE)
if list_items:
# Group consecutive list items
list_groups = len(
re.findall(r"(?:^[\s]*(?:[-*+]|\d+\.)\s+.*\n)+", text, re.MULTILINE)
)
if list_groups > 1:
separators.append(
self.SeparatorInfo(
pattern=r"\n(?=[\s]*(?:[-*+]|\d+\.)\s+)",
count=list_groups,
priority=5,
description=f"List groups ({list_groups} found)",
)
)
# Double newlines (paragraphs)
double_newlines = text.count("\n\n")
if double_newlines > 0:
separators.append(
self.SeparatorInfo(
pattern=r"\n\n",
count=double_newlines,
priority=6,
description=f"Paragraphs ({double_newlines} found)",
)
)
# Single newlines
single_newlines = text.count("\n") - (double_newlines * 2)
if single_newlines > 0:
separators.append(
self.SeparatorInfo(
pattern=r"\n",
count=single_newlines,
priority=7,
description=f"Lines ({single_newlines} found)",
)
)
# Sentence endings
sentences = re.findall(r"[.!?]+[\s\n]+", text)
if sentences:
separators.append(
self.SeparatorInfo(
pattern=r"(?<=[.!?])\s+",
count=len(sentences),
priority=8,
description=f"Sentences ({len(sentences)} found)",
)
)
# Sort by priority (lower number = higher priority)
separators.sort(key=lambda x: x.priority)
return separators
def _split_by_separator(self, text: str, separator_pattern: str) -> List[str]:
"""Split text by a separator pattern, preserving the separator."""
if not text:
return []
# Split while keeping the separator
parts = re.split(f"({separator_pattern})", text)
# Reconstruct pieces with separators
result = []
current = ""
for part in parts:
if part:
current += part
# If we just added a separator, save this piece
if re.match(separator_pattern, part):
if current.strip():
result.append(current)
current = ""
# Add any remaining text
if current.strip():
result.append(current)
# If no splits occurred, return the original text
if not result:
result = [text]
return result
def _split_by_sentences(self, text: str) -> List[str]:
"""
Split text into complete sentences, ensuring no mid-sentence breaks.
Returns speech_chunks that respect sentence boundaries and tries to balance chunk sizes.
"""
# Match sentence boundaries: year, exclamation, or question mark followed by space/newline
sentence_pattern = r"(?<=[.!?])\s+"
sentences = re.split(sentence_pattern, text)
if not sentences:
return [text]
# Filter out empty sentences
sentences = [s.strip() for s in sentences if s.strip()]
if not sentences:
return [text]
# If all sentences fit in one chunk, return as is
total_length = (
sum(len(s) for s in sentences) + len(sentences) - 1
) # +1 for spaces between
if total_length <= self.chunk_limit:
return [" ".join(sentences)]
# Build speech_chunks greedily first
speech_chunks = []
current_chunk = ""
for sentence in sentences:
# If adding this sentence would exceed limit and we have content, start new chunk
if (
current_chunk
and len(current_chunk) + len(sentence) + 1 > self.chunk_limit
):
speech_chunks.append(current_chunk)
current_chunk = sentence
else:
# Add sentence to current chunk
if current_chunk:
current_chunk += " " + sentence
else:
current_chunk = sentence
# Add final chunk
if current_chunk:
speech_chunks.append(current_chunk)
# Now balance the speech_chunks: if the last chunk is too small, redistribute
if len(speech_chunks) >= 2:
last_chunk_size = len(speech_chunks[-1])
# If last chunk is less than 40% of chunk_limit, try to rebalance
if last_chunk_size < self.chunk_limit * 0.5:
# Rebuild from sentences, distributing more evenly
speech_chunks = self._balance_sentence_chunks(sentences)
return speech_chunks if speech_chunks else [text]
def _balance_sentence_chunks(self, sentences: List[str]) -> List[str]:
"""
Distribute sentences across speech_chunks to minimize size variance.
Uses a greedy approach that looks ahead to avoid tiny final speech_chunks.
"""
if not sentences:
return []
total_length = sum(len(s) for s in sentences) + len(sentences) - 1
# Estimate number of speech_chunks needed
estimated_chunks = max(
1, (total_length + self.chunk_limit - 1) // self.chunk_limit
)
target_size = total_length / estimated_chunks
speech_chunks = []
current_chunk = ""
remaining_sentences = len(sentences)
for i, sentence in enumerate(sentences):
remaining_sentences -= 1
if not current_chunk:
current_chunk = sentence
else:
# Calculate what's left to process
remaining_text_length = sum(len(s) for s in sentences[i + 1 :])
if remaining_sentences > 0:
remaining_text_length += remaining_sentences # spaces
current_length = len(current_chunk)
new_length = current_length + len(sentence) + 1
# Decide whether to add to current chunk or start new one
# Start new chunk if:
# 1. Adding would exceed limit AND current chunk is at least 60% of target
# 2. OR we're getting close to target size and have plenty of text left
if (
new_length > self.chunk_limit
and current_length >= target_size * 0.7
):
speech_chunks.append(current_chunk)
current_chunk = sentence
elif (
current_length >= target_size * 0.9
and remaining_text_length > target_size * 0.5
):
# We're near target and there's enough remaining - start new chunk
speech_chunks.append(current_chunk)
current_chunk = sentence
else:
current_chunk += " " + sentence
if current_chunk:
speech_chunks.append(current_chunk)
return speech_chunks
def _merge_small_chunks(self, speech_chunks: List[str]) -> List[str]:
"""
Merge speech_chunks that are smaller than the limit to optimize chunk sizes.
Ensures the last chunk is not much smaller than the chunk_limit by merging it with the previous chunk if needed.
"""
if not speech_chunks:
return []
merged = []
current = speech_chunks[0]
for next_chunk in speech_chunks[1:]:
# If combining won't exceed limit, merge them
if len(current) + len(next_chunk) <= self.chunk_limit:
current += next_chunk
else:
merged.append(current)
current = next_chunk
# Add the last chunk
merged.append(current)
# If the last chunk is much smaller than chunk_limit, merge it with the previous one
# (unless there's only one chunk)
if len(merged) >= 2 and len(merged[-1]) < self.chunk_limit * 0.5:
# Merge last two speech_chunks
merged[-2] += merged[-1]
merged.pop(-1)
return merged
def _recursive_split(
self, text: str, separators: List[SeparatorInfo], separator_idx: int = 0
) -> List[str]:
"""
Recursively split text using available separators until speech_chunks fit the limit.
Always falls back to sentence-aware splitting to avoid mid-sentence breaks.
"""
# Base case: if text fits, return it
if len(text) <= self.chunk_limit:
return [text]
# If we've exhausted all separators, use sentence-aware splitting
if separator_idx >= len(separators):
return self._split_by_sentences(text)
# Try current separator
separator = separators[separator_idx]
splits = self._split_by_separator(text, separator.pattern)
# If no split occurred or only one piece, try next separator
if len(splits) <= 1:
return self._recursive_split(text, separators, separator_idx + 1)
# Process each split
result = []
for split in splits:
if len(split) <= self.chunk_limit:
result.append(split)
else:
# This split is still too large, recurse with next separator
sub_chunks = self._recursive_split(split, separators, separator_idx + 1)
result.extend(sub_chunks)
# Merge small consecutive speech_chunks
result = self._merge_small_chunks(result)
return result
def chunk(self, text: str, verbose: bool = False, headings: str = "") -> List[str]:
"""
Chunk the text using automatically detected separators.
Always splits on complete sentences.
Args:
text: The text to chunk
verbose: If True, print information about detected separators
headings: Optional headings/context to prepend to each chunk (string)
Returns:
List of text speech_chunks, each optionally prefixed with the provided headings
"""
if not text:
return []
# Detect available separators
separators = self._detect_separators(text)
if verbose:
print(f"Detected {len(separators)} separator types:")
for sep in separators:
print(f" - {sep.description} (priority {sep.priority})")
print()
# If no separators found, use sentence-aware splitting
if not separators:
if verbose:
print("No natural separators found, splitting by sentences")
speech_chunks = self._split_by_sentences(text)
else:
# Recursively split using detected separators
speech_chunks = self._recursive_split(text, separators)
# Clean up speech_chunks
speech_chunks = [chunk.strip() for chunk in speech_chunks if chunk.strip()]
# Add headings to each chunk if provided
if headings and headings.strip():
# Ensure headings end with newlines for proper formatting
formatted_headings = headings.strip()
if not formatted_headings.endswith("\n"):
formatted_headings += "\n\n"
else:
formatted_headings += "\n"
# Prepend headings to each chunk
speech_chunks = [f"#{formatted_headings}" + chunk for chunk in speech_chunks]
if verbose:
print(f"Created {len(speech_chunks)} speech_chunks")
if headings:
print(f"Added headings to each chunk: '{headings.strip()}'")
print(f"Chunk sizes: {[len(c) for c in speech_chunks]}")
return speech_chunks