Regex & NLP Clause Extraction

import re
import spacy
from typing import List, Dict, Optional
from dataclasses import dataclass, field
import logging

logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")

# Load spaCy model (ensure installation via: python -m spacy download en_core_web_sm)
try:
    nlp = spacy.load("en_core_web_sm")
except OSError:
    logging.error("spaCy model 'en_core_web_sm' not found. Install with: python -m spacy download en_core_web_sm")
    raise SystemExit(1)

@dataclass
class ClauseResult:
    clause_type: str
    text: str
    confidence: float
    start_idx: int
    end_idx: int
    metadata: Dict[str, str] = field(default_factory=dict)

# Deterministic regex anchors for high-frequency lease provisions
# Patterns use non-greedy capture groups and lookahead for paragraph boundaries
CLAUSE_PATTERNS: Dict[str, re.Pattern] = {
    "rent_escalation": re.compile(
        r"(?:annual\s+increase|rent\s+escalat(?:e|ion)|CPI\s+adjustment|base\s+rent\s+shall\s+increase)\s*[:;]?\s*(.*?)(?=\n{2,}|\Z)",
        re.IGNORECASE | re.DOTALL
    ),
    "maintenance": re.compile(
        r"(?:tenant|landlord|lessee|lessor)\s+shall\s+(?:maintain|repair|keep|be\s+responsible\s+for)\s+(.*?)(?=\n{2,}|\Z)",
        re.IGNORECASE | re.DOTALL
    ),
    "termination": re.compile(
        r"(?:early\s+termination|option\s+to\s+terminate|break\s+clause|right\s+to\s+cancel)\s*[:;]?\s*(.*?)(?=\n{2,}|\Z)",
        re.IGNORECASE | re.DOTALL
    )
}

def _calculate_confidence(match_text: str) -> float:
    """Heuristic confidence scoring leveraging NLP linguistic features."""
    doc = nlp(match_text)
    score = 0.55  # Base confidence for successful regex match

    # Boost for modal/legal obligation markers
    if any(token.pos_ == "VERB" and token.lemma_ in ("shall", "must", "agree", "covenant") for token in doc):
        score += 0.20

    # Boost for financial/temporal entities common in lease clauses
    if any(ent.label_ in ("MONEY", "DATE", "PERCENT", "CARDINAL") for ent in doc.ents):
        score += 0.15

    # Penalize overly short or fragmented matches
    if len(doc) < 4:
        score -= 0.25

    return max(0.0, min(1.0, score))

def extract_clauses_hybrid(lease_text: str, confidence_threshold: float = 0.65) -> List[ClauseResult]:
    """
    Hybrid Regex + NLP clause extraction pipeline for commercial/residential leases.

    Args:
        lease_text: Raw or pre-cleaned lease document text.
        confidence_threshold: Minimum NLP confidence score to accept a match.

    Returns:
        List of ClauseResult objects sorted by document position.
    """
    results = []

    # Normalize line endings and collapse excessive whitespace
    normalized_text = re.sub(r"\r\n", "\n", lease_text)
    normalized_text = re.sub(r"\n{3,}", "\n\n", normalized_text)

    for clause_type, pattern in CLAUSE_PATTERNS.items():
        for match in pattern.finditer(normalized_text):
            raw_match = match.group(1).strip()
            if not raw_match:
                continue

            confidence = _calculate_confidence(raw_match)

            if confidence >= confidence_threshold:
                # Extract entities for downstream schema mapping
                doc = nlp(raw_match)
                extracted_entities = {ent.label_: ent.text for ent in doc.ents}

                results.append(ClauseResult(
                    clause_type=clause_type,
                    text=raw_match,
                    confidence=round(confidence, 3),
                    start_idx=match.start(),
                    end_idx=match.end(),
                    metadata={"entities": extracted_entities}
                ))
            else:
                logging.debug(f"Filtered low-confidence match for {clause_type}: '{raw_match[:60]}...' (score: {confidence:.2f})")

    # Sort by original document position to preserve reading order
    results.sort(key=lambda x: x.start_idx)
    return results

if __name__ == "__main__":
    sample_lease = """
    4. RENT ESCALATION. Base Rent shall increase annually by three percent (3%)
    commencing on the first anniversary of the Commencement Date.

    7. MAINTENANCE OBLIGATIONS. Tenant shall maintain all interior HVAC systems
    and replace filters quarterly at its sole expense.

    12. TERMINATION RIGHTS. Option to terminate this lease requires ninety (90)
    days written notice prior to the expiration of the initial term.
    """

    extracted = extract_clauses_hybrid(sample_lease)
    for clause in extracted:
        print(f"[{clause.clause_type.upper()}] (Conf: {clause.confidence}) -> {clause.text[:80]}...")