# Install Docling
# pip install docling
 
from docling.document_converter import DocumentConverter
from docling.datamodel.pipeline_options import PdfPipelineOptions
 
# Configure for construction documents with OCR and table extraction
pipeline_options = PdfPipelineOptions()
pipeline_options.do_ocr = True  # Essential for scanned blueprints and specs
pipeline_options.do_table_structure = True  # Extract cost tables and schedules
pipeline_options.generate_page_images = True  # Preserve technical drawings
 
# Initialize converter with construction-optimized settings
converter = DocumentConverter(pipeline_options=pipeline_options)

def analyze_construction_contract(contract_path):
    """Extract key information from construction contracts"""
    
    # Convert contract to structured format
    result = converter.convert(contract_path)
    doc = result.document
    
    # Export to markdown for easier text analysis
    contract_text = doc.export_to_markdown()
    
    # Extract tables (often contain cost breakdowns, schedules)
    contract_tables = []
    for table in doc.tables:
        table_df = table.export_to_dataframe()
        contract_tables.append({
            'page': table.prov[0].page_no if table.prov else 'Unknown',
            'data': table_df
        })
    
    return {
        'full_text': contract_text,
        'tables': contract_tables,
        'images': [img.image for img in doc.pictures if hasattr(img, 'image')],
        'metadata': {
            'pages': len(doc.pages),
            'title': getattr(doc, 'title', 'Untitled Contract')
        }
    }
 
# Example usage
contract_analysis = analyze_construction_contract("project_contract.pdf")
print(f"Contract has {len(contract_analysis['tables'])} tables")
print(f"Extracted text length: {len(contract_analysis['full_text'])} characters")

def process_technical_specifications(spec_path):
    """Process technical specifications with table and image preservation"""
    
    result = converter.convert(spec_path)
    doc = result.document
    
    # Extract structured specification data
    spec_data = {
        'sections': [],
        'technical_tables': [],
        'diagrams': [],
        'requirements': []
    }
    
    # Process tables (material specs, performance requirements)
    for table in doc.tables:
        if table.num_rows > 1:  # Skip single-row headers
            table_data = {
                'location': f"Page {table.prov[0].page_no if table.prov else 'Unknown'}",
                'data': table.export_to_dataframe(),
                'caption': getattr(table, 'caption', '')
            }
            spec_data['technical_tables'].append(table_data)
    
    # Extract images/diagrams
    for img in doc.pictures:
        if hasattr(img, 'image'):
            spec_data['diagrams'].append({
                'page': img.prov[0].page_no if img.prov else 'Unknown',
                'caption': getattr(img, 'caption', ''),
                'image_data': img.image
            })
    
    # Convert to searchable text format
    spec_data['full_text'] = doc.export_to_markdown()
    
    return spec_data
 
# Process multiple specification documents
spec_files = ["electrical_specs.pdf", "structural_specs.pdf", "mechanical_specs.pdf"]
all_specs = {}
 
for spec_file in spec_files:
    print(f"Processing {spec_file}...")
    all_specs[spec_file] = process_technical_specifications(spec_file)
    print(f"Extracted {len(all_specs[spec_file]['technical_tables'])} tables")

import os
from pathlib import Path
import json
 
def batch_process_project_documents(project_folder):
    """Process all documents in a project folder"""
    
    # Supported file types for construction documents
    supported_extensions = ['.pdf', '.docx', '.xlsx', '.pptx', '.html']
    
    project_data = {
        'contracts': {},
        'specifications': {},
        'reports': {},
        'drawings': {},
        'other': {}
    }
    
    # Walk through project directory
    for root, dirs, files in os.walk(project_folder):
        for file in files:
            file_path = os.path.join(root, file)
            file_ext = Path(file).suffix.lower()
            
            if file_ext in supported_extensions:
                try:
                    print(f"Processing: {file}")
                    result = converter.convert(file_path)
                    
                    # Categorize document based on filename/content
                    category = categorize_document(file, result.document)
                    
                    # Extract key information
                    doc_info = {
                        'filename': file,
                        'path': file_path,
                        'pages': len(result.document.pages),
                        'tables': len(result.document.tables),
                        'images': len(result.document.pictures),
                        'text_preview': result.document.export_to_markdown()[:500],
                        'full_content': result.document.export_to_markdown()
                    }
                    
                    project_data[category][file] = doc_info
                    
                except Exception as e:
                    print(f"Error processing {file}: {str(e)}")
    
    return project_data
 
def categorize_document(filename, document):
    """Categorize document based on filename and content"""
    filename_lower = filename.lower()
    
    if any(word in filename_lower for word in ['contract', 'agreement', 'terms']):
        return 'contracts'
    elif any(word in filename_lower for word in ['spec', 'specification', 'requirement']):
        return 'specifications'
    elif any(word in filename_lower for word in ['report', 'inspection', 'test']):
        return 'reports'
    elif any(word in filename_lower for word in ['drawing', 'blueprint', 'plan', 'dwg']):
        return 'drawings'
    else:
        return 'other'
 
# Process entire project
project_documents = batch_process_project_documents("/path/to/project/documents")
 
# Save processed data for later use
with open('project_document_index.json', 'w') as f:
    # Remove full content for the index (too large)
    index_data = {}
    for category, docs in project_documents.items():
        index_data[category] = {}
        for filename, doc_info in docs.items():
            index_data[category][filename] = {
                k: v for k, v in doc_info.items() 
                if k != 'full_content'
            }
    json.dump(index_data, f, indent=2)
 
print(f"Processed {sum(len(docs) for docs in project_documents.values())} documents")

def create_searchable_knowledge_base(processed_documents):
    """Create a searchable knowledge base from processed documents"""
    
    # Combine all document content
    knowledge_base = []
    
    for category, docs in processed_documents.items():
        for filename, doc_info in docs.items():
            # Create searchable entries
            entry = {
                'source': filename,
                'category': category,
                'content': doc_info['full_content'],
                'metadata': {
                    'pages': doc_info['pages'],
                    'tables': doc_info['tables'],
                    'images': doc_info['images']
                }
            }
            knowledge_base.append(entry)
    
    return knowledge_base
 
# Example: Finding information across all project documents
def search_project_documents(knowledge_base, query):
    """Simple text search across processed documents"""
    results = []
    
    for entry in knowledge_base:
        if query.lower() in entry['content'].lower():
            # Extract context around the match
            content_lower = entry['content'].lower()
            match_pos = content_lower.find(query.lower())
            
            # Get surrounding context (500 chars before and after)
            start = max(0, match_pos - 500)
            end = min(len(entry['content']), match_pos + len(query) + 500)
            context = entry['content'][start:end]
            
            results.append({
                'source': entry['source'],
                'category': entry['category'],
                'context': context,
                'metadata': entry['metadata']
            })
    
    return results
 
# Create knowledge base and search
kb = create_searchable_knowledge_base(project_documents)
results = search_project_documents(kb, "safety requirements")
 
for result in results[:3]:  # Show top 3 results
    print(f"Found in: {result['source']} ({result['category']})")
    print(f"Context: {result['context'][:200]}...")
    print("---")

def extract_compliance_requirements(documents):
    """Extract compliance requirements from multiple documents"""
    
    compliance_data = {
        'safety_requirements': [],
        'building_codes': [],
        'environmental_standards': [],
        'quality_specifications': []
    }
    
    # Keywords for different compliance areas
    compliance_keywords = {
        'safety_requirements': ['OSHA', 'safety', 'PPE', 'hazard', 'protection'],
        'building_codes': ['building code', 'IBC', 'structural', 'fire code'],
        'environmental_standards': ['EPA', 'environmental', 'emissions', 'waste'],
        'quality_specifications': ['quality', 'inspection', 'testing', 'standards']
    }
    
    for doc_name, doc_info in documents.items():
        content = doc_info['full_content'].lower()
        
        for category, keywords in compliance_keywords.items():
            for keyword in keywords:
                if keyword in content:
                    # Extract sentences containing the keyword
                    sentences = content.split('.')
                    relevant_sentences = [s.strip() for s in sentences if keyword in s]
                    
                    compliance_data[category].extend([{
                        'source': doc_name,
                        'requirement': sentence,
                        'keyword': keyword
                    } for sentence in relevant_sentences])
    
    return compliance_data

# Example integration with project management APIs
def sync_with_project_management(processed_docs, pm_system_api):
    """Sync processed document data with project management system"""
    
    for category, docs in processed_docs.items():
        for filename, doc_info in docs.items():
            # Extract key dates from contracts/schedules
            if category == 'contracts' and doc_info['tables']:
                # Upload extracted schedule data
                schedule_data = extract_schedule_from_tables(doc_info['tables'])
                pm_system_api.update_project_schedule(schedule_data)
            
            # Upload document metadata
            pm_system_api.add_document_metadata({
                'filename': filename,
                'category': category,
                'page_count': doc_info['pages'],
                'processed_date': datetime.now(),
                'searchable_content': doc_info['text_preview']
            })