main.py

#!/usr/bin/env python3
"""
Main script for Semantic Log Line Classifier.

Processes log files through a pipeline and classifies lines using LLM-assisted
regex pattern generation.
"""

import argparse
import sys
from pathlib import Path
import time
from typing import Optional

from log_classifier.pipeline import Pipeline
from log_classifier.classifier import (
    Classifier,
    create_llm_client,
    AnthropicConfig,
    OpenAIConfig,
    ClaudeModels,
    OpenAIModels,
)
from log_classifier.classifier.cache_manager import new_cache_manager
from log_classifier.reporter import Reporter


def parse_args():
    """Parse command-line arguments."""
    parser = argparse.ArgumentParser(
        description="Semantic Log Line Classifier - Classify log lines using LLM-assisted pattern matching",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog="""
Examples:
  # Using Anthropic with environment variables
  export ANTHROPIC_API_KEY="your-api-key"
  python main.py input.log output/

  # Using OpenAI with environment variables
  export OPENAI_API_KEY="your-api-key"
  python main.py input.log output/ --provider openai

  # Using explicit API key and model
  python main.py input.log output/ --provider anthropic --api-key "sk-..." --model claude-sonnet-4-20250514

  # Using OpenAI with custom model
  python main.py input.log output/ --provider openai --api-key "sk-..." --model gpt-4o --max-tokens 2048
        """
    )
    
    # Required arguments
    parser.add_argument(
        "input_file",
        type=Path,
        help="Path to input log file"
    )
    
    parser.add_argument(
        "output_dir",
        type=Path,
        help="Path to output directory for reports"
    )
    
    # LLM Configuration
    parser.add_argument(
        "--provider",
        choices=["anthropic", "openai", "claude", "gpt"],
        default="anthropic",
        help="LLM provider (default: anthropic)"
    )
    
    parser.add_argument(
        "--api-key",
        type=str,
        help="API key for LLM provider (can also use environment variables)"
    )
    
    parser.add_argument(
        "--model",
        type=str,
        help="Model identifier (default depends on provider)"
    )
    
    parser.add_argument(
        "--max-tokens",
        type=int,
        help="Maximum tokens per request (default: 1024)"
    )
    
    parser.add_argument(
        "--temperature",
        type=float,
        help="Sampling temperature (default: 0.0 for deterministic output)"
    )
    
    # Optional arguments
    parser.add_argument(
        "--verbose", "-v",
        action="store_true",
        help="Enable verbose output"
    )
    
    parser.add_argument(
        "--cache-dir",
        type=Path,
        help="Directory for LLM cache files (default: ~/.log_classifier_cache)"
    )
    
    return parser.parse_args()


def create_llm_client_from_args(args, cache_manager=None):
    """
    Create LLM client based on command-line arguments.
    
    Args:
        args: Parsed command-line arguments
        cache_manager: Optional cache manager for caching completions
        
    Returns:
        LLM client instance
    """
    # Prepare kwargs for create_llm_client
    kwargs = {}
    if args.api_key:
        kwargs["api_key"] = args.api_key
    if args.model:
        kwargs["model"] = args.model
    if args.max_tokens:
        kwargs["max_tokens"] = args.max_tokens
    if args.temperature is not None:
        kwargs["temperature"] = args.temperature
    
    # Normalize provider name
    provider = args.provider.lower()
    if provider == "claude":
        provider = "anthropic"
    elif provider == "gpt":
        provider = "openai"
    
    try:
        llm_client = create_llm_client(provider, cache_manager=cache_manager, **kwargs)
        return llm_client
    except ValueError as e:
        print(f"Error: {e}", file=sys.stderr)
        sys.exit(1)
    except ImportError as e:
        print(f"Error: {e}", file=sys.stderr)
        print(f"Hint: Install required package with: pip install {'anthropic' if provider == 'anthropic' else 'openai'}", file=sys.stderr)
        sys.exit(1)


def process_log_file(
    input_file: Path,
    output_dir: Path,
    llm_client,
    verbose: bool = False,
) -> None:
    """
    Process log file and generate classification reports.
    
    Args:
        input_file: Path to input log file
        output_dir: Path to output directory
        llm_client: LLM client instance (handles caching internally)
        verbose: Enable verbose output
    """
    # Validate input file
    if not input_file.exists():
        print(f"Error: Input file not found: {input_file}", file=sys.stderr)
        sys.exit(1)
    
    if not input_file.is_file():
        print(f"Error: Input path is not a file: {input_file}", file=sys.stderr)
        sys.exit(1)
    
    # Create output directory
    output_dir.mkdir(parents=True, exist_ok=True)
    
    # Initialize components
    pipeline = Pipeline.default()
    classifier = Classifier(llm_client)
    reporter = Reporter(output_dir)
    
    if verbose:
        print(f"Processing log file: {input_file}")
        print(f"Output directory: {output_dir}")
        print(f"LLM Provider: {type(llm_client).__name__}")
        print()
    
    # Process log file line by line
    processed_lines = 0
    classified_lines = 0
    
    try:
        with open(input_file, 'r', encoding='utf-8') as f:
            for line_num, line in enumerate(f, 1):
                line = line.strip()
                if not line:
                    continue  # Skip empty lines
                
                processed_lines += 1
                
                if verbose and processed_lines % 100 == 0:
                    print(f"Processed {processed_lines} lines...", file=sys.stderr)
                
                # Process line through pipeline
                processed = pipeline.run([line])
                if processed:
                    # Classify the processed line
                    try:
                        class_name = classifier.classify(processed, line, line_num)
                        classified_lines += 1
                        
                        if verbose and classified_lines <= 10:
                            print(f"Line {line_num}: Classified as '{class_name}'", file=sys.stderr)
                    except Exception as e:
                        print(f"Warning: Failed to classify line {line_num}: {e}", file=sys.stderr)
                        if verbose:
                            import traceback
                            traceback.print_exc()
    
    except FileNotFoundError:
        print(f"Error: Could not read file: {input_file}", file=sys.stderr)
        sys.exit(1)
    except Exception as e:
        print(f"Error: Unexpected error processing file: {e}", file=sys.stderr)
        if verbose:
            import traceback
            traceback.print_exc()
        sys.exit(1)
    
    if verbose:
        print(f"\nProcessing complete!")
        print(f"  Total lines processed: {processed_lines}")
        print(f"  Lines classified: {classified_lines}")
        print()
    
    # Generate reports
    classes = classifier.get_all_classes()
    
    if verbose:
        print(f"Generating reports...")
        print(f"  Total classes: {len(classes)}")
    
    try:
        reporter.generate(classes)
        if verbose:
            print(f"Reports generated in: {output_dir}")
    except Exception as e:
        print(f"Error: Failed to generate reports: {e}", file=sys.stderr)
        if verbose:
            import traceback
            traceback.print_exc()
        sys.exit(1)


def main():
    """Main entry point."""
    args = parse_args()
    
    # Setup cache manager if cache directory is provided
    cache_manager = None
    if args.cache_dir:
        cache_manager = new_cache_manager(args.cache_dir)
        print(f"Cache directory: {args.cache_dir}")
    else:
        print("Caching disabled (no --cache-dir specified)")
    
    # Create LLM client (with cache_manager if provided)
    llm_client = create_llm_client_from_args(args, cache_manager=cache_manager)
    
    # Process log file
    process_log_file(
        input_file=args.input_file,
        output_dir=args.output_dir,
        llm_client=llm_client,
        verbose=args.verbose,
    )
    
    print(f"Classification complete! Reports saved to: {args.output_dir}")


if __name__ == "__main__":
    start_time = time.time()
    main()
    end_time = time.time()
    print(f"Time taken: {end_time - start_time} seconds")