Improve code formatting, language detection, and PDF processing efficiency

- Add comprehensive language detection for .env files and Solidity contracts
- Fix code block formatting issues in generated documents
- Remove redundant file information from document headers
- Add chunked PDF processing to prevent memory exhaustion on large repositories
- Implement PDF merging with pdfunite/pdftk for scalable document generation
- Add configurable resource limits for Pandoc operations
- Preserve future-ready API methods with allow(dead_code) annotations

Key improvements:
* Solidity files now properly detected as 'solidity' instead of 'javascript'
* Environment files (.env, .env.local, etc.) get bash syntax highlighting
* Fixed spaced text rendering by removing problematic LaTeX listings option
* Large repositories can now generate PDFs through individual file processing
* Memory-efficient processing prevents system resource exhaustion
* Enhanced error handling for timeout and memory limit scenarios

Author: 0xheartcode

src/config.rs

@@ -108,6 +108,7 @@ impl Default for IgnoreConfig {
 
 impl Config {
     /// Load configuration from a file, falling back to defaults if not found
+    #[allow(dead_code)]
     pub fn load_from_file<P: AsRef<Path>>(path: P) -> Result<Self> {
         let config_path = path.as_ref();
 
@@ -123,6 +124,7 @@ impl Config {
     }
 
     /// Load configuration from the current directory or user's home directory
+    #[allow(dead_code)]
     pub fn load_default() -> Result<Self> {
         // Try to load from current directory first
         let local_config = Path::new("scrollcast.toml");
@@ -143,6 +145,7 @@ impl Config {
     }
 
     /// Save configuration to a file
+    #[allow(dead_code)]
     pub fn save_to_file<P: AsRef<Path>>(&self, path: P) -> Result<()> {
         let content = toml::to_string_pretty(self)
             .context("Failed to serialize configuration")?;
@@ -152,6 +155,7 @@ impl Config {
     }
 
     /// Create a sample configuration file
+    #[allow(dead_code)]
     pub fn create_sample_config<P: AsRef<Path>>(path: P) -> Result<()> {
         let sample_config = Config {
             output: OutputConfig {
@@ -197,6 +201,7 @@ impl Config {
     }
 
     /// Get theme mode as enum
+    #[allow(dead_code)]
     pub fn get_theme_mode(&self) -> ThemeMode {
         match self.theme.mode.as_str() {
             "dark" => ThemeMode::Dark,
@@ -205,11 +210,13 @@ impl Config {
     }
 
     /// Get output directory path
+    #[allow(dead_code)]
     pub fn get_output_dir(&self) -> PathBuf {
         PathBuf::from(&self.output.folder)
     }
 
     /// Get output filename with fallback
+    #[allow(dead_code)]
     pub fn get_output_filename(&self, fallback: &str) -> String {
         self.output.filename
             .as_ref()
@@ -218,6 +225,7 @@ impl Config {
     }
 
     /// Ensure output directory exists
+    #[allow(dead_code)]
     pub fn ensure_output_dir(&self) -> Result<PathBuf> {
         let output_dir = self.get_output_dir();
 

src/file_processor.rs

@@ -39,6 +39,7 @@ impl FileProcessor {
         }
     }
 
+    #[allow(dead_code)]
     pub fn with_ignore_config(mut self, config: IgnoreConfig) -> Self {
         self.ignore_config = config;
         self
@@ -54,6 +55,7 @@ impl FileProcessor {
         self
     }
 
+    #[allow(dead_code)]
     pub fn load_ignore_config_from_path<P: AsRef<Path>>(mut self, path: P) -> Result<Self> {
         let ignore_file_path = path.as_ref().join("scrollcast.ignore");
         if ignore_file_path.exists() {

src/main.rs

@@ -283,11 +283,11 @@ async fn main() -> Result<()> {
     let temp_markdown = temp_dir.join(format!("{}_temp.md", repo_name));
 
     if needs_chunking {
-        process_files_in_chunks(files, repo_name, effective_chunk_size, &temp_markdown, include_toc, verbose, memory_limit, max_file_size_mb).await
+        process_files_in_chunks(&files, repo_name, effective_chunk_size, &temp_markdown, include_toc, verbose, memory_limit, max_file_size_mb).await
             .context("Failed to process files in chunks")?;
     } else {
         let markdown_generator = MarkdownGenerator::new(include_toc, true);
-        let markdown_content = markdown_generator.generate_markdown(files, repo_name)
+        let markdown_content = markdown_generator.generate_markdown(&files, repo_name)
             .context("Failed to generate markdown")?;
         fs::write(&temp_markdown, &markdown_content)
             .context("Failed to write temporary markdown file")?;
@@ -302,20 +302,42 @@ async fn main() -> Result<()> {
     }
 
     // Configure Pandoc
+    let use_chunked_pdf = matches!(output_format, OutputFormat::Pdf) && 
+        (files.len() > 10 || total_size > 1_000_000);
+    
+    if use_chunked_pdf && verbose {
+        println!("📦 Using chunked PDF processing for better memory efficiency");
+    }
+    
     let pandoc_config = PandocConfig {
         output_format,
         highlight_style: theme,
         include_toc,
         syntax_definitions: Vec::new(),
+        use_chunked_pdf,
     };
 
     let converter = PandocConverter::new(pandoc_config)
         .context("Failed to initialize Pandoc converter")?;
 
     // Convert to final format
     println!("{}", "🔄 Converting to final format...".color(Color::Cyan));
-    converter.convert_markdown_to_document(&temp_markdown, output_path, verbose).await
-        .context("Failed to convert markdown to final format")?;
+    
+    if use_chunked_pdf {
+        // For chunked PDF processing, we need individual markdown files
+        let temp_dir = std::env::temp_dir();
+        let chunk_markdowns = generate_individual_markdowns(&files, repo_name, include_toc, &temp_dir, verbose)?;
+        converter.convert_markdown_chunks_to_pdf(&chunk_markdowns, output_path, verbose).await
+            .context("Failed to convert markdown chunks to PDF")?;
+        
+        // Clean up temporary markdown files
+        for markdown_file in &chunk_markdowns {
+            let _ = fs::remove_file(markdown_file);
+        }
+    } else {
+        converter.convert_markdown_to_document(&temp_markdown, output_path, verbose).await
+            .context("Failed to convert markdown to final format")?;
+    }
 
     // Keep temporary file for debugging
     // let _ = fs::remove_file(&temp_markdown);
@@ -341,7 +363,7 @@ async fn main() -> Result<()> {
 }
 
 async fn process_files_in_chunks(
-    files: Vec<FileInfo>,
+    files: &[FileInfo],
     repo_name: &str,
     chunk_size: usize,
     output_path: &Path,
@@ -360,7 +382,7 @@ async fn process_files_in_chunks(
     // Add table of contents for all files
     if include_toc {
         final_markdown.push_str("## Table of Contents\n\n");
-        for file in &files {
+        for file in files {
             let sanitized_path = file.path.replace(['/', '\\'], "-").replace('.', "-");
             let escaped_path = escape_markdown_special_chars(&file.path);
             final_markdown.push_str(&format!("- [{}](#{sanitized_path})\n", escaped_path));
@@ -371,7 +393,7 @@ async fn process_files_in_chunks(
     // Add file tree
     final_markdown.push_str("## File Structure\n\n");
     final_markdown.push_str("```\n");
-    for file in &files {
+    for file in files {
         final_markdown.push_str(&format!("{}\n", file.path));
     }
     final_markdown.push_str("```\n\n");
@@ -382,7 +404,7 @@ async fn process_files_in_chunks(
     // Process files in chunks
     let chunks: Vec<&[FileInfo]> = files.chunks(chunk_size).collect();
     let total_chunks = chunks.len();
-    let mut global_page_number = 1; // Start after title/TOC page
+    let mut _global_page_number = 1; // Start after title/TOC page
     let mut file_counter = 0;
 
     for (chunk_index, chunk) in chunks.iter().enumerate() {
@@ -394,7 +416,7 @@ async fn process_files_in_chunks(
         // Process each file in the chunk
         for file in chunk.iter() {
             file_counter += 1;
-            global_page_number += 1; // Each file gets a new page
+            _global_page_number += 1; // Each file gets a new page
 
             if verbose {
                 sys.refresh_memory();
@@ -432,8 +454,7 @@ async fn process_files_in_chunks(
 
             // Add file header with page numbers
             final_markdown.push_str(&format!("### {} {{#{sanitized_path}}}\n\n", escaped_path));
-            final_markdown.push_str(&format!("**File:** {} | **Size:** {} | **File #{} | Page {}**\n\n", 
-                escaped_path, format_file_size(file.size), file_counter, global_page_number));
+            final_markdown.push_str(&format!("**Size:** {}\n\n", format_file_size(file.size)));
 
             if let Some(language) = &file.language {
                 final_markdown.push_str(&format!("```{}\n", language));
@@ -443,15 +464,12 @@ async fn process_files_in_chunks(
 
             final_markdown.push_str(&processed_content);
 
-            if !file.content.ends_with('\n') {
+            // Ensure there's always a newline before closing backticks
+            if !processed_content.ends_with('\n') {
                 final_markdown.push('\n');
             }
 
             final_markdown.push_str("```\n\n");
-            
-            // Add file info with page numbering
-            final_markdown.push_str(&format!("*File size: {} | File #{} of {} | Page {}*\n\n", 
-                format_file_size(file.size), file_counter, files.len(), global_page_number));
             final_markdown.push_str("---\n\n");
         }
 
@@ -594,6 +612,77 @@ fn list_themes() -> Result<()> {
     Ok(())
 }
 
+fn generate_individual_markdowns(files: &[FileInfo], repo_name: &str, include_toc: bool, temp_dir: &Path, verbose: bool) -> Result<Vec<PathBuf>> {
+    let mut markdown_files = Vec::new();
+    
+    // Generate title page markdown
+    let title_markdown = temp_dir.join("scrollcast_title.md");
+    let mut title_content = String::new();
+    title_content.push_str(&format!("# {}\n\n", repo_name));
+    title_content.push_str(&format!("Generated on: {}\n\n", chrono::Utc::now().format("%Y-%m-%d %H:%M:%S UTC")));
+    
+    if include_toc {
+        title_content.push_str("## Table of Contents\n\n");
+        for file in files {
+            let escaped_path = escape_markdown_special_chars(&file.path);
+            title_content.push_str(&format!("- {}\n", escaped_path));
+        }
+        title_content.push_str("\n");
+    }
+    
+    // Add file tree
+    title_content.push_str("## File Structure\n\n");
+    title_content.push_str("```\n");
+    for file in files {
+        title_content.push_str(&format!("{}\n", file.path));
+    }
+    title_content.push_str("```\n\n");
+    
+    fs::write(&title_markdown, &title_content)
+        .context("Failed to write title markdown")?;
+    markdown_files.push(title_markdown);
+    
+    if verbose {
+        println!("   📄 Generated title page");
+    }
+    
+    // Generate individual file markdowns
+    for (index, file) in files.iter().enumerate() {
+        let file_markdown = temp_dir.join(format!("scrollcast_file_{}.md", index));
+        let mut file_content = String::new();
+        
+        let sanitized_path = file.path.replace(['/', '\\'], "-").replace('.', "-");
+        let escaped_path = escape_markdown_special_chars(&file.path);
+        file_content.push_str(&format!("## {} {{#{sanitized_path}}}\n\n", escaped_path));
+        file_content.push_str(&format!("**Size:** {}\n\n", format_file_size(file.size)));
+        
+        if let Some(language) = &file.language {
+            file_content.push_str(&format!("```{}\n", language));
+        } else {
+            file_content.push_str("```\n");
+        }
+        
+        let processed_content = process_content_for_latex(&file.content);
+        file_content.push_str(&processed_content);
+        
+        if !processed_content.ends_with('\n') {
+            file_content.push('\n');
+        }
+        
+        file_content.push_str("```\n\n");
+        
+        fs::write(&file_markdown, &file_content)
+            .context("Failed to write file markdown")?;
+        markdown_files.push(file_markdown);
+        
+        if verbose {
+            println!("   📄 Generated markdown for: {}", file.path);
+        }
+    }
+    
+    Ok(markdown_files)
+}
+
 fn list_languages() -> Result<()> {
     println!("{}", "Supported programming languages:".color(Color::Blue).bold());
 

src/markdown_generator.rs

@@ -24,7 +24,7 @@ impl MarkdownGenerator {
         }
     }
 
-    pub fn generate_markdown(&self, files: Vec<FileInfo>, repo_name: &str) -> Result<String> {
+    pub fn generate_markdown(&self, files: &[FileInfo], repo_name: &str) -> Result<String> {
         let mut markdown = String::new();
 
         // Title and metadata
@@ -34,7 +34,7 @@ impl MarkdownGenerator {
         // Table of contents
         if self.include_toc {
             markdown.push_str("## Table of Contents\n\n");
-            for file in &files {
+            for file in files {
                 let sanitized_path = file.path.replace(['/', '\\'], "-").replace('.', "-");
                 let escaped_path = self.escape_markdown_special_chars(&file.path);
                 markdown.push_str(&format!("- [{}](#{sanitized_path})\n", escaped_path));
@@ -53,20 +53,19 @@ impl MarkdownGenerator {
         // File contents
         markdown.push_str("## File Contents\n\n");
 
-        let total_files = files.len();
-        let mut global_page_number = 1; // Start after title/TOC page
+        let _total_files = files.len();
+        let mut _global_page_number = 1; // Start after title/TOC page
 
-        for (file_index, file) in files.into_iter().enumerate() {
-            let file_counter = file_index + 1;
-            global_page_number += 1; // Each file gets a new page
+        for (file_index, file) in files.iter().enumerate() {
+            let _file_counter = file_index + 1;
+            _global_page_number += 1; // Each file gets a new page
 
             // Add page break before each file (except the first one)
             markdown.push_str("\n\\newpage\n\n");
             let sanitized_path = file.path.replace(['/', '\\'], "-").replace('.', "-");
             let escaped_path = self.escape_markdown_special_chars(&file.path);
             markdown.push_str(&format!("### {} {{#{sanitized_path}}}\n\n", escaped_path));
-            markdown.push_str(&format!("**File:** {} | **Size:** {} | **File #{} | Page {}**\n\n", 
-                escaped_path, MarkdownGenerator::format_file_size(file.size), file_counter, global_page_number));
+            markdown.push_str(&format!("**Size:** {}\n\n", MarkdownGenerator::format_file_size(file.size)));
 
             if let Some(language) = &file.language {
                 markdown.push_str(&format!("```{}\n", language));
@@ -78,15 +77,12 @@ impl MarkdownGenerator {
             let processed_content = self.process_content_for_latex(&file.content);
             markdown.push_str(&processed_content);
 
-            if !file.content.ends_with('\n') {
+            // Ensure there's always a newline before closing backticks
+            if !processed_content.ends_with('\n') {
                 markdown.push('\n');
             }
 
             markdown.push_str("```\n\n");
-            
-            // Add file info with page numbering
-            markdown.push_str(&format!("*File size: {} | File #{} of {} | Page {}*\n\n", 
-                MarkdownGenerator::format_file_size(file.size), file_counter, total_files, global_page_number));
             markdown.push_str("---\n\n");
         }
 
@@ -135,6 +131,20 @@ impl MarkdownGenerator {
 
     pub fn detect_language(file_path: &str) -> Option<String> {
         let path = Path::new(file_path);
+        
+        // Handle special cases first
+        if let Some(file_name) = path.file_name() {
+            let file_name_str = file_name.to_string_lossy();
+            // Handle .env files and variants like .env.local, .env.production, etc.
+            if file_name_str.starts_with(".env") {
+                return Some("bash".to_string());
+            }
+            // Handle Dockerfile variants
+            if file_name_str.to_lowercase().starts_with("dockerfile") {
+                return Some("dockerfile".to_string());
+            }
+        }
+        
         let extension = path.extension()?.to_str()?;
 
         let language = match extension.to_lowercase().as_str() {
@@ -169,7 +179,8 @@ impl MarkdownGenerator {
             "r" => "r",
             "sql" => "sql",
             "dockerfile" => "dockerfile",
-            "sol" => "javascript",  // Use JavaScript highlighting for Solidity as fallback
+            "env" => "bash",  // Environment files use shell-like syntax
+            "sol" => "solidity",
             "vy" => "python",     // Vyper (use python highlighting as fallback)
             "move" => "rust",     // Move language (use rust as fallback)
             _ => return None,
@@ -201,7 +212,7 @@ impl MarkdownGenerator {
                 let mut current_line = String::new();
                 let chars: Vec<char> = line.chars().collect();
 
-                for (i, &ch) in chars.iter().enumerate() {
+                for (_i, &ch) in chars.iter().enumerate() {
                     current_line.push(ch);
 
                     // Break at 100 characters or at natural breakpoints