Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions warehouse/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
"""
Snowflake-Compatible Data Warehouse Package
Provides ORM-based access to live Snowflake data for fast querying and LLM integration.
"""

from .db_setup import HybridDataWarehouse, YearlongDataWarehouse, create_session
from .models import Base, Metric, Dimension, Fact, FactDimension
# Import query functions for easy access
from .query_wrapper import create_session as query_create_session, query_facts, convert_jargons

__version__ = "2.1.0" # Updated with proper star schema design
__all__ = [
'HybridDataWarehouse',
'YearlongDataWarehouse', # Backward compatibility alias
'create_session',
'query_create_session',
'query_facts',
'convert_jargons',
'Base',
'Metric',
'Dimension',
'Fact',
'FactDimension'
]
22 changes: 22 additions & 0 deletions warehouse/data_ingestion.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
"""
Data Ingestion Module for Year-long Data Warehouse
Processes and loads data from various sources into the SQLite warehouse.
"""

import pandas as pd
import dask.dataframe as dd
import os
import json
import sys
from datetime import datetime, date
from typing import List, Dict, Any, Optional
from .db_setup import YearlongDataWarehouse
import logging

# Add parent directory to path for module imports
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class DataIngestionManager:
Loading