Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -248,8 +248,8 @@ MIT License - see [LICENSE](LICENSE) for details.

## Author

**Ajay Maurya** - AI Engineer
[LinkedIn](https://linkedin.com/in/ajaymaurya) | [GitHub](https://github.com/ajaymaurya)
**Ajay Maurya** - AI Engineer
[LinkedIn](https://linkedin.com/in/ajaymauryabbn) | [GitHub](https://github.com/ajaymauryabbn)

---

Expand Down
26 changes: 13 additions & 13 deletions examples/basic_usage.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))

from sql_eval import Evaluator
from sql_eval.datasets import load_ecommerce, get_dataset_info
from sql_eval.datasets import get_dataset_info, load_ecommerce
from sql_eval.llm_providers import get_provider


Expand All @@ -29,7 +29,7 @@ def main():
provider_name = 'ollama'
else:
provider_name = 'openai'

# Show dataset info
print("\n" + "=" * 50)
print("Dataset Information")
Expand All @@ -38,42 +38,42 @@ def main():
print(f"Name: {info['name']}")
print(f"Questions: {info['num_questions']}")
print(f"Tables: {', '.join(info['tables'])}")
print(f"\nDifficulty breakdown:")
print("\nDifficulty breakdown:")
for diff, count in info['difficulty_breakdown'].items():
print(f" {diff}: {count}")

# Load dataset
print("\n" + "=" * 50)
print("Loading Dataset")
print("=" * 50)
test_cases, schema, db = load_ecommerce(with_db=True)
print(f"Loaded {len(test_cases)} test cases")

# Limit to first 5 for quick demo
test_cases = test_cases[:5]
print(f"Running evaluation on {len(test_cases)} questions (limited for demo)")

# Initialize provider
print(f"\nUsing LLM provider: {provider_name}")
try:
provider = get_provider(provider_name)
except Exception as e:
print(f"Error initializing provider: {e}")
return

# Create evaluator
evaluator = Evaluator(
llm_provider=provider,
schema=schema,
db_connector=db,
verbose=True
)

# Run evaluation
print("\n" + "=" * 50)
print("Running Evaluation")
print("=" * 50)

try:
report = evaluator.evaluate(
test_cases,
Expand All @@ -83,24 +83,24 @@ def main():
print(f"\nError during evaluation: {e}")
print("\nTip: Make sure your LLM provider is properly configured")
return

# Show detailed results
print("\n" + "=" * 50)
print("Detailed Results")
print("=" * 50)

for result in report.results:
status = "✓" if result.exact_match else "✗"
print(f"\n{status} {result.question_id}: {result.question[:50]}...")
print(f" Expected: {result.ground_truth_sql[:60]}...")
print(f" Got: {result.generated_sql[:60]}...")
if not result.exact_match and result.partial_scores:
print(f" Structural score: {result.partial_scores.overall_score:.1%}")

# Cleanup
if db:
db.disconnect()

print("\n" + "=" * 50)
print("Done!")
print("=" * 50)
Expand Down
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,8 @@ target-version = ['py310', 'py311', 'py312']

[tool.ruff]
line-length = 100

[tool.ruff.lint]
select = ["E", "F", "W", "I"]
ignore = ["E501"]

Expand Down
4 changes: 2 additions & 2 deletions sql_eval/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@
__author__ = "Ajay Maurya"

from sql_eval.core.evaluator import Evaluator
from sql_eval.core.models import EvaluationCase, EvaluationResult, EvaluationReport
from sql_eval.core.models import EvaluationCase, EvaluationReport, EvaluationResult

__all__ = [
"Evaluator",
"EvaluationCase",
"EvaluationCase",
"EvaluationResult",
"EvaluationReport",
]
Loading