@@ -173,8 +180,8 @@
Interpretation Types
if (labelSelect) {
// Preserve current selection
const currentLabel = labelSelect.value;
- // Get unique labels from schema
- const labels = [...new Set(schema.nodes?.map(n => n.label) || [])].sort();
+ // Get unique labels from schema, filtering out empty/null values
+ const labels = [...new Set(schema.nodes?.map(n => n.label).filter(l => l) || [])].sort();
labelSelect.innerHTML = '
' +
labels.map(l => `
`).join('');
// Restore selection if still valid
@@ -186,8 +193,8 @@
Interpretation Types
if (relSelect) {
// Preserve current selection
const currentRel = relSelect.value;
- // Get unique relationship types from schema
- const rels = [...new Set(schema.edges?.map(e => e.rel_type) || [])].sort();
+ // Get unique relationship types from schema, filtering out empty/null values
+ const rels = [...new Set(schema.edges?.map(e => e.rel_type).filter(r => r) || [])].sort();
relSelect.innerHTML = '
' +
rels.map(r => `
`).join('');
// Restore selection if still valid
diff --git a/scidk/ui/templates/plugins.html b/scidk/ui/templates/plugins.html
index 331d537..3f09938 100644
--- a/scidk/ui/templates/plugins.html
+++ b/scidk/ui/templates/plugins.html
@@ -1,5 +1,5 @@
{% extends 'base.html' %}
-{% block title %}SciDK - Plugins{% endblock %}
+{% block title %}-SciDK-> Plugins{% endblock %}
{% block content %}
Plugins
Plugin registry and management UI will appear here.
diff --git a/scidk/ui/templates/settings.html b/scidk/ui/templates/settings.html
index c892300..bccc7fe 100644
--- a/scidk/ui/templates/settings.html
+++ b/scidk/ui/templates/settings.html
@@ -1,23 +1,109 @@
{% extends 'base.html' %}
-{% block title %}SciDK - Settings{% endblock %}
+{% block title %}-SciDK-> Settings{% endblock %}
+{% block head %}
+
+{% endblock %}
{% block content %}
-
Settings
-
Basic runtime information and counts.
-
- - Host: {{ info.host }}
- - Port: {{ info.port }}
- - Debug: {{ info.debug }}
- - Datasets: {{ info.dataset_count }}
- - Interpreters: {{ info.interpreter_count }}
-
-
- Channel: {{ info.channel or 'stable' }}
- Providers: {{ info.providers }}
- Files viewer: {{ info.files_viewer or '(default)' }}
-
+
+
+
+
+
+
+
+
+ General
+ Basic runtime information and counts.
+
+ - Host: {{ info.host }}
+ - Port: {{ info.port }}
+ - Debug: {{ info.debug }}
+ - Datasets: {{ info.dataset_count }}
+ - Interpreters: {{ info.interpreter_count }}
+
+
+ Channel: {{ info.channel or 'stable' }}
+ Providers: {{ info.providers }}
+ Files viewer: {{ info.files_viewer or '(default)' }}
+
+
-
- Neo4j Connection
+
+
+ Neo4j Connection
+ Configure Neo4j database connection and settings.
@@ -57,10 +143,11 @@
Neo4j Connection
You can also set env vars: NEO4J_URI, NEO4J_USER, NEO4J_PASSWORD, SCIDK_NEO4J_DATABASE
If your Neo4j has authentication disabled, set environment variable NEO4J_AUTH=none before starting the app.
-
+
-
- Interpreters
+
+
+ Interpreters
Registered interpreter mappings and selection rules.
Mappings (extension → interpreter ids)
@@ -175,19 +262,24 @@ Interpreter toggles
fetchEffective().then(render);
})();
-
+
-
- Plugins
+
+
+ Plugins
Plugin registry summary.
- Registered interpreter count: {{ interp_count or 0 }}
- Extensions mapped: {{ ext_count or 0 }}
-
+
-
- Rclone Interpretation
+
+
+ Rclone
+ Configure rclone settings for interpretation and mounts.
+
+ Interpretation
Tune streaming-based interpretation from rclone remotes. For very large scans, consider mounting the remote.
@@ -203,10 +295,8 @@
Rclone Interpretation
-
-
- Rclone Mounts
+ Mounts
Manage rclone mounts under ./data/mounts.
@@ -242,9 +332,282 @@
Rclone Mounts
Note: On Windows, cmount/WinFsp may be required; this UI targets Linux/macOS primarily.
-
-{% endblock %}
-{% block head %}
+
+
+
+
+ Integrations
+ Configure integration mappings, API endpoints, and matching options.
+
+ API Endpoint Mappings
+ Define API endpoints that map to Label types in SciDK.
+
+
+
+
Add New Endpoint
+
+
+
+
+
+
+
+
Registered Endpoints
+
+
No endpoints registered yet
+
Table Format Registry
+ Manage table formats for importing CSV, TSV, Excel, and Parquet files as link sources.
+
+
+
+
Add Custom Format
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Registered Formats
+
+
+ Fuzzy Matching Options
+ Configure fuzzy matching algorithms for entity resolution in link creation.
+
+
+
+
Global Fuzzy Matching Settings
+
+
+
+
+
+
+
Levenshtein: general fuzzy matching | Jaro-Winkler: names | Phonetic: sound-alike
+
+
+
+
+
Minimum similarity score (0-100%) to consider a match
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ Advanced Options
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
Hybrid Matching Architecture
+
+ Phase 1 (Client-Side): Pre-import matching using rapidfuzz - match external API/CSV data before pushing to Neo4j.
+
+
+ Phase 2 (Server-Side): Post-import matching using Neo4j APOC functions - ultra-fast in-database entity resolution for existing nodes.
+
+
+
+
+
+
{% endblock %}
diff --git a/scidk/ui/templates/workbook.html b/scidk/ui/templates/workbook.html
index 7f5d1d9..3f507bf 100644
--- a/scidk/ui/templates/workbook.html
+++ b/scidk/ui/templates/workbook.html
@@ -1,5 +1,5 @@
{% extends 'base.html' %}
-{% block title %}SciDK - Workbook Viewer{% endblock %}
+{% block title %}-SciDK-> Workbook Viewer{% endblock %}
{% block content %}
← Back to Files
{% if not dataset %}
diff --git a/scidk/web/routes/__init__.py b/scidk/web/routes/__init__.py
index 24eccba..6118be3 100644
--- a/scidk/web/routes/__init__.py
+++ b/scidk/web/routes/__init__.py
@@ -36,6 +36,8 @@ def register_blueprints(app):
from . import api_annotations
from . import api_labels
from . import api_links
+ from . import api_integrations
+ from . import api_settings
# Register UI blueprint
app.register_blueprint(ui.bp)
@@ -51,4 +53,6 @@ def register_blueprints(app):
app.register_blueprint(api_providers.bp)
app.register_blueprint(api_annotations.bp)
app.register_blueprint(api_labels.bp)
- app.register_blueprint(api_links.bp)
+ app.register_blueprint(api_integrations.bp)
+ app.register_blueprint(api_links.bp) # Keep for backward compatibility
+ app.register_blueprint(api_settings.bp)
diff --git a/scidk/web/routes/api_admin.py b/scidk/web/routes/api_admin.py
index 3c04f69..4c18dd8 100644
--- a/scidk/web/routes/api_admin.py
+++ b/scidk/web/routes/api_admin.py
@@ -367,3 +367,76 @@ def api_admin_cleanup_test_labels():
except Exception as e:
return jsonify({'error': str(e)}), 500
+
+@bp.post('/admin/cleanup-test-endpoints')
+def api_admin_cleanup_test_endpoints():
+ """Remove test API endpoints from the database (endpoints with test prefixes).
+
+ This endpoint cleans up API endpoints created during testing that accumulate over time.
+
+ Returns:
+ JSON with counts of deleted endpoints
+ """
+ try:
+ import sqlite3
+
+ # Test endpoint patterns to delete
+ test_patterns = [
+ 'Test%', # Test Users API, etc
+ 'E2E%', # E2E test endpoints
+ 'Secure%', # Secure API from auth tests
+ 'Updated%', # Updated API from update tests
+ 'Bearer%', # Bearer API from auth tests
+ 'API%Key%', # API Key API from auth tests
+ '%JSONPath%', # JSONPath API tests
+ 'Original%', # Original API from edit tests
+ 'Delete%', # Delete Me API from delete tests
+ 'Cancel%', # Cancel Test API from cancel tests
+ ]
+
+ # Use settings DB (where API endpoints are stored, not path_index)
+ settings_db = current_app.config.get('SCIDK_SETTINGS_DB', 'scidk_settings.db')
+ conn = sqlite3.connect(settings_db)
+ conn.execute('PRAGMA journal_mode=WAL')
+ try:
+ cur = conn.cursor()
+
+ # Check if api_endpoints table exists
+ cur.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='api_endpoints'")
+ if not cur.fetchone():
+ return jsonify({
+ 'deleted_endpoints': 0,
+ 'message': 'API endpoints table does not exist'
+ }), 200
+
+ # Collect endpoint names that match test patterns
+ deleted_endpoints = []
+ total_deleted = 0
+
+ for pattern in test_patterns:
+ cur.execute("SELECT name FROM api_endpoints WHERE name LIKE ?", (pattern,))
+ matching_endpoints = [row[0] for row in cur.fetchall()]
+ deleted_endpoints.extend(matching_endpoints)
+
+ # Delete matching endpoints
+ cur.execute("DELETE FROM api_endpoints WHERE name LIKE ?", (pattern,))
+ total_deleted += cur.rowcount
+
+ conn.commit()
+
+ return jsonify({
+ 'deleted_endpoints': total_deleted,
+ 'endpoint_names': deleted_endpoints[:10] + (['...'] if len(deleted_endpoints) > 10 else []),
+ 'total_test_endpoints_found': len(deleted_endpoints),
+ 'message': f'Successfully deleted {total_deleted} test endpoints'
+ }), 200
+
+ finally:
+ try:
+ conn.close()
+ except Exception:
+ pass
+
+ except Exception as e:
+ return jsonify({'error': str(e)}), 500
+
diff --git a/scidk/web/routes/api_integrations.py b/scidk/web/routes/api_integrations.py
new file mode 100644
index 0000000..33ba2ba
--- /dev/null
+++ b/scidk/web/routes/api_integrations.py
@@ -0,0 +1,397 @@
+"""
+Blueprint for Integrations API routes.
+
+Provides REST endpoints for:
+- Integration definitions CRUD
+- Preview and execution of integration jobs
+- Job status tracking
+"""
+from flask import Blueprint, jsonify, request, current_app
+
+bp = Blueprint('integrations', __name__, url_prefix='/api')
+
+
+def _get_link_service():
+ """Get or create LinkService instance."""
+ from ...services.link_service import LinkService
+ if 'link_service' not in current_app.extensions.get('scidk', {}):
+ if 'scidk' not in current_app.extensions:
+ current_app.extensions['scidk'] = {}
+ current_app.extensions['scidk']['link_service'] = LinkService(current_app)
+ return current_app.extensions['scidk']['link_service']
+
+
+@bp.route('/integrations', methods=['GET'])
+def list_links():
+ """
+ Get all link definitions.
+
+ Returns:
+ {
+ "status": "success",
+ "links": [
+ {
+ "id": "uuid",
+ "name": "Author to File",
+ "source_type": "csv",
+ "target_type": "label",
+ "match_strategy": "property",
+ "relationship_type": "AUTHORED",
+ ...
+ }
+ ]
+ }
+ """
+ try:
+ service = _get_link_service()
+ links = service.list_link_definitions()
+ return jsonify({
+ 'status': 'success',
+ 'links': links
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/integrations/', methods=['GET'])
+def get_link(link_id):
+ """
+ Get a specific link definition by ID.
+
+ Returns:
+ {
+ "status": "success",
+ "link": {...}
+ }
+ """
+ try:
+ service = _get_link_service()
+ link = service.get_link_definition(link_id)
+
+ if not link:
+ return jsonify({
+ 'status': 'error',
+ 'error': f'Link "{link_id}" not found'
+ }), 404
+
+ return jsonify({
+ 'status': 'success',
+ 'link': link
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/integrations', methods=['POST'])
+def create_or_update_link():
+ """
+ Create or update a link definition.
+
+ Request body:
+ {
+ "id": "optional-uuid",
+ "name": "Author to File",
+ "source_type": "csv",
+ "source_config": {
+ "csv_data": "name,email,file_path\\nAlice,alice@ex.com,file1.txt"
+ },
+ "target_type": "label",
+ "target_config": {
+ "label": "File"
+ },
+ "match_strategy": "property",
+ "match_config": {
+ "source_field": "file_path",
+ "target_field": "path"
+ },
+ "relationship_type": "AUTHORED",
+ "relationship_props": {
+ "date": "2024-01-15"
+ }
+ }
+
+ Returns:
+ {
+ "status": "success",
+ "link": {...}
+ }
+ """
+ try:
+ data = request.get_json(force=True, silent=True) or {}
+
+ if not data.get('name'):
+ return jsonify({
+ 'status': 'error',
+ 'error': 'Link name is required'
+ }), 400
+
+ service = _get_link_service()
+ link = service.save_link_definition(data)
+
+ return jsonify({
+ 'status': 'success',
+ 'link': link
+ }), 200
+ except ValueError as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 400
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/integrations/', methods=['DELETE'])
+def delete_link(link_id):
+ """
+ Delete a link definition.
+
+ Returns:
+ {
+ "status": "success",
+ "message": "Link deleted"
+ }
+ """
+ try:
+ service = _get_link_service()
+ deleted = service.delete_link_definition(link_id)
+
+ if not deleted:
+ return jsonify({
+ 'status': 'error',
+ 'error': f'Link "{link_id}" not found'
+ }), 404
+
+ return jsonify({
+ 'status': 'success',
+ 'message': f'Link "{link_id}" deleted'
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/integrations//preview', methods=['POST'])
+def preview_link(link_id):
+ """
+ Preview link matches (dry-run).
+
+ Request body (optional):
+ {
+ "limit": 10
+ }
+
+ Returns:
+ {
+ "status": "success",
+ "matches": [
+ {
+ "source": {"name": "Alice", "email": "alice@ex.com", ...},
+ "target": {"path": "file1.txt", ...}
+ }
+ ],
+ "count": 5
+ }
+ """
+ try:
+ service = _get_link_service()
+ link = service.get_link_definition(link_id)
+
+ if not link:
+ return jsonify({
+ 'status': 'error',
+ 'error': f'Link "{link_id}" not found'
+ }), 404
+
+ data = request.get_json(force=True, silent=True) or {}
+ limit = data.get('limit', 10)
+
+ matches = service.preview_matches(link, limit=limit)
+
+ return jsonify({
+ 'status': 'success',
+ 'matches': matches,
+ 'count': len(matches)
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/integrations//execute', methods=['POST'])
+def execute_link(link_id):
+ """
+ Execute link job (create relationships in Neo4j).
+
+ Returns:
+ {
+ "status": "success",
+ "job_id": "uuid"
+ }
+ """
+ try:
+ service = _get_link_service()
+ job_id = service.execute_link_job(link_id)
+
+ return jsonify({
+ 'status': 'success',
+ 'job_id': job_id
+ }), 200
+ except ValueError as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 404
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/integrations/jobs/', methods=['GET'])
+def get_job_status(job_id):
+ """
+ Get job status and progress.
+
+ Returns:
+ {
+ "status": "success",
+ "job": {
+ "id": "uuid",
+ "link_def_id": "uuid",
+ "status": "completed",
+ "preview_count": 0,
+ "executed_count": 23,
+ "error": null,
+ "started_at": 1234567890.123,
+ "completed_at": 1234567895.456
+ }
+ }
+ """
+ try:
+ service = _get_link_service()
+ job = service.get_job_status(job_id)
+
+ if not job:
+ return jsonify({
+ 'status': 'error',
+ 'error': f'Job "{job_id}" not found'
+ }), 404
+
+ return jsonify({
+ 'status': 'success',
+ 'job': job
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/integrations/jobs', methods=['GET'])
+def list_jobs():
+ """
+ List recent link jobs.
+
+ Query params:
+ - limit: Maximum number of jobs to return (default: 20)
+
+ Returns:
+ {
+ "status": "success",
+ "jobs": [...]
+ }
+ """
+ try:
+ limit = int(request.args.get('limit', 20))
+ service = _get_link_service()
+ jobs = service.list_jobs(limit=limit)
+
+ return jsonify({
+ 'status': 'success',
+ 'jobs': jobs
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/integrations/available-labels', methods=['GET'])
+def get_available_labels():
+ """
+ Get list of available labels for dropdown population.
+
+ Returns:
+ {
+ "status": "success",
+ "labels": [
+ {"name": "Person", "properties": [...]},
+ {"name": "File", "properties": [...]}
+ ]
+ }
+ """
+ try:
+ from ...services.label_service import LabelService
+ label_service = LabelService(current_app)
+ labels = label_service.list_labels()
+
+ return jsonify({
+ 'status': 'success',
+ 'labels': labels
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/integrations/migrate', methods=['POST'])
+def migrate_links():
+ """
+ Migrate existing link definitions to Label→Label model.
+
+ Returns:
+ {
+ "status": "success",
+ "report": {
+ "migrated": [...],
+ "skipped": [...],
+ "errors": [...]
+ }
+ }
+ """
+ try:
+ from ...services.link_migration import migrate_all_links, generate_migration_report
+ service = _get_link_service()
+
+ results = migrate_all_links(service)
+ report_text = generate_migration_report(results)
+
+ return jsonify({
+ 'status': 'success',
+ 'results': results,
+ 'report': report_text
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
diff --git a/scidk/web/routes/api_labels.py b/scidk/web/routes/api_labels.py
index 8231a82..2e2ebd0 100644
--- a/scidk/web/routes/api_labels.py
+++ b/scidk/web/routes/api_labels.py
@@ -483,3 +483,212 @@ def batch_delete_labels():
except Exception as e:
return jsonify({'status': 'error', 'error': str(e)}), 500
+
+
+@bp.route('/labels//instances', methods=['GET'])
+def get_label_instances(name):
+ """
+ Get instances of a label from Neo4j.
+
+ Query params:
+ - limit: max number of instances (default: 100)
+ - offset: pagination offset (default: 0)
+
+ Returns:
+ {
+ "status": "success",
+ "instances": [
+ {"id": "...", "properties": {"name": "John", "age": 30}},
+ ...
+ ],
+ "total": 150,
+ "limit": 100,
+ "offset": 0
+ }
+ """
+ try:
+ service = _get_label_service()
+ limit = int(request.args.get('limit', 100))
+ offset = int(request.args.get('offset', 0))
+
+ result = service.get_label_instances(name, limit=limit, offset=offset)
+
+ if result.get('status') == 'error':
+ return jsonify(result), 500
+
+ return jsonify(result), 200
+
+ except ValueError as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 404
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/labels//instance-count', methods=['GET'])
+def get_label_instance_count(name):
+ """
+ Get count of instances for a label from Neo4j.
+
+ Returns:
+ {
+ "status": "success",
+ "count": 42
+ }
+ """
+ try:
+ service = _get_label_service()
+ result = service.get_label_instance_count(name)
+
+ if result.get('status') == 'error':
+ return jsonify(result), 500
+
+ return jsonify(result), 200
+
+ except ValueError as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 404
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/labels//instances/', methods=['PATCH'])
+def update_label_instance(name, instance_id):
+ """
+ Update a single property of a label instance in Neo4j.
+
+ Request body:
+ {
+ "property": "name",
+ "value": "New Value"
+ }
+
+ Returns:
+ {
+ "status": "success",
+ "instance": {...}
+ }
+ """
+ try:
+ data = request.get_json(force=True, silent=True) or {}
+ property_name = data.get('property')
+ property_value = data.get('value')
+
+ if not property_name:
+ return jsonify({
+ 'status': 'error',
+ 'error': 'Property name is required'
+ }), 400
+
+ service = _get_label_service()
+ result = service.update_label_instance(name, instance_id, property_name, property_value)
+
+ if result.get('status') == 'error':
+ return jsonify(result), 500
+
+ return jsonify(result), 200
+
+ except ValueError as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 404
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/labels/import/eda', methods=['POST'])
+def import_eda_file():
+ """
+ Import experimental design from NC3Rs EDA file.
+
+ Expects multipart/form-data with 'file' field containing .eda file.
+
+ Returns:
+ {
+ "status": "success",
+ "imported": {
+ "labels": 3,
+ "relationships": 5
+ },
+ "labels": [...]
+ }
+ """
+ import tempfile
+ import os
+ from werkzeug.utils import secure_filename
+ from ...interpreters.eda_interpreter import parse_eda_file, eda_to_labels
+
+ try:
+ # Check if file present
+ if 'file' not in request.files:
+ return jsonify({'status': 'error', 'error': 'No file provided'}), 400
+
+ file = request.files['file']
+ if file.filename == '':
+ return jsonify({'status': 'error', 'error': 'Empty filename'}), 400
+
+ if not file.filename.endswith('.eda'):
+ return jsonify({'status': 'error', 'error': 'File must be .eda format'}), 400
+
+ # Save to temporary file
+ filename = secure_filename(file.filename)
+ with tempfile.NamedTemporaryFile(delete=False, suffix='.eda') as tmp:
+ file.save(tmp.name)
+ tmp_path = tmp.name
+
+ try:
+ # Parse EDA file
+ eda_nodes, eda_edges = parse_eda_file(tmp_path)
+ labels_to_create = eda_to_labels(eda_nodes, eda_edges)
+
+ # Create labels
+ service = _get_label_service()
+ created = []
+ skipped = []
+
+ for label_def in labels_to_create:
+ try:
+ result = service.save_label(label_def)
+ created.append(result)
+ except Exception as e:
+ # Skip duplicates
+ skipped.append(label_def['name'])
+ continue
+
+ total_relationships = sum(len(l.get('relationships', [])) for l in labels_to_create)
+
+ response = {
+ 'status': 'success',
+ 'imported': {
+ 'labels': len(created),
+ 'relationships': total_relationships
+ },
+ 'labels': created
+ }
+
+ if skipped:
+ response['skipped'] = skipped
+
+ return jsonify(response), 200
+
+ finally:
+ # Clean up temp file
+ if os.path.exists(tmp_path):
+ os.remove(tmp_path)
+
+ except Exception as e:
+ return jsonify({'status': 'error', 'error': str(e)}), 500
diff --git a/scidk/web/routes/api_links.py b/scidk/web/routes/api_links.py
index ef67a17..fd69586 100644
--- a/scidk/web/routes/api_links.py
+++ b/scidk/web/routes/api_links.py
@@ -1,12 +1,18 @@
"""
-Blueprint for Links API routes.
+Blueprint for Links API routes (DEPRECATED).
+
+**DEPRECATED**: This module is kept for backward compatibility only.
+Use api_integrations.py instead. All /api/links/* endpoints redirect to /api/integrations/*
Provides REST endpoints for:
-- Link definitions CRUD
-- Preview and execution of link jobs
-- Job status tracking
+- Link definitions CRUD (deprecated, use integrations)
+- Preview and execution of link jobs (deprecated, use integrations)
+- Job status tracking (deprecated, use integrations)
"""
-from flask import Blueprint, jsonify, request, current_app
+from flask import Blueprint, jsonify, request, current_app, redirect, url_for
+import logging
+
+logger = logging.getLogger(__name__)
bp = Blueprint('links', __name__, url_prefix='/api')
@@ -26,6 +32,8 @@ def list_links():
"""
Get all link definitions.
+ **DEPRECATED**: Use /api/integrations instead.
+
Returns:
{
"status": "success",
@@ -42,6 +50,7 @@ def list_links():
]
}
"""
+ logger.warning("DEPRECATED: /api/links endpoint called. Use /api/integrations instead.")
try:
service = _get_link_service()
links = service.list_link_definitions()
@@ -331,3 +340,67 @@ def list_jobs():
'status': 'error',
'error': str(e)
}), 500
+
+
+@bp.route('/links/available-labels', methods=['GET'])
+def get_available_labels():
+ """
+ Get list of available labels for dropdown population.
+
+ Returns:
+ {
+ "status": "success",
+ "labels": [
+ {"name": "Person", "properties": [...]},
+ {"name": "File", "properties": [...]}
+ ]
+ }
+ """
+ try:
+ from ...services.label_service import LabelService
+ label_service = LabelService(current_app)
+ labels = label_service.list_labels()
+
+ return jsonify({
+ 'status': 'success',
+ 'labels': labels
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/links/migrate', methods=['POST'])
+def migrate_links():
+ """
+ Migrate existing link definitions to Label→Label model.
+
+ Returns:
+ {
+ "status": "success",
+ "report": {
+ "migrated": [...],
+ "skipped": [...],
+ "errors": [...]
+ }
+ }
+ """
+ try:
+ from ...services.link_migration import migrate_all_links, generate_migration_report
+ service = _get_link_service()
+
+ results = migrate_all_links(service)
+ report_text = generate_migration_report(results)
+
+ return jsonify({
+ 'status': 'success',
+ 'results': results,
+ 'report': report_text
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
diff --git a/scidk/web/routes/api_settings.py b/scidk/web/routes/api_settings.py
new file mode 100644
index 0000000..148337b
--- /dev/null
+++ b/scidk/web/routes/api_settings.py
@@ -0,0 +1,747 @@
+"""
+Blueprint for Settings API routes.
+
+Provides REST endpoints for:
+- API endpoint registry CRUD
+- Endpoint connection testing
+- Settings persistence
+"""
+from flask import Blueprint, jsonify, request, current_app
+import requests
+from jsonpath_ng import parse as jsonpath_parse
+
+bp = Blueprint('settings', __name__, url_prefix='/api')
+
+
+def _get_endpoint_registry():
+ """Get or create APIEndpointRegistry instance."""
+ from ...core.api_endpoint_registry import APIEndpointRegistry, get_encryption_key
+
+ if 'api_endpoint_registry' not in current_app.extensions.get('scidk', {}):
+ if 'scidk' not in current_app.extensions:
+ current_app.extensions['scidk'] = {}
+
+ # Get settings DB path
+ settings_db = current_app.config.get('SCIDK_SETTINGS_DB', 'scidk_settings.db')
+ encryption_key = get_encryption_key()
+
+ current_app.extensions['scidk']['api_endpoint_registry'] = APIEndpointRegistry(
+ db_path=settings_db,
+ encryption_key=encryption_key
+ )
+
+ return current_app.extensions['scidk']['api_endpoint_registry']
+
+
+@bp.route('/settings/api-endpoints', methods=['GET'])
+def list_api_endpoints():
+ """
+ Get all registered API endpoints.
+
+ Returns:
+ {
+ "status": "success",
+ "endpoints": [
+ {
+ "id": "uuid",
+ "name": "Users API",
+ "url": "https://api.example.com/users",
+ "auth_method": "bearer",
+ "json_path": "$.data[*]",
+ "target_label": "User",
+ "field_mappings": {"email": "email", "name": "fullName"},
+ "created_at": 1234567890.123,
+ "updated_at": 1234567890.123
+ }
+ ]
+ }
+ """
+ try:
+ registry = _get_endpoint_registry()
+ endpoints = registry.list_endpoints()
+ return jsonify({
+ 'status': 'success',
+ 'endpoints': endpoints
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/api-endpoints/', methods=['GET'])
+def get_api_endpoint(endpoint_id):
+ """
+ Get a specific API endpoint by ID.
+
+ Returns:
+ {
+ "status": "success",
+ "endpoint": {...}
+ }
+ """
+ try:
+ registry = _get_endpoint_registry()
+ endpoint = registry.get_endpoint(endpoint_id)
+
+ if not endpoint:
+ return jsonify({
+ 'status': 'error',
+ 'error': f'Endpoint "{endpoint_id}" not found'
+ }), 404
+
+ return jsonify({
+ 'status': 'success',
+ 'endpoint': endpoint
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/api-endpoints', methods=['POST'])
+def create_api_endpoint():
+ """
+ Create a new API endpoint configuration.
+
+ Request body:
+ {
+ "name": "Users API",
+ "url": "https://api.example.com/users",
+ "auth_method": "bearer", // "none", "bearer", or "api_key"
+ "auth_value": "token123", // optional
+ "json_path": "$.data[*]", // optional
+ "target_label": "User", // optional
+ "field_mappings": { // optional
+ "email": "email",
+ "name": "fullName"
+ }
+ }
+
+ Returns:
+ {
+ "status": "success",
+ "endpoint": {...}
+ }
+ """
+ try:
+ data = request.get_json()
+ if not data:
+ return jsonify({
+ 'status': 'error',
+ 'error': 'Request body must be JSON'
+ }), 400
+
+ registry = _get_endpoint_registry()
+ endpoint = registry.create_endpoint(data)
+
+ return jsonify({
+ 'status': 'success',
+ 'endpoint': endpoint
+ }), 201
+ except ValueError as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 400
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/api-endpoints/', methods=['PUT', 'PATCH'])
+def update_api_endpoint(endpoint_id):
+ """
+ Update an existing API endpoint.
+
+ Request body: Same as create, but all fields optional
+
+ Returns:
+ {
+ "status": "success",
+ "endpoint": {...}
+ }
+ """
+ try:
+ data = request.get_json()
+ if not data:
+ return jsonify({
+ 'status': 'error',
+ 'error': 'Request body must be JSON'
+ }), 400
+
+ registry = _get_endpoint_registry()
+ endpoint = registry.update_endpoint(endpoint_id, data)
+
+ return jsonify({
+ 'status': 'success',
+ 'endpoint': endpoint
+ }), 200
+ except ValueError as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 400
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/api-endpoints/', methods=['DELETE'])
+def delete_api_endpoint(endpoint_id):
+ """
+ Delete an API endpoint.
+
+ Returns:
+ {
+ "status": "success"
+ }
+ """
+ try:
+ registry = _get_endpoint_registry()
+ deleted = registry.delete_endpoint(endpoint_id)
+
+ if not deleted:
+ return jsonify({
+ 'status': 'error',
+ 'error': f'Endpoint "{endpoint_id}" not found'
+ }), 404
+
+ return jsonify({
+ 'status': 'success'
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/api-endpoints//test', methods=['POST'])
+def test_api_endpoint(endpoint_id):
+ """
+ Test an API endpoint connection and return sample data.
+
+ Returns:
+ {
+ "status": "success",
+ "test_result": {
+ "success": true,
+ "status_code": 200,
+ "sample_data": [...], // First 5 records after JSONPath extraction
+ "total_records": 100, // Total records found
+ "error": null
+ }
+ }
+ """
+ try:
+ registry = _get_endpoint_registry()
+ endpoint = registry.get_endpoint(endpoint_id)
+
+ if not endpoint:
+ return jsonify({
+ 'status': 'error',
+ 'error': f'Endpoint "{endpoint_id}" not found'
+ }), 404
+
+ # Get decrypted auth value
+ auth_value = registry.get_decrypted_auth(endpoint_id)
+
+ # Build request headers
+ headers = {}
+ if endpoint['auth_method'] == 'bearer' and auth_value:
+ headers['Authorization'] = f'Bearer {auth_value}'
+ elif endpoint['auth_method'] == 'api_key' and auth_value:
+ headers['X-API-Key'] = auth_value
+
+ # Make request
+ try:
+ response = requests.get(endpoint['url'], headers=headers, timeout=10)
+ response.raise_for_status()
+ except requests.exceptions.Timeout:
+ return jsonify({
+ 'status': 'success',
+ 'test_result': {
+ 'success': False,
+ 'error': 'Request timed out after 10 seconds'
+ }
+ }), 200
+ except requests.exceptions.RequestException as e:
+ return jsonify({
+ 'status': 'success',
+ 'test_result': {
+ 'success': False,
+ 'error': str(e)
+ }
+ }), 200
+
+ # Parse JSON response
+ try:
+ data = response.json()
+ except Exception:
+ return jsonify({
+ 'status': 'success',
+ 'test_result': {
+ 'success': False,
+ 'error': 'Response is not valid JSON'
+ }
+ }), 200
+
+ # Apply JSONPath if specified
+ records = data
+ if endpoint.get('json_path'):
+ try:
+ jsonpath_expr = jsonpath_parse(endpoint['json_path'])
+ matches = [match.value for match in jsonpath_expr.find(data)]
+ records = matches
+ except Exception as e:
+ return jsonify({
+ 'status': 'success',
+ 'test_result': {
+ 'success': False,
+ 'error': f'JSONPath error: {str(e)}'
+ }
+ }), 200
+
+ # Ensure records is a list
+ if not isinstance(records, list):
+ records = [records]
+
+ # Return sample
+ return jsonify({
+ 'status': 'success',
+ 'test_result': {
+ 'success': True,
+ 'status_code': response.status_code,
+ 'sample_data': records[:5], # First 5 records
+ 'total_records': len(records),
+ 'error': None
+ }
+ }), 200
+
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+def _get_table_format_registry():
+ """Get or create TableFormatRegistry instance."""
+ from ...core.table_format_registry import TableFormatRegistry
+
+ if 'table_format_registry' not in current_app.extensions.get('scidk', {}):
+ if 'scidk' not in current_app.extensions:
+ current_app.extensions['scidk'] = {}
+
+ # Get settings DB path
+ settings_db = current_app.config.get('SCIDK_SETTINGS_DB', 'scidk_settings.db')
+
+ current_app.extensions['scidk']['table_format_registry'] = TableFormatRegistry(
+ db_path=settings_db
+ )
+
+ return current_app.extensions['scidk']['table_format_registry']
+
+
+@bp.route('/settings/table-formats', methods=['GET'])
+def list_table_formats():
+ """
+ Get all registered table format configurations.
+
+ Query params:
+ - include_preprogrammed: Include pre-programmed formats (default: true)
+
+ Returns:
+ {
+ "status": "success",
+ "formats": [...]
+ }
+ """
+ try:
+ include_preprogrammed = request.args.get('include_preprogrammed', 'true').lower() == 'true'
+ registry = _get_table_format_registry()
+ formats = registry.list_formats(include_preprogrammed=include_preprogrammed)
+ return jsonify({
+ 'status': 'success',
+ 'formats': formats
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/table-formats/', methods=['GET'])
+def get_table_format(format_id):
+ """Get a specific table format by ID."""
+ try:
+ registry = _get_table_format_registry()
+ format_config = registry.get_format(format_id)
+
+ if not format_config:
+ return jsonify({
+ 'status': 'error',
+ 'error': f'Format "{format_id}" not found'
+ }), 404
+
+ return jsonify({
+ 'status': 'success',
+ 'format': format_config
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/table-formats', methods=['POST'])
+def create_table_format():
+ """Create a new table format configuration."""
+ try:
+ data = request.get_json()
+ if not data:
+ return jsonify({
+ 'status': 'error',
+ 'error': 'Request body must be JSON'
+ }), 400
+
+ registry = _get_table_format_registry()
+ format_config = registry.create_format(data)
+
+ return jsonify({
+ 'status': 'success',
+ 'format': format_config
+ }), 201
+ except ValueError as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 400
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/table-formats/', methods=['PUT', 'PATCH'])
+def update_table_format(format_id):
+ """Update an existing table format."""
+ try:
+ data = request.get_json()
+ if not data:
+ return jsonify({
+ 'status': 'error',
+ 'error': 'Request body must be JSON'
+ }), 400
+
+ registry = _get_table_format_registry()
+ format_config = registry.update_format(format_id, data)
+
+ return jsonify({
+ 'status': 'success',
+ 'format': format_config
+ }), 200
+ except ValueError as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 400
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/table-formats/', methods=['DELETE'])
+def delete_table_format(format_id):
+ """Delete a table format."""
+ try:
+ registry = _get_table_format_registry()
+ deleted = registry.delete_format(format_id)
+
+ if not deleted:
+ return jsonify({
+ 'status': 'error',
+ 'error': f'Format "{format_id}" not found'
+ }), 404
+
+ return jsonify({
+ 'status': 'success'
+ }), 200
+ except ValueError as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 400
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/table-formats/detect', methods=['POST'])
+def detect_table_format():
+ """Auto-detect table format from uploaded file."""
+ try:
+ if 'file' not in request.files:
+ return jsonify({
+ 'status': 'error',
+ 'error': 'No file provided'
+ }), 400
+
+ file = request.files['file']
+ if not file or not file.filename:
+ return jsonify({
+ 'status': 'error',
+ 'error': 'Invalid file'
+ }), 400
+
+ file_content = file.read()
+ registry = _get_table_format_registry()
+ detected = registry.detect_format(file_content, filename=file.filename)
+
+ if 'error' in detected:
+ return jsonify({
+ 'status': 'error',
+ 'error': detected['error']
+ }), 400
+
+ return jsonify({
+ 'status': 'success',
+ 'detected': detected
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/table-formats//preview', methods=['POST'])
+def preview_table_data(format_id):
+ """Preview table data using a format configuration."""
+ try:
+ if 'file' not in request.files:
+ return jsonify({
+ 'status': 'error',
+ 'error': 'No file provided'
+ }), 400
+
+ file = request.files['file']
+ if not file or not file.filename:
+ return jsonify({
+ 'status': 'error',
+ 'error': 'Invalid file'
+ }), 400
+
+ file_content = file.read()
+ num_rows = int(request.args.get('num_rows', 5))
+
+ registry = _get_table_format_registry()
+ preview = registry.preview_data(file_content, format_id, num_rows=num_rows)
+
+ if 'error' in preview:
+ return jsonify({
+ 'status': 'error',
+ 'error': preview['error']
+ }), 400
+
+ return jsonify({
+ 'status': 'success',
+ 'preview': preview
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+def _get_fuzzy_matching_service():
+ """Get or create FuzzyMatchingService instance."""
+ from ...core.fuzzy_matching import FuzzyMatchingService
+
+ if 'fuzzy_matching_service' not in current_app.extensions.get('scidk', {}):
+ if 'scidk' not in current_app.extensions:
+ current_app.extensions['scidk'] = {}
+
+ # Get settings DB path
+ settings_db = current_app.config.get('SCIDK_SETTINGS_DB', 'scidk_settings.db')
+
+ current_app.extensions['scidk']['fuzzy_matching_service'] = FuzzyMatchingService(
+ db_path=settings_db
+ )
+
+ return current_app.extensions['scidk']['fuzzy_matching_service']
+
+
+@bp.route('/settings/fuzzy-matching', methods=['GET'])
+def get_fuzzy_matching_settings():
+ """
+ Get global fuzzy matching settings.
+
+ Returns:
+ {
+ "status": "success",
+ "settings": {
+ "algorithm": "levenshtein",
+ "threshold": 0.80,
+ "case_sensitive": false,
+ ...
+ }
+ }
+ """
+ try:
+ service = _get_fuzzy_matching_service()
+ settings = service.get_global_settings()
+
+ return jsonify({
+ 'status': 'success',
+ 'settings': settings.to_dict()
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/fuzzy-matching', methods=['POST', 'PUT'])
+def update_fuzzy_matching_settings():
+ """
+ Update global fuzzy matching settings.
+
+ Request body:
+ {
+ "algorithm": "levenshtein",
+ "threshold": 0.75,
+ "case_sensitive": false,
+ "normalize_whitespace": true,
+ "strip_punctuation": true,
+ "phonetic_enabled": false,
+ "phonetic_algorithm": "metaphone",
+ "min_string_length": 3,
+ "max_comparisons": 10000,
+ "show_confidence_scores": true
+ }
+
+ Returns:
+ {
+ "status": "success",
+ "settings": {...}
+ }
+ """
+ try:
+ data = request.get_json()
+ if not data:
+ return jsonify({
+ 'status': 'error',
+ 'error': 'Request body must be JSON'
+ }), 400
+
+ service = _get_fuzzy_matching_service()
+ settings = service.update_global_settings(data)
+
+ return jsonify({
+ 'status': 'success',
+ 'settings': settings.to_dict()
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
+
+
+@bp.route('/settings/fuzzy-matching/preview', methods=['POST'])
+def preview_fuzzy_matching():
+ """
+ Preview fuzzy matching results for external data.
+
+ Request body:
+ {
+ "external_records": [
+ {"name": "Jon Smith", "email": "jon@example.com"},
+ ...
+ ],
+ "existing_nodes": [
+ {"name": "John Smith", "email": "john@example.com"},
+ ...
+ ],
+ "match_key": "name",
+ "settings": { // Optional override
+ "algorithm": "levenshtein",
+ "threshold": 0.75
+ }
+ }
+
+ Returns:
+ {
+ "status": "success",
+ "matches": [
+ {
+ "external_record": {...},
+ "matched_node": {...} or null,
+ "confidence": 0.85,
+ "is_match": true
+ }
+ ]
+ }
+ """
+ try:
+ data = request.get_json()
+ if not data:
+ return jsonify({
+ 'status': 'error',
+ 'error': 'Request body must be JSON'
+ }), 400
+
+ external_records = data.get('external_records', [])
+ existing_nodes = data.get('existing_nodes', [])
+ match_key = data.get('match_key')
+
+ if not match_key:
+ return jsonify({
+ 'status': 'error',
+ 'error': 'match_key is required'
+ }), 400
+
+ service = _get_fuzzy_matching_service()
+
+ # Parse settings override if provided
+ settings = None
+ if 'settings' in data and data['settings']:
+ from ...core.fuzzy_matching import FuzzyMatchSettings
+ settings = FuzzyMatchSettings.from_dict(data['settings'])
+
+ matches = service.match_external_data(
+ external_records,
+ existing_nodes,
+ match_key,
+ settings
+ )
+
+ return jsonify({
+ 'status': 'success',
+ 'matches': matches,
+ 'total_external': len(external_records),
+ 'total_matched': sum(1 for m in matches if m['is_match'])
+ }), 200
+ except Exception as e:
+ return jsonify({
+ 'status': 'error',
+ 'error': str(e)
+ }), 500
diff --git a/scidk/web/routes/ui.py b/scidk/web/routes/ui.py
index 396e385..730d5d6 100644
--- a/scidk/web/routes/ui.py
+++ b/scidk/web/routes/ui.py
@@ -188,10 +188,16 @@ def labels():
return render_template('labels.html')
+@bp.get('/integrate')
+def integrate():
+ """Integration definitions page for relationship creation workflows."""
+ return render_template('integrations.html')
+
+
@bp.get('/links')
-def links():
- """Link definitions page for relationship creation workflows."""
- return render_template('links.html')
+def links_redirect():
+ """Backward compatibility redirect: /links → /integrate"""
+ return redirect(url_for('ui.integrate'))
@bp.get('/settings')
diff --git a/tests/test_api_endpoint_registry.py b/tests/test_api_endpoint_registry.py
new file mode 100644
index 0000000..faa4d5c
--- /dev/null
+++ b/tests/test_api_endpoint_registry.py
@@ -0,0 +1,314 @@
+"""
+Tests for API Endpoint Registry.
+
+Tests CRUD operations, encryption, and validation.
+"""
+import pytest
+import tempfile
+import os
+from scidk.core.api_endpoint_registry import APIEndpointRegistry, get_encryption_key
+from cryptography.fernet import Fernet
+
+
+@pytest.fixture
+def registry():
+ """Create a temporary registry for testing."""
+ # Use temporary database
+ with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as tmp:
+ db_path = tmp.name
+
+ # Generate test encryption key
+ encryption_key = Fernet.generate_key().decode()
+
+ reg = APIEndpointRegistry(db_path=db_path, encryption_key=encryption_key)
+
+ yield reg
+
+ # Cleanup
+ reg.db.close()
+ if os.path.exists(db_path):
+ os.unlink(db_path)
+
+
+def test_create_endpoint(registry):
+ """Test creating a new API endpoint."""
+ endpoint_data = {
+ 'name': 'Test API',
+ 'url': 'https://api.example.com/users',
+ 'auth_method': 'bearer',
+ 'auth_value': 'secret_token_123',
+ 'json_path': '$.data[*]',
+ 'target_label': 'User',
+ 'field_mappings': {
+ 'email': 'email',
+ 'full_name': 'name'
+ }
+ }
+
+ endpoint = registry.create_endpoint(endpoint_data)
+
+ assert endpoint['id'] is not None
+ assert endpoint['name'] == 'Test API'
+ assert endpoint['url'] == 'https://api.example.com/users'
+ assert endpoint['auth_method'] == 'bearer'
+ assert endpoint['json_path'] == '$.data[*]'
+ assert endpoint['target_label'] == 'User'
+ assert endpoint['field_mappings'] == {'email': 'email', 'full_name': 'name'}
+ assert 'auth_value' not in endpoint # Should not be included by default
+
+
+def test_create_endpoint_validation(registry):
+ """Test endpoint creation validation."""
+ # Missing name
+ with pytest.raises(ValueError, match="Endpoint name is required"):
+ registry.create_endpoint({'url': 'https://example.com'})
+
+ # Missing URL
+ with pytest.raises(ValueError, match="Endpoint URL is required"):
+ registry.create_endpoint({'name': 'Test'})
+
+
+def test_create_duplicate_name(registry):
+ """Test that duplicate names are rejected."""
+ data = {
+ 'name': 'Duplicate Test',
+ 'url': 'https://api.example.com/users'
+ }
+
+ # First creation should succeed
+ registry.create_endpoint(data)
+
+ # Second creation with same name should fail
+ with pytest.raises(ValueError, match="already exists"):
+ registry.create_endpoint(data)
+
+
+def test_get_endpoint(registry):
+ """Test retrieving an endpoint by ID."""
+ data = {
+ 'name': 'Get Test',
+ 'url': 'https://api.example.com/data'
+ }
+
+ created = registry.create_endpoint(data)
+ endpoint_id = created['id']
+
+ retrieved = registry.get_endpoint(endpoint_id)
+
+ assert retrieved is not None
+ assert retrieved['id'] == endpoint_id
+ assert retrieved['name'] == 'Get Test'
+
+
+def test_get_nonexistent_endpoint(registry):
+ """Test getting a nonexistent endpoint returns None."""
+ result = registry.get_endpoint('nonexistent-id')
+ assert result is None
+
+
+def test_get_endpoint_by_name(registry):
+ """Test retrieving an endpoint by name."""
+ data = {
+ 'name': 'Name Search Test',
+ 'url': 'https://api.example.com/search'
+ }
+
+ created = registry.create_endpoint(data)
+ retrieved = registry.get_endpoint_by_name('Name Search Test')
+
+ assert retrieved is not None
+ assert retrieved['id'] == created['id']
+ assert retrieved['name'] == 'Name Search Test'
+
+
+def test_list_endpoints(registry):
+ """Test listing all endpoints."""
+ # Create multiple endpoints
+ registry.create_endpoint({'name': 'API 1', 'url': 'https://example.com/1'})
+ registry.create_endpoint({'name': 'API 2', 'url': 'https://example.com/2'})
+ registry.create_endpoint({'name': 'API 3', 'url': 'https://example.com/3'})
+
+ endpoints = registry.list_endpoints()
+
+ assert len(endpoints) == 3
+ names = [e['name'] for e in endpoints]
+ assert 'API 1' in names
+ assert 'API 2' in names
+ assert 'API 3' in names
+
+
+def test_list_endpoints_empty(registry):
+ """Test listing endpoints when none exist."""
+ endpoints = registry.list_endpoints()
+ assert endpoints == []
+
+
+def test_update_endpoint(registry):
+ """Test updating an endpoint."""
+ data = {
+ 'name': 'Original Name',
+ 'url': 'https://example.com/original',
+ 'auth_method': 'none'
+ }
+
+ created = registry.create_endpoint(data)
+ endpoint_id = created['id']
+
+ # Update
+ updates = {
+ 'name': 'Updated Name',
+ 'url': 'https://example.com/updated',
+ 'auth_method': 'bearer',
+ 'auth_value': 'new_token',
+ 'target_label': 'UpdatedLabel'
+ }
+
+ updated = registry.update_endpoint(endpoint_id, updates)
+
+ assert updated['name'] == 'Updated Name'
+ assert updated['url'] == 'https://example.com/updated'
+ assert updated['auth_method'] == 'bearer'
+ assert updated['target_label'] == 'UpdatedLabel'
+
+
+def test_update_nonexistent_endpoint(registry):
+ """Test updating a nonexistent endpoint raises error."""
+ with pytest.raises(ValueError, match="not found"):
+ registry.update_endpoint('nonexistent-id', {'name': 'Test'})
+
+
+def test_update_endpoint_name_conflict(registry):
+ """Test that renaming to an existing name is rejected."""
+ registry.create_endpoint({'name': 'Endpoint 1', 'url': 'https://example.com/1'})
+ created2 = registry.create_endpoint({'name': 'Endpoint 2', 'url': 'https://example.com/2'})
+
+ # Try to rename Endpoint 2 to Endpoint 1
+ with pytest.raises(ValueError, match="already exists"):
+ registry.update_endpoint(created2['id'], {'name': 'Endpoint 1'})
+
+
+def test_delete_endpoint(registry):
+ """Test deleting an endpoint."""
+ data = {
+ 'name': 'Delete Test',
+ 'url': 'https://example.com/delete'
+ }
+
+ created = registry.create_endpoint(data)
+ endpoint_id = created['id']
+
+ # Verify it exists
+ assert registry.get_endpoint(endpoint_id) is not None
+
+ # Delete
+ result = registry.delete_endpoint(endpoint_id)
+ assert result is True
+
+ # Verify it's gone
+ assert registry.get_endpoint(endpoint_id) is None
+
+
+def test_delete_nonexistent_endpoint(registry):
+ """Test deleting a nonexistent endpoint returns False."""
+ result = registry.delete_endpoint('nonexistent-id')
+ assert result is False
+
+
+def test_auth_value_encryption(registry):
+ """Test that auth values are encrypted at rest."""
+ data = {
+ 'name': 'Encryption Test',
+ 'url': 'https://example.com/secure',
+ 'auth_method': 'bearer',
+ 'auth_value': 'super_secret_token'
+ }
+
+ endpoint = registry.create_endpoint(data)
+ endpoint_id = endpoint['id']
+
+ # Get decrypted auth value
+ decrypted = registry.get_decrypted_auth(endpoint_id)
+ assert decrypted == 'super_secret_token'
+
+ # Verify it's encrypted in the database
+ cursor = registry.db.execute(
+ "SELECT auth_value_encrypted FROM api_endpoints WHERE id = ?",
+ (endpoint_id,)
+ )
+ row = cursor.fetchone()
+ encrypted_value = row[0]
+
+ # Encrypted value should be different from original
+ assert encrypted_value != 'super_secret_token'
+ assert len(encrypted_value) > len('super_secret_token')
+
+
+def test_auth_value_optional(registry):
+ """Test that auth value is optional."""
+ data = {
+ 'name': 'No Auth Test',
+ 'url': 'https://example.com/public',
+ 'auth_method': 'none'
+ }
+
+ endpoint = registry.create_endpoint(data)
+ assert endpoint['auth_method'] == 'none'
+
+ # Getting auth for endpoint with no auth should return None
+ decrypted = registry.get_decrypted_auth(endpoint['id'])
+ assert decrypted is None
+
+
+def test_field_mappings_serialization(registry):
+ """Test that field mappings are correctly serialized/deserialized."""
+ data = {
+ 'name': 'Mappings Test',
+ 'url': 'https://example.com/api',
+ 'field_mappings': {
+ 'api_field_1': 'label_prop_1',
+ 'api_field_2': 'label_prop_2',
+ 'nested.field': 'flat_field'
+ }
+ }
+
+ endpoint = registry.create_endpoint(data)
+ retrieved = registry.get_endpoint(endpoint['id'])
+
+ assert retrieved['field_mappings'] == data['field_mappings']
+ assert isinstance(retrieved['field_mappings'], dict)
+
+
+def test_default_values(registry):
+ """Test that optional fields have sensible defaults."""
+ data = {
+ 'name': 'Minimal Test',
+ 'url': 'https://example.com/minimal'
+ }
+
+ endpoint = registry.create_endpoint(data)
+
+ assert endpoint['auth_method'] == 'none'
+ assert endpoint['json_path'] == ''
+ assert endpoint['target_label'] == ''
+ assert endpoint['field_mappings'] == {}
+
+
+def test_get_encryption_key_from_env(monkeypatch):
+ """Test getting encryption key from environment variable."""
+ test_key = Fernet.generate_key().decode()
+ monkeypatch.setenv('SCIDK_API_ENCRYPTION_KEY', test_key)
+
+ key = get_encryption_key()
+ assert key == test_key
+
+
+def test_get_encryption_key_generates_ephemeral(monkeypatch):
+ """Test that encryption key is generated when not in environment."""
+ monkeypatch.delenv('SCIDK_API_ENCRYPTION_KEY', raising=False)
+
+ key = get_encryption_key()
+ assert key is not None
+ assert len(key) > 0
+
+ # Verify it's a valid Fernet key
+ Fernet(key.encode()) # Should not raise
diff --git a/tests/test_eda_interpreter.py b/tests/test_eda_interpreter.py
new file mode 100644
index 0000000..bb88263
--- /dev/null
+++ b/tests/test_eda_interpreter.py
@@ -0,0 +1,297 @@
+"""
+Unit tests for EDA file interpreter.
+"""
+import json
+import zipfile
+import tempfile
+from pathlib import Path
+import pytest
+
+from scidk.interpreters.eda_interpreter import parse_eda_file, eda_to_labels
+
+
+def create_test_eda_file(data, filename='test.eda'):
+ """Helper to create a test EDA file."""
+ tmp = tempfile.NamedTemporaryFile(suffix='.eda', delete=False)
+ with zipfile.ZipFile(tmp.name, 'w') as zf:
+ zf.writestr('model', json.dumps(data))
+ tmp.close()
+ return tmp.name
+
+
+def test_parse_eda_file_single_node():
+ """Test parsing .eda file with single node."""
+ eda_data = {
+ 'childShapes': [
+ {
+ 'resourceId': 'n0',
+ 'stencil': {'id': 'Treatment'},
+ 'properties': {'name': 'Drug A', 'dose': '10mg'},
+ 'propertyTypes': {'name': 'String', 'dose': 'String'},
+ 'outgoing': [],
+ 'incoming': []
+ }
+ ]
+ }
+
+ tmp_path = create_test_eda_file(eda_data)
+ try:
+ nodes, edges = parse_eda_file(tmp_path)
+ assert len(nodes) == 1
+ assert len(edges) == 0
+ assert nodes[0]['stencil']['id'] == 'Treatment'
+ assert nodes[0]['properties']['name'] == 'Drug A'
+ finally:
+ Path(tmp_path).unlink()
+
+
+def test_parse_eda_file_with_edges():
+ """Test parsing .eda file with nodes and edges."""
+ eda_data = {
+ 'childShapes': [
+ {
+ 'resourceId': 'n0',
+ 'stencil': {'id': 'Treatment'},
+ 'properties': {'name': 'Drug A'},
+ 'propertyTypes': {'name': 'String'},
+ 'outgoing': [],
+ 'incoming': []
+ },
+ {
+ 'resourceId': 'n1',
+ 'stencil': {'id': 'Subject'},
+ 'properties': {'id': 'Mouse001'},
+ 'propertyTypes': {'id': 'String'},
+ 'outgoing': [],
+ 'incoming': []
+ },
+ {
+ 'resourceId': 'e0',
+ 'stencil': {'id': 'APPLIED_TO'},
+ 'target': {'resourceId': 'n1'},
+ 'properties': {'name': 'applies'},
+ 'propertyTypes': {},
+ 'outgoing': [{'resourceId': 'n1'}],
+ 'incoming': [{'resourceId': 'n0'}]
+ }
+ ]
+ }
+
+ tmp_path = create_test_eda_file(eda_data)
+ try:
+ nodes, edges = parse_eda_file(tmp_path)
+ assert len(nodes) == 2
+ assert len(edges) == 1
+ assert edges[0]['stencil']['id'] == 'APPLIED_TO'
+ finally:
+ Path(tmp_path).unlink()
+
+
+def test_parse_eda_file_not_found():
+ """Test parsing nonexistent file raises FileNotFoundError."""
+ with pytest.raises(FileNotFoundError):
+ parse_eda_file('/nonexistent/file.eda')
+
+
+def test_parse_eda_file_wrong_extension():
+ """Test parsing file with wrong extension raises ValueError."""
+ tmp = tempfile.NamedTemporaryFile(suffix='.txt', delete=False)
+ tmp.close()
+ try:
+ with pytest.raises(ValueError, match='Not an EDA file'):
+ parse_eda_file(tmp.name)
+ finally:
+ Path(tmp.name).unlink()
+
+
+def test_eda_to_labels_basic():
+ """Test converting EDA nodes to labels."""
+ eda_nodes = [
+ {
+ 'resourceId': 'n0',
+ 'stencil': {'id': 'Treatment'},
+ 'properties': {'name': 'Drug A', 'dose': '10mg'},
+ 'propertyTypes': {'name': 'String', 'dose': 'String'},
+ 'outgoing': [],
+ 'incoming': []
+ }
+ ]
+
+ labels = eda_to_labels(eda_nodes)
+
+ assert len(labels) == 1
+ assert labels[0]['name'] == 'Treatment'
+ assert len(labels[0]['properties']) == 2
+
+ prop_names = [p['name'] for p in labels[0]['properties']]
+ assert 'name' in prop_names
+ assert 'dose' in prop_names
+
+ # Check type mapping
+ for prop in labels[0]['properties']:
+ assert prop['type'] == 'string'
+ assert prop['required'] == False
+
+
+def test_eda_to_labels_type_mapping():
+ """Test EDA to scidk type mapping."""
+ eda_nodes = [
+ {
+ 'resourceId': 'n0',
+ 'stencil': {'id': 'Measurement'},
+ 'properties': {
+ 'text': 'value',
+ 'count': 42,
+ 'weight': 3.14,
+ 'active': True,
+ 'date': '2024-01-01'
+ },
+ 'propertyTypes': {
+ 'text': 'String',
+ 'count': 'Integer',
+ 'weight': 'Float',
+ 'active': 'Boolean',
+ 'date': 'Date'
+ },
+ 'outgoing': [],
+ 'incoming': []
+ }
+ ]
+
+ labels = eda_to_labels(eda_nodes)
+ props = {p['name']: p['type'] for p in labels[0]['properties']}
+
+ assert props['text'] == 'string'
+ assert props['count'] == 'number'
+ assert props['weight'] == 'number'
+ assert props['active'] == 'boolean'
+ assert props['date'] == 'date'
+
+
+def test_eda_to_labels_with_relationships():
+ """Test converting EDA nodes with relationships."""
+ eda_nodes = [
+ {
+ 'resourceId': 'n0',
+ 'stencil': {'id': 'Treatment'},
+ 'properties': {'name': 'Drug A'},
+ 'propertyTypes': {'name': 'String'},
+ 'outgoing': [{'target': 'n1'}],
+ 'incoming': []
+ },
+ {
+ 'resourceId': 'n1',
+ 'stencil': {'id': 'Subject'},
+ 'properties': {'id': 'Mouse001'},
+ 'propertyTypes': {'id': 'String'},
+ 'outgoing': [],
+ 'incoming': [{'target': 'n0'}]
+ }
+ ]
+
+ labels = eda_to_labels(eda_nodes)
+
+ assert len(labels) == 2
+
+ treatment = next(l for l in labels if l['name'] == 'Treatment')
+ assert len(treatment['relationships']) == 1
+ assert treatment['relationships'][0]['type'] == 'APPLIED_TO'
+ assert treatment['relationships'][0]['target_label'] == 'Subject'
+
+
+def test_eda_to_labels_dedupe_properties():
+ """Test that duplicate properties are merged."""
+ eda_nodes = [
+ {
+ 'resourceId': 'n0',
+ 'stencil': {'id': 'Subject'},
+ 'properties': {'id': 'M001', 'weight': 25},
+ 'propertyTypes': {'id': 'String', 'weight': 'Integer'},
+ 'outgoing': [],
+ 'incoming': []
+ },
+ {
+ 'resourceId': 'n1',
+ 'stencil': {'id': 'Subject'},
+ 'properties': {'id': 'M002', 'age': 8},
+ 'propertyTypes': {'id': 'String', 'age': 'Integer'},
+ 'outgoing': [],
+ 'incoming': []
+ }
+ ]
+
+ labels = eda_to_labels(eda_nodes)
+
+ # Should have only one Subject label
+ assert len(labels) == 1
+ assert labels[0]['name'] == 'Subject'
+
+ # Should have properties from both nodes
+ prop_names = [p['name'] for p in labels[0]['properties']]
+ assert 'id' in prop_names
+ assert 'weight' in prop_names
+ assert 'age' in prop_names
+
+
+def test_eda_to_labels_with_explicit_edges():
+ """Test converting with explicit edge objects."""
+ eda_nodes = [
+ {
+ 'resourceId': 'n0',
+ 'stencil': {'id': 'Experiment'},
+ 'properties': {'name': 'Exp1'},
+ 'propertyTypes': {'name': 'String'},
+ 'outgoing': [],
+ 'incoming': []
+ },
+ {
+ 'resourceId': 'n1',
+ 'stencil': {'id': 'Subject'},
+ 'properties': {'id': 'M001'},
+ 'propertyTypes': {'id': 'String'},
+ 'outgoing': [],
+ 'incoming': []
+ }
+ ]
+
+ eda_edges = [
+ {
+ 'resourceId': 'e0',
+ 'stencil': {'id': 'INCLUDES'},
+ 'incoming': [{'resourceId': 'n0'}],
+ 'outgoing': [{'resourceId': 'n1'}],
+ 'properties': {},
+ 'propertyTypes': {}
+ }
+ ]
+
+ labels = eda_to_labels(eda_nodes, eda_edges)
+
+ experiment = next(l for l in labels if l['name'] == 'Experiment')
+ assert len(experiment['relationships']) == 1
+ assert experiment['relationships'][0]['type'] == 'INCLUDES'
+ assert experiment['relationships'][0]['target_label'] == 'Subject'
+
+
+def test_eda_to_labels_empty_input():
+ """Test with empty input."""
+ labels = eda_to_labels([])
+ assert labels == []
+
+
+def test_eda_to_labels_missing_stencil():
+ """Test handling nodes with missing stencil."""
+ eda_nodes = [
+ {
+ 'resourceId': 'n0',
+ 'properties': {'name': 'value'},
+ 'propertyTypes': {'name': 'String'},
+ 'outgoing': [],
+ 'incoming': []
+ }
+ ]
+
+ labels = eda_to_labels(eda_nodes)
+ # Should skip nodes without stencil or create with default
+ # Current implementation skips them
+ assert len(labels) == 0 or labels[0]['name'] == 'Unknown'
diff --git a/tests/test_fuzzy_matching.py b/tests/test_fuzzy_matching.py
new file mode 100644
index 0000000..5aee5eb
--- /dev/null
+++ b/tests/test_fuzzy_matching.py
@@ -0,0 +1,341 @@
+"""
+Tests for Fuzzy Matching Service.
+
+Tests settings persistence, client-side matching (Phase 1),
+and server-side Cypher generation (Phase 2).
+"""
+import pytest
+import tempfile
+import os
+from scidk.core.fuzzy_matching import FuzzyMatchingService, FuzzyMatchSettings
+
+# Check if rapidfuzz is available for client-side matching tests
+try:
+ import rapidfuzz
+ RAPIDFUZZ_AVAILABLE = True
+except ImportError:
+ RAPIDFUZZ_AVAILABLE = False
+
+requires_rapidfuzz = pytest.mark.skipif(
+ not RAPIDFUZZ_AVAILABLE,
+ reason="rapidfuzz not installed (optional dependency for Phase 1 client-side matching)"
+)
+
+
+@pytest.fixture
+def service():
+ """Create a temporary fuzzy matching service for testing."""
+ with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as tmp:
+ db_path = tmp.name
+
+ svc = FuzzyMatchingService(db_path=db_path)
+
+ yield svc
+
+ # Cleanup
+ svc.db.close()
+ if os.path.exists(db_path):
+ os.unlink(db_path)
+
+
+def test_global_default_seeded(service):
+ """Test that global default settings are seeded on initialization."""
+ settings = service.get_global_settings()
+
+ assert settings.algorithm == 'levenshtein'
+ assert settings.threshold == 0.80
+ assert settings.case_sensitive is False
+ assert settings.normalize_whitespace is True
+ assert settings.strip_punctuation is True
+
+
+def test_update_global_settings(service):
+ """Test updating global fuzzy matching settings."""
+ updates = {
+ 'algorithm': 'jaro_winkler',
+ 'threshold': 0.75,
+ 'case_sensitive': True
+ }
+
+ updated = service.update_global_settings(updates)
+
+ assert updated.algorithm == 'jaro_winkler'
+ assert updated.threshold == 0.75
+ assert updated.case_sensitive is True
+ # Other fields should remain unchanged
+ assert updated.normalize_whitespace is True
+
+
+def test_settings_to_dict(service):
+ """Test serializing settings to dictionary."""
+ settings = service.get_global_settings()
+ data = settings.to_dict()
+
+ assert isinstance(data, dict)
+ assert 'algorithm' in data
+ assert 'threshold' in data
+ assert data['algorithm'] == 'levenshtein'
+
+
+def test_settings_from_dict():
+ """Test deserializing settings from dictionary."""
+ data = {
+ 'algorithm': 'jaro_winkler',
+ 'threshold': 0.90,
+ 'case_sensitive': True
+ }
+
+ settings = FuzzyMatchSettings.from_dict(data)
+
+ assert settings.algorithm == 'jaro_winkler'
+ assert settings.threshold == 0.90
+ assert settings.case_sensitive is True
+
+
+# ==========================================
+# Phase 1: Client-Side Matching Tests
+# ==========================================
+
+@requires_rapidfuzz
+def test_match_external_data_exact(service):
+ """Test exact matching (no fuzzy logic)."""
+ external_records = [
+ {'name': 'John Smith', 'email': 'john@example.com'},
+ {'name': 'Jane Doe', 'email': 'jane@example.com'},
+ {'name': 'Unknown Person', 'email': 'unknown@example.com'}
+ ]
+
+ existing_nodes = [
+ {'name': 'John Smith', 'id': 1},
+ {'name': 'Jane Doe', 'id': 2}
+ ]
+
+ settings = FuzzyMatchSettings(algorithm='exact')
+ matches = service.match_external_data(
+ external_records,
+ existing_nodes,
+ 'name',
+ settings
+ )
+
+ assert len(matches) == 3
+ assert matches[0]['is_match'] is True
+ assert matches[0]['matched_node']['id'] == 1
+ assert matches[1]['is_match'] is True
+ assert matches[1]['matched_node']['id'] == 2
+ assert matches[2]['is_match'] is False
+
+
+@requires_rapidfuzz
+def test_match_external_data_levenshtein(service):
+ """Test Levenshtein fuzzy matching."""
+ external_records = [
+ {'name': 'Jon Smith'}, # Typo in "John"
+ {'name': 'Jane Doe'}, # Exact match
+ {'name': 'Completely Different'}
+ ]
+
+ existing_nodes = [
+ {'name': 'John Smith', 'id': 1},
+ {'name': 'Jane Doe', 'id': 2}
+ ]
+
+ settings = FuzzyMatchSettings(algorithm='levenshtein', threshold=0.80)
+ matches = service.match_external_data(
+ external_records,
+ existing_nodes,
+ 'name',
+ settings
+ )
+
+ # "Jon Smith" should match "John Smith" (high similarity)
+ assert matches[0]['is_match'] is True
+ assert matches[0]['matched_node']['id'] == 1
+ assert matches[0]['confidence'] > 0.80
+
+ # "Jane Doe" exact match
+ assert matches[1]['is_match'] is True
+ assert matches[1]['confidence'] > 0.95
+
+ # "Completely Different" should not match
+ assert matches[2]['is_match'] is False
+
+
+def test_normalize_string_case_insensitive(service):
+ """Test string normalization with case insensitivity."""
+ settings = FuzzyMatchSettings(case_sensitive=False)
+ result = service._normalize_string('John SMITH', settings)
+ assert result == 'john smith'
+
+
+def test_normalize_string_strip_punctuation(service):
+ """Test string normalization with punctuation stripping."""
+ settings = FuzzyMatchSettings(strip_punctuation=True)
+ result = service._normalize_string("O'Brien, John", settings)
+ # Punctuation should be removed
+ assert ',' not in result
+ assert "'" not in result
+
+
+def test_normalize_whitespace(service):
+ """Test whitespace normalization."""
+ settings = FuzzyMatchSettings(normalize_whitespace=True)
+ result = service._normalize_string('John Smith ', settings)
+ assert result == 'john smith' # Normalized to single spaces, trimmed
+
+
+@requires_rapidfuzz
+def test_match_with_missing_key(service):
+ """Test matching when external record is missing the match key."""
+ external_records = [
+ {'email': 'john@example.com'}, # Missing 'name'
+ ]
+
+ existing_nodes = [
+ {'name': 'John Smith', 'id': 1}
+ ]
+
+ matches = service.match_external_data(
+ external_records,
+ existing_nodes,
+ 'name',
+ FuzzyMatchSettings()
+ )
+
+ assert len(matches) == 1
+ assert matches[0]['is_match'] is False
+ assert 'reason' in matches[0]
+
+
+@requires_rapidfuzz
+def test_match_with_short_string(service):
+ """Test matching with strings below min_string_length."""
+ external_records = [
+ {'name': 'Jo'}, # Too short (< 3 chars)
+ ]
+
+ existing_nodes = [
+ {'name': 'John', 'id': 1}
+ ]
+
+ settings = FuzzyMatchSettings(min_string_length=3)
+ matches = service.match_external_data(
+ external_records,
+ existing_nodes,
+ 'name',
+ settings
+ )
+
+ assert matches[0]['is_match'] is False
+ assert 'too short' in matches[0].get('reason', '').lower()
+
+
+# ==========================================
+# Phase 2: Server-Side Cypher Generation Tests
+# ==========================================
+
+def test_generate_cypher_exact_match(service):
+ """Test Cypher generation for exact matching."""
+ cypher = service.generate_cypher_fuzzy_match(
+ source_label='Person',
+ target_label='Company',
+ source_property='name',
+ target_property='contact_name',
+ relationship_type='WORKS_AT',
+ settings=FuzzyMatchSettings(algorithm='exact')
+ )
+
+ assert 'MATCH (source:Person)' in cypher
+ assert 'MATCH' in cypher and 'target:Company' in cypher
+ assert 'source.name = target.contact_name' in cypher
+ assert 'CREATE (source)-[:WORKS_AT' in cypher
+
+
+def test_generate_cypher_levenshtein(service):
+ """Test Cypher generation for Levenshtein matching."""
+ settings = FuzzyMatchSettings(algorithm='levenshtein', threshold=0.80)
+ cypher = service.generate_cypher_fuzzy_match(
+ source_label='Person',
+ target_label='Company',
+ source_property='name',
+ target_property='contact_name',
+ relationship_type='WORKS_AT',
+ settings=settings
+ )
+
+ assert 'apoc.text.levenshteinSimilarity' in cypher
+ assert '>= 0.8' in cypher
+ assert 'confidence' in cypher
+
+
+def test_generate_cypher_jaro_winkler(service):
+ """Test Cypher generation for Jaro-Winkler matching."""
+ settings = FuzzyMatchSettings(algorithm='jaro_winkler', threshold=0.85)
+ cypher = service.generate_cypher_fuzzy_match(
+ source_label='Person',
+ target_label='Organization',
+ source_property='full_name',
+ target_property='owner_name',
+ relationship_type='OWNS',
+ settings=settings
+ )
+
+ assert 'apoc.text.jaroWinklerDistance' in cypher
+ assert '>= 0.85' in cypher
+
+
+def test_generate_cypher_phonetic_soundex(service):
+ """Test Cypher generation for phonetic (soundex) matching."""
+ settings = FuzzyMatchSettings(
+ algorithm='phonetic',
+ phonetic_enabled=True,
+ phonetic_algorithm='soundex'
+ )
+ cypher = service.generate_cypher_fuzzy_match(
+ source_label='Person',
+ target_label='Person',
+ source_property='last_name',
+ target_property='surname',
+ relationship_type='SIMILAR_TO',
+ settings=settings
+ )
+
+ assert 'apoc.text.phonetic' in cypher
+ assert 'soundex' in cypher.lower() or 'phonetic' in cypher.lower()
+
+
+def test_generate_cypher_phonetic_metaphone(service):
+ """Test Cypher generation for phonetic (metaphone) matching."""
+ settings = FuzzyMatchSettings(
+ algorithm='phonetic',
+ phonetic_enabled=True,
+ phonetic_algorithm='metaphone'
+ )
+ cypher = service.generate_cypher_fuzzy_match(
+ source_label='Author',
+ target_label='Contributor',
+ source_property='name',
+ target_property='name',
+ relationship_type='SAME_PERSON',
+ settings=settings
+ )
+
+ assert 'apoc.text.doubleMetaphone' in cypher
+
+
+def test_cypher_includes_labels_and_properties(service):
+ """Test that generated Cypher includes correct labels and properties."""
+ cypher = service.generate_cypher_fuzzy_match(
+ source_label='Customer',
+ target_label='Order',
+ source_property='customer_email',
+ target_property='buyer_email',
+ relationship_type='PLACED',
+ settings=FuzzyMatchSettings()
+ )
+
+ assert 'Customer' in cypher
+ assert 'Order' in cypher
+ assert 'customer_email' in cypher
+ assert 'buyer_email' in cypher
+ assert 'PLACED' in cypher
diff --git a/tests/test_labels_api.py b/tests/test_labels_api.py
index 04cda49..9f2b5da 100644
--- a/tests/test_labels_api.py
+++ b/tests/test_labels_api.py
@@ -438,3 +438,70 @@ def test_batch_delete_labels_partial_success(client):
# Verify the existing label was deleted
get_response = client.get('/api/labels/DeleteExists')
assert get_response.status_code == 404
+
+
+def test_get_label_instances_no_neo4j(client):
+ """Test getting instances when Neo4j is not configured."""
+ # Create a label first
+ payload = {
+ 'name': 'Person',
+ 'properties': [
+ {'name': 'name', 'type': 'string', 'required': False},
+ {'name': 'age', 'type': 'number', 'required': False}
+ ],
+ 'relationships': []
+ }
+ client.post('/api/labels', json=payload)
+
+ # Try to get instances (will fail without Neo4j)
+ response = client.get('/api/labels/Person/instances')
+ data = response.get_json()
+ # Without Neo4j configured, this should return an error
+ assert data['status'] == 'error'
+
+
+def test_get_instance_count_no_neo4j(client):
+ """Test getting instance count when Neo4j is not configured."""
+ # Create a label first
+ payload = {
+ 'name': 'Person',
+ 'properties': [
+ {'name': 'name', 'type': 'string', 'required': False}
+ ],
+ 'relationships': []
+ }
+ client.post('/api/labels', json=payload)
+
+ # Try to get count (will fail without Neo4j)
+ response = client.get('/api/labels/Person/instance-count')
+ data = response.get_json()
+ assert data['status'] == 'error'
+
+
+def test_update_instance_label_not_found(client):
+ """Test updating instance for non-existent label."""
+ response = client.patch('/api/labels/NonExistent/instances/some-id', json={
+ 'property': 'name',
+ 'value': 'John'
+ })
+ assert response.status_code == 404
+
+
+def test_update_instance_missing_property(client):
+ """Test updating instance without property name."""
+ # Create a label first
+ payload = {
+ 'name': 'Person',
+ 'properties': [{'name': 'name', 'type': 'string', 'required': False}],
+ 'relationships': []
+ }
+ client.post('/api/labels', json=payload)
+
+ # Try to update without property
+ response = client.patch('/api/labels/Person/instances/some-id', json={
+ 'value': 'John'
+ })
+ assert response.status_code == 400
+ data = response.get_json()
+ assert data['status'] == 'error'
+ assert 'required' in data['error'].lower()
diff --git a/tests/test_links_api.py b/tests/test_links_api.py
index b108d0b..59ef90b 100644
--- a/tests/test_links_api.py
+++ b/tests/test_links_api.py
@@ -10,6 +10,8 @@
- POST /api/links//execute - execute link job
- GET /api/links/jobs/ - get job status
- GET /api/links/jobs - list jobs
+- GET /api/links/available-labels - get available labels for dropdowns
+- POST /api/links/migrate - migrate existing links to Label→Label model
"""
import json
import pytest
@@ -26,21 +28,26 @@ def test_list_links_empty(client):
def test_create_link_success(client):
- """Test creating a link definition with all required fields."""
+ """Test creating a link definition with all required fields (Label→Label model)."""
+ # Create labels first
+ client.post('/api/labels', json={
+ 'name': 'Author',
+ 'properties': [{'name': 'email', 'type': 'string'}],
+ 'relationships': []
+ })
+ client.post('/api/labels', json={
+ 'name': 'File',
+ 'properties': [{'name': 'path', 'type': 'string'}],
+ 'relationships': []
+ })
+
payload = {
'name': 'Authors to Files',
- 'source_type': 'csv',
- 'source_config': {
- 'csv_data': 'name,email,file_path\nAlice,alice@ex.com,file1.txt'
- },
- 'target_type': 'label',
- 'target_config': {
- 'label': 'File'
- },
- 'match_strategy': 'property',
+ 'source_label': 'Author',
+ 'target_label': 'File',
+ 'match_strategy': 'table_import',
'match_config': {
- 'source_field': 'file_path',
- 'target_field': 'path'
+ 'table_data': 'name,email,file_path\nAlice,alice@ex.com,file1.txt'
},
'relationship_type': 'AUTHORED',
'relationship_props': {
@@ -54,9 +61,9 @@ def test_create_link_success(client):
assert data['status'] == 'success'
assert 'link' in data
assert data['link']['name'] == 'Authors to Files'
- assert data['link']['source_type'] == 'csv'
- assert data['link']['target_type'] == 'label'
- assert data['link']['match_strategy'] == 'property'
+ assert data['link']['source_label'] == 'Author'
+ assert data['link']['target_label'] == 'File'
+ assert data['link']['match_strategy'] == 'table_import'
assert data['link']['relationship_type'] == 'AUTHORED'
assert 'id' in data['link']
@@ -78,11 +85,10 @@ def test_create_link_missing_name(client):
def test_create_link_invalid_source_type(client):
- """Test creating link with invalid source_type fails."""
+ """Test creating link without source_label fails (Label→Label model)."""
payload = {
'name': 'Bad Link',
- 'source_type': 'invalid',
- 'target_type': 'label',
+ 'target_label': 'File',
'match_strategy': 'property',
'relationship_type': 'RELATED'
}
@@ -91,15 +97,14 @@ def test_create_link_invalid_source_type(client):
assert response.status_code == 400
data = response.get_json()
assert data['status'] == 'error'
- assert 'source_type' in data['error'].lower()
+ assert 'source_label' in data['error'].lower()
def test_create_link_invalid_target_type(client):
- """Test creating link with invalid target_type fails."""
+ """Test creating link without target_label fails (Label→Label model)."""
payload = {
'name': 'Bad Link',
- 'source_type': 'graph',
- 'target_type': 'invalid',
+ 'source_label': 'Person',
'match_strategy': 'property',
'relationship_type': 'RELATED'
}
@@ -108,15 +113,22 @@ def test_create_link_invalid_target_type(client):
assert response.status_code == 400
data = response.get_json()
assert data['status'] == 'error'
- assert 'target_type' in data['error'].lower()
+ assert 'target_label' in data['error'].lower()
def test_create_link_invalid_match_strategy(client):
"""Test creating link with invalid match_strategy fails."""
+ # Create labels first
+ client.post('/api/labels', json={
+ 'name': 'TestLabel',
+ 'properties': [],
+ 'relationships': []
+ })
+
payload = {
'name': 'Bad Link',
- 'source_type': 'graph',
- 'target_type': 'label',
+ 'source_label': 'TestLabel',
+ 'target_label': 'TestLabel',
'match_strategy': 'invalid',
'relationship_type': 'RELATED'
}
@@ -130,10 +142,17 @@ def test_create_link_invalid_match_strategy(client):
def test_create_link_missing_relationship_type(client):
"""Test creating link without relationship_type fails."""
+ # Create labels first
+ client.post('/api/labels', json={
+ 'name': 'Person',
+ 'properties': [],
+ 'relationships': []
+ })
+
payload = {
'name': 'Bad Link',
- 'source_type': 'graph',
- 'target_type': 'label',
+ 'source_label': 'Person',
+ 'target_label': 'Person',
'match_strategy': 'property'
}
@@ -146,13 +165,23 @@ def test_create_link_missing_relationship_type(client):
def test_get_link_success(client):
"""Test retrieving an existing link definition."""
- # First create a link
+ # Create labels first
+ client.post('/api/labels', json={
+ 'name': 'Person',
+ 'properties': [{'name': 'email', 'type': 'string'}],
+ 'relationships': []
+ })
+ client.post('/api/labels', json={
+ 'name': 'File',
+ 'properties': [{'name': 'path', 'type': 'string'}],
+ 'relationships': []
+ })
+
+ # Create a link
payload = {
'name': 'Test Link',
- 'source_type': 'graph',
- 'source_config': {'label': 'Person'},
- 'target_type': 'label',
- 'target_config': {'label': 'File'},
+ 'source_label': 'Person',
+ 'target_label': 'File',
'match_strategy': 'property',
'match_config': {'source_field': 'email', 'target_field': 'author'},
'relationship_type': 'AUTHORED',
@@ -180,13 +209,23 @@ def test_get_link_not_found(client):
def test_update_link_success(client):
"""Test updating an existing link definition."""
+ # Create labels first
+ client.post('/api/labels', json={
+ 'name': 'Person',
+ 'properties': [{'name': 'email', 'type': 'string'}],
+ 'relationships': []
+ })
+ client.post('/api/labels', json={
+ 'name': 'File',
+ 'properties': [{'name': 'path', 'type': 'string'}],
+ 'relationships': []
+ })
+
# Create a link
payload = {
'name': 'Original Name',
- 'source_type': 'graph',
- 'source_config': {'label': 'Person'},
- 'target_type': 'label',
- 'target_config': {'label': 'File'},
+ 'source_label': 'Person',
+ 'target_label': 'File',
'match_strategy': 'property',
'match_config': {'source_field': 'email', 'target_field': 'author'},
'relationship_type': 'AUTHORED',
@@ -210,13 +249,23 @@ def test_update_link_success(client):
def test_delete_link_success(client):
"""Test deleting a link definition."""
+ # Create labels first
+ client.post('/api/labels', json={
+ 'name': 'Person',
+ 'properties': [{'name': 'email', 'type': 'string'}],
+ 'relationships': []
+ })
+ client.post('/api/labels', json={
+ 'name': 'File',
+ 'properties': [{'name': 'path', 'type': 'string'}],
+ 'relationships': []
+ })
+
# Create a link
payload = {
'name': 'To Delete',
- 'source_type': 'graph',
- 'source_config': {'label': 'Person'},
- 'target_type': 'label',
- 'target_config': {'label': 'File'},
+ 'source_label': 'Person',
+ 'target_label': 'File',
'match_strategy': 'property',
'match_config': {'source_field': 'email', 'target_field': 'author'},
'relationship_type': 'AUTHORED',
@@ -246,6 +295,18 @@ def test_delete_link_not_found(client):
def test_list_links_after_create(client):
"""Test that created links appear in list."""
+ # Create labels first
+ client.post('/api/labels', json={
+ 'name': 'Person',
+ 'properties': [{'name': 'email', 'type': 'string'}],
+ 'relationships': []
+ })
+ client.post('/api/labels', json={
+ 'name': 'File',
+ 'properties': [{'name': 'path', 'type': 'string'}],
+ 'relationships': []
+ })
+
# Get initial count
initial_response = client.get('/api/links')
initial_count = len(initial_response.get_json()['links'])
@@ -255,10 +316,8 @@ def test_list_links_after_create(client):
for i in range(3):
payload = {
'name': f'Link {i}',
- 'source_type': 'graph',
- 'source_config': {'label': 'Person'},
- 'target_type': 'label',
- 'target_config': {'label': 'File'},
+ 'source_label': 'Person',
+ 'target_label': 'File',
'match_strategy': 'property',
'match_config': {'source_field': 'email', 'target_field': 'author'},
'relationship_type': f'REL_{i}',
@@ -312,3 +371,203 @@ def test_get_job_status_not_found(client):
assert response.status_code == 404
data = response.get_json()
assert data['status'] == 'error'
+
+
+# === Label→Label Refactor Tests ===
+
+
+def test_create_link_with_labels(client):
+ """Test creating a link with source_label and target_label (new model)."""
+ # First, create labels
+ client.post('/api/labels', json={
+ 'name': 'Person',
+ 'properties': [{'name': 'email', 'type': 'string'}],
+ 'relationships': []
+ })
+ client.post('/api/labels', json={
+ 'name': 'File',
+ 'properties': [{'name': 'path', 'type': 'string'}],
+ 'relationships': []
+ })
+
+ payload = {
+ 'name': 'Person to File',
+ 'source_label': 'Person',
+ 'target_label': 'File',
+ 'match_strategy': 'property',
+ 'match_config': {
+ 'source_field': 'email',
+ 'target_field': 'author_email'
+ },
+ 'relationship_type': 'AUTHORED'
+ }
+
+ response = client.post('/api/links', json=payload)
+ assert response.status_code == 200
+ data = response.get_json()
+ assert data['status'] == 'success'
+ assert data['link']['source_label'] == 'Person'
+ assert data['link']['target_label'] == 'File'
+ assert data['link']['match_strategy'] == 'property'
+
+
+def test_create_link_missing_source_label(client):
+ """Test that source_label is required."""
+ payload = {
+ 'name': 'Bad Link',
+ 'target_label': 'File',
+ 'match_strategy': 'property',
+ 'relationship_type': 'RELATED'
+ }
+
+ response = client.post('/api/links', json=payload)
+ assert response.status_code == 400
+ data = response.get_json()
+ assert data['status'] == 'error'
+ assert 'source_label' in data['error'].lower()
+
+
+def test_create_link_missing_target_label(client):
+ """Test that target_label is required."""
+ payload = {
+ 'name': 'Bad Link',
+ 'source_label': 'Person',
+ 'match_strategy': 'property',
+ 'relationship_type': 'RELATED'
+ }
+
+ response = client.post('/api/links', json=payload)
+ assert response.status_code == 400
+ data = response.get_json()
+ assert data['status'] == 'error'
+ assert 'target_label' in data['error'].lower()
+
+
+def test_create_link_nonexistent_label(client):
+ """Test that labels must exist in registry."""
+ payload = {
+ 'name': 'Bad Link',
+ 'source_label': 'NonexistentLabel',
+ 'target_label': 'AlsoDoesNotExist',
+ 'match_strategy': 'property',
+ 'relationship_type': 'RELATED'
+ }
+
+ response = client.post('/api/links', json=payload)
+ assert response.status_code == 400
+ data = response.get_json()
+ assert data['status'] == 'error'
+
+
+def test_create_link_fuzzy_match_strategy(client):
+ """Test creating link with fuzzy match strategy."""
+ # Create labels first
+ client.post('/api/labels', json={
+ 'name': 'Author',
+ 'properties': [{'name': 'name', 'type': 'string'}],
+ 'relationships': []
+ })
+ client.post('/api/labels', json={
+ 'name': 'Document',
+ 'properties': [{'name': 'author_name', 'type': 'string'}],
+ 'relationships': []
+ })
+
+ payload = {
+ 'name': 'Author to Document (Fuzzy)',
+ 'source_label': 'Author',
+ 'target_label': 'Document',
+ 'match_strategy': 'fuzzy',
+ 'match_config': {
+ 'source_field': 'name',
+ 'target_field': 'author_name',
+ 'threshold': 85
+ },
+ 'relationship_type': 'AUTHORED'
+ }
+
+ response = client.post('/api/links', json=payload)
+ assert response.status_code == 200
+ data = response.get_json()
+ assert data['status'] == 'success'
+ assert data['link']['match_strategy'] == 'fuzzy'
+
+
+def test_create_link_table_import_strategy(client):
+ """Test creating link with table_import match strategy."""
+ # Create labels
+ client.post('/api/labels', json={
+ 'name': 'Project',
+ 'properties': [{'name': 'name', 'type': 'string'}],
+ 'relationships': []
+ })
+
+ payload = {
+ 'name': 'Import Projects from CSV',
+ 'source_label': 'Project',
+ 'target_label': 'Project',
+ 'match_strategy': 'table_import',
+ 'match_config': {
+ 'table_data': 'name,budget\nProject A,100000\nProject B,200000'
+ },
+ 'relationship_type': 'RELATED_TO'
+ }
+
+ response = client.post('/api/links', json=payload)
+ assert response.status_code == 200
+ data = response.get_json()
+ assert data['status'] == 'success'
+ assert data['link']['match_strategy'] == 'table_import'
+
+
+def test_create_link_api_endpoint_strategy(client):
+ """Test creating link with api_endpoint match strategy."""
+ # Create labels
+ client.post('/api/labels', json={
+ 'name': 'User',
+ 'properties': [{'name': 'id', 'type': 'number'}],
+ 'relationships': []
+ })
+
+ payload = {
+ 'name': 'Fetch Users from API',
+ 'source_label': 'User',
+ 'target_label': 'User',
+ 'match_strategy': 'api_endpoint',
+ 'match_config': {
+ 'url': 'https://api.example.com/users',
+ 'json_path': '$.data.users[*]'
+ },
+ 'relationship_type': 'RELATED_TO'
+ }
+
+ response = client.post('/api/links', json=payload)
+ assert response.status_code == 200
+ data = response.get_json()
+ assert data['status'] == 'success'
+ assert data['link']['match_strategy'] == 'api_endpoint'
+
+
+def test_get_available_labels(client):
+ """Test getting available labels for dropdown population."""
+ # Create some labels
+ client.post('/api/labels', json={
+ 'name': 'Person',
+ 'properties': [{'name': 'name', 'type': 'string'}],
+ 'relationships': []
+ })
+ client.post('/api/labels', json={
+ 'name': 'File',
+ 'properties': [{'name': 'path', 'type': 'string'}],
+ 'relationships': []
+ })
+
+ response = client.get('/api/links/available-labels')
+ assert response.status_code == 200
+ data = response.get_json()
+ assert data['status'] == 'success'
+ assert 'labels' in data
+ assert len(data['labels']) >= 2
+ label_names = [l['name'] for l in data['labels']]
+ assert 'Person' in label_names
+ assert 'File' in label_names
diff --git a/tests/test_table_format_registry.py b/tests/test_table_format_registry.py
new file mode 100644
index 0000000..9b342c5
--- /dev/null
+++ b/tests/test_table_format_registry.py
@@ -0,0 +1,332 @@
+"""
+Tests for Table Format Registry.
+
+Tests CRUD operations, format detection, and data preview.
+"""
+import pytest
+import tempfile
+import os
+import io
+import csv
+from scidk.core.table_format_registry import TableFormatRegistry
+
+
+@pytest.fixture
+def registry():
+ """Create a temporary registry for testing."""
+ # Use temporary database
+ with tempfile.NamedTemporaryFile(suffix='.db', delete=False) as tmp:
+ db_path = tmp.name
+
+ reg = TableFormatRegistry(db_path=db_path)
+
+ yield reg
+
+ # Cleanup
+ reg.db.close()
+ if os.path.exists(db_path):
+ os.unlink(db_path)
+
+
+def test_preprogrammed_formats_seeded(registry):
+ """Test that preprogrammed formats are seeded on initialization."""
+ formats = registry.list_formats(include_preprogrammed=True)
+
+ # Should have at least the 4 preprogrammed formats
+ preprogrammed = [f for f in formats if f['is_preprogrammed']]
+ assert len(preprogrammed) == 4
+
+ names = {f['name'] for f in preprogrammed}
+ assert 'CSV (Standard)' in names
+ assert 'TSV (Standard)' in names
+ assert 'Excel (Standard)' in names
+ assert 'Parquet (Standard)' in names
+
+
+def test_create_format(registry):
+ """Test creating a new table format."""
+ format_data = {
+ 'name': 'My CSV Format',
+ 'file_type': 'csv',
+ 'delimiter': ';',
+ 'encoding': 'utf-8',
+ 'has_header': True,
+ 'header_row': 0,
+ 'target_label': 'Person',
+ 'description': 'Custom semicolon-separated format'
+ }
+
+ format_config = registry.create_format(format_data)
+
+ assert format_config['id'] is not None
+ assert format_config['name'] == 'My CSV Format'
+ assert format_config['file_type'] == 'csv'
+ assert format_config['delimiter'] == ';'
+ assert format_config['encoding'] == 'utf-8'
+ assert format_config['has_header'] is True
+ assert format_config['header_row'] == 0
+ assert format_config['target_label'] == 'Person'
+ assert format_config['description'] == 'Custom semicolon-separated format'
+ assert format_config['is_preprogrammed'] is False
+
+
+def test_create_format_validation(registry):
+ """Test format creation validation."""
+ # Missing name
+ with pytest.raises(ValueError, match="Format name is required"):
+ registry.create_format({'file_type': 'csv'})
+
+ # Missing file_type
+ with pytest.raises(ValueError, match="File type is required"):
+ registry.create_format({'name': 'Test'})
+
+ # Invalid file_type
+ with pytest.raises(ValueError, match="File type must be one of"):
+ registry.create_format({'name': 'Test', 'file_type': 'invalid'})
+
+
+def test_create_duplicate_name(registry):
+ """Test that duplicate names are rejected."""
+ data = {
+ 'name': 'Duplicate Format',
+ 'file_type': 'csv'
+ }
+
+ # First creation should succeed
+ registry.create_format(data)
+
+ # Second creation with same name should fail
+ with pytest.raises(ValueError, match="already exists"):
+ registry.create_format(data)
+
+
+def test_get_format(registry):
+ """Test retrieving a format by ID."""
+ data = {
+ 'name': 'Get Test Format',
+ 'file_type': 'tsv',
+ 'delimiter': '\t'
+ }
+
+ created = registry.create_format(data)
+ format_id = created['id']
+
+ retrieved = registry.get_format(format_id)
+
+ assert retrieved is not None
+ assert retrieved['id'] == format_id
+ assert retrieved['name'] == 'Get Test Format'
+ assert retrieved['file_type'] == 'tsv'
+
+
+def test_get_nonexistent_format(registry):
+ """Test retrieving a format that doesn't exist."""
+ result = registry.get_format('nonexistent-id')
+ assert result is None
+
+
+def test_list_formats(registry):
+ """Test listing all formats."""
+ # Create custom formats
+ registry.create_format({'name': 'Custom CSV', 'file_type': 'csv'})
+ registry.create_format({'name': 'Custom TSV', 'file_type': 'tsv'})
+
+ # List all formats
+ all_formats = registry.list_formats(include_preprogrammed=True)
+ assert len(all_formats) >= 6 # 4 preprogrammed + 2 custom
+
+ # List only custom formats
+ custom_formats = registry.list_formats(include_preprogrammed=False)
+ assert len(custom_formats) == 2
+ assert all(not f['is_preprogrammed'] for f in custom_formats)
+
+
+def test_update_format(registry):
+ """Test updating an existing format."""
+ # Create format
+ data = {
+ 'name': 'Update Test',
+ 'file_type': 'csv',
+ 'delimiter': ','
+ }
+ created = registry.create_format(data)
+ format_id = created['id']
+
+ # Update format
+ updates = {
+ 'name': 'Updated Name',
+ 'delimiter': ';',
+ 'target_label': 'NewLabel'
+ }
+ updated = registry.update_format(format_id, updates)
+
+ assert updated['name'] == 'Updated Name'
+ assert updated['delimiter'] == ';'
+ assert updated['target_label'] == 'NewLabel'
+ assert updated['file_type'] == 'csv' # Unchanged
+
+
+def test_update_nonexistent_format(registry):
+ """Test updating a format that doesn't exist."""
+ with pytest.raises(ValueError, match="not found"):
+ registry.update_format('nonexistent-id', {'name': 'New Name'})
+
+
+def test_update_preprogrammed_format(registry):
+ """Test that preprogrammed formats cannot be updated."""
+ formats = registry.list_formats(include_preprogrammed=True)
+ preprogrammed = [f for f in formats if f['is_preprogrammed']][0]
+
+ with pytest.raises(ValueError, match="Cannot modify preprogrammed"):
+ registry.update_format(preprogrammed['id'], {'name': 'New Name'})
+
+
+def test_delete_format(registry):
+ """Test deleting a format."""
+ # Create format
+ data = {'name': 'Delete Test', 'file_type': 'csv'}
+ created = registry.create_format(data)
+ format_id = created['id']
+
+ # Delete format
+ deleted = registry.delete_format(format_id)
+ assert deleted is True
+
+ # Verify deleted
+ retrieved = registry.get_format(format_id)
+ assert retrieved is None
+
+
+def test_delete_nonexistent_format(registry):
+ """Test deleting a format that doesn't exist."""
+ deleted = registry.delete_format('nonexistent-id')
+ assert deleted is False
+
+
+def test_delete_preprogrammed_format(registry):
+ """Test that preprogrammed formats cannot be deleted."""
+ formats = registry.list_formats(include_preprogrammed=True)
+ preprogrammed = [f for f in formats if f['is_preprogrammed']][0]
+
+ with pytest.raises(ValueError, match="Cannot delete preprogrammed"):
+ registry.delete_format(preprogrammed['id'])
+
+
+def test_detect_format_csv(registry):
+ """Test auto-detecting CSV format."""
+ # Create sample CSV content
+ csv_content = "Name,Email,Age\nJohn,john@example.com,30\nJane,jane@example.com,25"
+ file_bytes = csv_content.encode('utf-8')
+
+ detected = registry.detect_format(file_bytes, filename='test.csv')
+
+ assert detected['file_type'] == 'csv'
+ assert detected['delimiter'] == ','
+ assert detected['encoding'] == 'utf-8'
+ assert detected['has_header'] is True
+ assert detected['sample_columns'] == ['Name', 'Email', 'Age']
+
+
+def test_detect_format_tsv(registry):
+ """Test auto-detecting TSV format."""
+ # Create sample TSV content
+ tsv_content = "Name\tEmail\tAge\nJohn\tjohn@example.com\t30\nJane\tjane@example.com\t25"
+ file_bytes = tsv_content.encode('utf-8')
+
+ detected = registry.detect_format(file_bytes, filename='test.tsv')
+
+ assert detected['file_type'] == 'tsv'
+ assert detected['delimiter'] == '\t'
+ assert detected['encoding'] == 'utf-8'
+ assert detected['has_header'] is True
+ assert detected['sample_columns'] == ['Name', 'Email', 'Age']
+
+
+def test_detect_format_fallback_encoding(registry):
+ """Test detecting format with non-UTF-8 encoding falls back to latin-1."""
+ # Create binary data that's not valid UTF-8 but valid latin-1
+ invalid_utf8_bytes = bytes([0xFF, 0xFE, 0xFD])
+
+ detected = registry.detect_format(invalid_utf8_bytes, filename='test.csv')
+
+ # Should fall back to latin-1 encoding
+ assert detected['encoding'] in ['latin-1', 'utf-16'] # Could detect either
+ assert detected['file_type'] == 'csv'
+
+
+def test_preview_data_csv(registry):
+ """Test previewing CSV data."""
+ # Create CSV format
+ format_data = {
+ 'name': 'Preview Test CSV',
+ 'file_type': 'csv',
+ 'delimiter': ',',
+ 'encoding': 'utf-8',
+ 'has_header': True,
+ 'header_row': 0
+ }
+ format_config = registry.create_format(format_data)
+ format_id = format_config['id']
+
+ # Create sample CSV content
+ csv_content = "Name,Email,Age\nJohn,john@example.com,30\nJane,jane@example.com,25\nBob,bob@example.com,35"
+ file_bytes = csv_content.encode('utf-8')
+
+ preview = registry.preview_data(file_bytes, format_id, num_rows=2)
+
+ assert 'error' not in preview
+ assert preview['columns'] == ['Name', 'Email', 'Age']
+ assert len(preview['rows']) == 2
+ assert preview['rows'][0]['Name'] == 'John'
+ assert preview['rows'][1]['Name'] == 'Jane'
+ assert preview['total_rows'] == 3
+
+
+def test_preview_data_invalid_format(registry):
+ """Test previewing data with non-existent format."""
+ file_bytes = b"test"
+ preview = registry.preview_data(file_bytes, 'nonexistent-id')
+
+ assert 'error' in preview
+ assert 'not found' in preview['error']
+
+
+def test_column_mappings(registry):
+ """Test creating format with column mappings."""
+ format_data = {
+ 'name': 'Mapped Format',
+ 'file_type': 'csv',
+ 'column_mappings': {
+ 'Name': {
+ 'label_property': 'full_name',
+ 'type_hint': 'string',
+ 'ignore': False
+ },
+ 'Age': {
+ 'label_property': 'age_years',
+ 'type_hint': 'number',
+ 'ignore': False
+ }
+ }
+ }
+
+ format_config = registry.create_format(format_data)
+
+ assert 'column_mappings' in format_config
+ assert 'Name' in format_config['column_mappings']
+ assert format_config['column_mappings']['Name']['label_property'] == 'full_name'
+ assert format_config['column_mappings']['Age']['type_hint'] == 'number'
+
+
+def test_format_with_sheet_name(registry):
+ """Test creating Excel format with sheet name."""
+ format_data = {
+ 'name': 'Excel with Sheet',
+ 'file_type': 'excel',
+ 'sheet_name': 'Data'
+ }
+
+ format_config = registry.create_format(format_data)
+
+ assert format_config['sheet_name'] == 'Data'
+ assert format_config['file_type'] == 'excel'