Master to develop

beagle/__init__.py

@@ -1 +1 @@
-__version__ = "2.0.0"
+__version__ = "2.1.1"

beagle/settings.py

@@ -44,6 +44,7 @@
     "runner.apps.RunnerConfig",
     "beagle_etl.apps.BeagleEtlConfig",
     "file_system.apps.FileSystemConfig",
+    "file_manager.apps.FileManagerConfig",
     "notifier.apps.NotifierConfig",
     "study.apps.StudyConfig",
     "django.contrib.admin",
@@ -62,8 +63,8 @@
     "drf_yasg",
     "advanced_filters",
     "ddtrace.contrib.django",
-    "echo_client",
     "smile_client",
+    "django_celery_results",
 ]
 
 
@@ -251,6 +252,7 @@
 RABBITMQ_URL = os.environ.get("BEAGLE_RABBITMQ_URL", "localhost")
 
 CELERY_BROKER_URL = "amqp://%s:%s@%s/" % (RABBITMQ_USERNAME, RABBITMQ_PASSWORD, RABBITMQ_URL)
+CELERY_RESULT_BACKEND = "django-db"  # Use Django database as result backend for chord support
 CELERY_ACCEPT_CONTENT = ["json"]
 CELERY_TASK_SERIALIZER = "json"
 CELERY_RESULT_SERIALIZER = "json"
@@ -297,17 +299,16 @@
     "CALLBACK": "beagle_etl.smile_service.smile_callback.persist_message",
 }
 
-IMPORT_FILE_GROUP = os.environ.get("BEAGLE_IMPORT_FILE_GROUP", "1a1b29cf-3bc2-4f6c-b376-d4c5d701166a")
+IMPORT_FILE_GROUP = os.environ.get("BEAGLE_IMPORT_FILE_GROUP")
 
-POOLED_NORMAL_FILE_GROUP = os.environ.get("BEAGLE_POOLED_NORMAL_FILE_GROUP", "b6857a56-5d45-451f-b4f6-26148946080f")
+POOLED_NORMAL_FILE_GROUP = os.environ.get("BEAGLE_POOLED_NORMAL_FILE_GROUP")
 
-DMP_BAM_FILE_GROUP = os.environ.get("BEAGLE_DMP_BAM_FILE_GROUP", "9ace63bf-ed55-461c-9ac0-1c5ee710d957")
+DMP_BAM_FILE_GROUP = os.environ.get("BEAGLE_DMP_BAM_FILE_GROUP")
 
 RIDGEBACK_URL = os.environ.get("BEAGLE_RIDGEBACK_URL", "http://localhost:5003")
 
 LOG_PATH = os.environ.get("BEAGLE_LOG_PATH", "beagle-server.log")
 
-ECHO_LOG_PATH = os.environ.get("BEAGLE_ECHO_LOG_PATH", "echo_client.log")
 SMILE_LOG_PATH = os.environ.get("BEAGLE_SMILE_LOG_PATH", "smile_client.log")
 
 LOGGING = {
@@ -321,26 +322,18 @@
     },
     "handlers": {
         "console": {
-            "level": "DEBUG",
+            "level": "INFO",
             "class": "logging.StreamHandler",
             "formatter": "simple",
         },
         "file": {
-            "level": "DEBUG",
+            "level": "INFO",
             "class": "logging.handlers.RotatingFileHandler",
             "filename": LOG_PATH,
             "maxBytes": 209715200,
             "backupCount": 10,
             "formatter": "simple",
         },
-        "echo_client_log": {
-            "level": "DEBUG",
-            "class": "logging.handlers.RotatingFileHandler",
-            "filename": ECHO_LOG_PATH,
-            "maxBytes": 209715200,
-            "backupCount": 10,
-            "formatter": "simple",
-        },
         "smile_client_log": {
             "level": "DEBUG",
             "class": "logging.handlers.RotatingFileHandler",
@@ -351,14 +344,17 @@
         },
     },
     "loggers": {
-        "django_auth_ldap": {"level": "DEBUG", "handlers": ["console"]},
+        "django_auth_ldap": {"level": "INFO", "handlers": ["console"]},
         "django": {
             "handlers": ["file", "console"],
-            "level": "DEBUG",
+            "level": "INFO",
             "propagate": True,
         },
-        "echo_client": {"level": "DEBUG", "handlers": ["echo_client_log", "console"]},
-        "smile_client": {"level": "DEBUG", "handlers": ["echo_client_log", "console"]},
+        "smile_client": {"level": "INFO", "handlers": ["smile_client_log", "console"]},
+    },
+    "root": {
+        "handlers": ["file", "console"],
+        "level": "INFO",
     },
 }
 
@@ -387,10 +383,12 @@
 BEAGLE_DEFAULT_QUEUE = os.environ.get("BEAGLE_DEFAULT_QUEUE", "beagle_default_queue")
 BEAGLE_CHECK_FILES_QUEUE = os.environ.get("BEAGLE_CHECK_FILES_QUEUE", "beagle_check_files_queue")
 BEAGLE_JOB_SCHEDULER_QUEUE = os.environ.get("BEAGLE_JOB_SCHEDULER_QUEUE", "beagle_job_scheduler_queue")
+BEAGLE_FILE_MANAGER_QUEUE = os.environ.get("BEAGLE_FILE_MANAGER_QUEUE", "beagle_file_manager_queue")
 BEAGLE_SHARED_TMPDIR = os.environ.get("BEAGLE_SHARED_TMPDIR", "/juno/work/ci/temp")
 BEAGLE_TMPDIR = os.environ.get("BEAGLE_TMPDIR", "/tmp")
 
-PROCESS_SMILE_MESSAGES_PERIOD = os.environ.get("BEAGLE_PROCESS_SMILE_MESSAGES_PERIOD", 900)
+PROCESS_SMILE_MESSAGES_PERIOD = os.environ.get("BEAGLE_PROCESS_SMILE_MESSAGES_PERIOD", 90)
+PROCESS_REQUEST_CALLBACK_PERIOD = os.environ.get("BEAGLE_PROCESS_REQUEST_CALLBACK_PERIOD", 900)
 CHECK_JOB_STATUS_PERIOD = os.environ.get("BEAGLE_CHECK_JOB_STATUS_PERIOD", 60)
 PROCESS_TRIGGERS_PERIOD = os.environ.get("BEAGLE_PROCESS_TRIGGERS_PERIOD", 120)
 CHECK_JOB_TIMEOUTS = os.environ.get("BEAGLE_CHECK_JOB_TIMEOUTS", 86400.0)
@@ -465,7 +463,7 @@
 MAPPING = json.loads(os.environ.get("BEAGLE_FILE_MAPPING", "{}"))
 COPY_FILE_PERMISSION = 0o644
 COPY_DIR_PERMISSION = 0o750
-COPY_GROUP_OWNERSHIP = os.environ.get("BEAGLE_GROUP_OWNERSHIP", "cmoigo")
+STAGE_DAYS = int(os.environ.get("BEAGLE_STAGE_DAYS", 30))
 
 FASTQ_DEFAULT_LOCATION_PREFIX = os.environ.get("BEAGLE_FASTQ_DEFAULT_LOCATION_PREFIX")
 FASTQ_IRIS_LOCATION_PREFIX = os.environ.get("BEAGLE_FASTQ_IRIS_LOCATION_PREFIX")
@@ -549,15 +547,3 @@
     "HC_IMPACT": {None: "UNKNOWN", "null": "UNKNOWN", "IMPACT505_BAITS": "IMPACT505"},
 }
 SHELL_PLUS = "ipython"
-
-
-ECHO_SETTINGS = {
-    "USERNAME": os.getenv("ECHO_USERNAME"),
-    "PASSWORD": os.getenv("ECHO_PASSWORD"),
-    "HOST": os.getenv("ECHO_RABBITMQ_HOST"),
-    "PORT": os.getenv("ECHO_RABBITMQ_PORT"),
-    "VHOST": os.getenv("ECHO_VHOST", "/"),
-    "ECHO_TASK_QUEUE": os.getenv("ECHO_TASK_QUEUE"),
-    "ECHO_CONFIRMATION_QUEUE": os.getenv("ECHO_CONFIRMATION_QUEUE"),
-    "CALLBACK": os.getenv("ECHO_CALLBACK"),
-}

beagle/urls.py

@@ -41,6 +41,10 @@
 router = routers.DefaultRouter()
 router.register("register", UserRequestViewSet)
 
+# Customize Django Admin
+admin.site.site_header = f"Voyager - Beagle v{__version__}"
+admin.site.site_title = "Voyager"
+admin.site.index_title = "Beagle Administration"
 
 urlpatterns = [
     url(r"^$", schema_view.with_ui("swagger", cache_timeout=0), name="schema-swagger-ui"),

beagle_etl/admin.py

@@ -6,10 +6,6 @@
     ETLConfiguration,
     SMILEMessage,
     RequestCallbackJob,
-    NormalizerModel,
-    ValidatorModel,
-    SkipProject,
-    CopyFileTask,
 )
 from advanced_filters.admin import AdminAdvancedFiltersMixin
 
@@ -42,38 +38,7 @@ class RequestCallbackJobAdmin(ModelAdmin):
     ordering = ("-created_date",)
 
 
-class NormalizerAdmin(ModelAdmin):
-    list_display = (
-        "condition",
-        "normalizer",
-    )
-
-
-class ValidatorAdmin(ModelAdmin):
-    list_display = ("name",)
-
-
-class SkipProjectAdmin(ModelAdmin):
-    list_display = ("skip_projects",)
-
-
-class CopyFileTaskAdmin(ModelAdmin):
-    list_display = (
-        "id",
-        "source",
-        "destination",
-        link_relation("smile_message"),
-        "created_date",
-        "status",
-    )
-    ordering = ("-created_date",)
-
-
 admin.site.register(Operator, OperatorAdmin)
 admin.site.register(ETLConfiguration, AssayAdmin)
 admin.site.register(SMILEMessage, SMILEMessagesAdmin)
 admin.site.register(RequestCallbackJob, RequestCallbackJobAdmin)
-admin.site.register(NormalizerModel, NormalizerAdmin)
-admin.site.register(ValidatorModel, ValidatorAdmin)
-admin.site.register(SkipProject, SkipProjectAdmin)
-admin.site.register(CopyFileTask, CopyFileTaskAdmin)

beagle_etl/celery.py

@@ -37,6 +37,7 @@ def setup_task_logger(logger, *args, **kwargs):
     "runner.tasks.create_jobs_from_request": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "runner.tasks.create_jobs_from_chaining": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "runner.tasks.create_jobs_from_pairs": {"queue": settings.BEAGLE_RUNNER_QUEUE},
+    "runner.tasks.create_operator_run_from_jobs": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "runner.tasks.add_pipeline_to_cache": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "runner.tasks.running_job": {"queue": settings.BEAGLE_RUNNER_QUEUE},
     "runner.tasks.terminate_job": {"queue": settings.BEAGLE_RUNNER_QUEUE},
@@ -47,19 +48,20 @@ def setup_task_logger(logger, *args, **kwargs):
     "file_system.tasks.check_fastq_files": {"queue": settings.BEAGLE_CHECK_FILES_QUEUE},
     "beagle_etl.tasks.job_processor": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
     "beagle_etl.tasks.process_smile_events": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
-    "beagle_etl.jobs.metadb_jobs.new_request": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
-    "beagle_etl.jobs.metadb_jobs.update_request_job": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
-    "beagle_etl.jobs.metadb_jobs.update_sample_job": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
+    "beagle_etl.tasks.process_job_with_lock": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
     "beagle_etl.jobs.metadb_jobs.update_job": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
     "beagle_etl.jobs.metadb_jobs.not_supported": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
     "beagle_etl.jobs.metadb_jobs.request_callback": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
     "beagle_etl.jobs.metadb_jobs.calculate_checksum": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
+    "file_manager.tasks.stage_file_job": {"queue": settings.BEAGLE_FILE_MANAGER_QUEUE},
+    "file_manager.tasks.check_for_clean_up": {"queue": settings.BEAGLE_FILE_MANAGER_QUEUE},
+    "file_manager.tasks.clean_up_file": {"queue": settings.BEAGLE_FILE_MANAGER_QUEUE},
 }
 
 app.conf.beat_schedule = {
     "process_request_callback_jobs": {
         "task": "beagle_etl.tasks.process_request_callback_jobs",
-        "schedule": settings.PROCESS_SMILE_MESSAGES_PERIOD,
+        "schedule": settings.PROCESS_REQUEST_CALLBACK_PERIOD,
         "options": {"queue": settings.BEAGLE_DEFAULT_QUEUE},
     },
     "process_smile_imports": {
@@ -97,4 +99,9 @@ def setup_task_logger(logger, *args, **kwargs):
         "schedule": crontab(day_of_week=1, hour=0, minute=0),
         "options": {"queue": settings.BEAGLE_CHECK_FILES_QUEUE},
     },
+    "check_staged_file_cleanup": {
+        "task": "file_manager.tasks.check_for_clean_up",
+        "schedule": crontab(hour=2, minute=0),
+        "options": {"queue": settings.BEAGLE_FILE_MANAGER_QUEUE},
+    },
 }

beagle_etl/exceptions.py

@@ -100,3 +100,11 @@ class IncorrectlyFormattedPrimaryId(ETLExceptions):
     """
 
     code = 112
+
+
+class FailedToFetchRequestMetadata(ETLExceptions):
+    """
+    Failed to fetch request metadata
+    """
+
+    code = 113