diff --git a/docs/common/latex_generation.rst b/docs/common/latex_generation.rst index 929722d32..0d852a5c1 100644 --- a/docs/common/latex_generation.rst +++ b/docs/common/latex_generation.rst @@ -100,7 +100,7 @@ to access the object for which we’re generating data. from django import forms - from tom_dataproducts.models import ReducedDatum + from tom_dataproducts.models import PhotometryReducedDatum, SpectroscopyReducedDatum from tom_publications.latex import GenericLatexProcessor, GenericLatexForm from tom_targets.models import Target @@ -110,18 +110,17 @@ to access the object for which we’re generating data. form_class = TargetDataLatexForm def create_latex_table_data(self, cleaned_data): - target = Target.objects.get(pk=cleaned_data.get('model_pk')) - data = ReducedDatum.objects.filter(target=target, data_type=cleaned_data.get('data_type')) - - table_data = {} - if cleaned_data.get('data_type') == 'photometry': - for datum in data: - for key, value in json.loads(datum.value).items(): - table_data.setdefault(key, []).append(value) - elif cleaned_data.get('data_type') == 'spectroscopy': - ... - - return table_data + target = Target.objects.get(pk=cleaned_data.get('model_pk')) + table_data = {} + if cleaned_data.get('data_type') == 'photometry': + data = PhotometryReducedDatum.objects.filter(target=target) + for brightness, bandpass in data.values_list('brightness', 'bandpass'): + table_data.setdefault('brightness', []).append(brightness) + table_data.setdefault('bandpass', []).append(bandpass) + elif cleaned_data.get('data_type') == 'spectroscopy': + ... + + return table_data The above example only shows the photometric table generation, but spectroscopic can be left as an exercise to the reader. @@ -184,7 +183,7 @@ TOM. Here’s our final ``target_data_latex_processor.py``: from django import forms - from tom_dataproducts.models import ReducedDatum + from tom_dataproducts.models import PhotometryReducedDatum, SpectroscopyReducedDatum from tom_publications.latex import GenericLatexProcessor, GenericLatexForm from tom_targets.models import Target @@ -202,13 +201,12 @@ TOM. Here’s our final ``target_data_latex_processor.py``: def create_latex_table_data(self, cleaned_data): target = Target.objects.get(pk=cleaned_data.get('model_pk')) - data = ReducedDatum.objects.filter(target=target, data_type=cleaned_data.get('data_type')) - table_data = {} if cleaned_data.get('data_type') == 'photometry': - for datum in data: - for key, value in json.loads(datum.value).items(): - table_data.setdefault(key, []).append(value) + data = PhotometryReducedDatum.objects.filter(target=target) + for brightness, bandpass in data.values_list('brightness', 'bandpass'): + table_data.setdefault('brightness', []).append(brightness) + table_data.setdefault('bandpass', []).append(bandpass) elif cleaned_data.get('data_type') == 'spectroscopy': ... diff --git a/docs/common/permissions.rst b/docs/common/permissions.rst index a4dd46fef..663077bb4 100644 --- a/docs/common/permissions.rst +++ b/docs/common/permissions.rst @@ -139,3 +139,15 @@ The above code will allow all users in the groups that the example user belongs ``ReducedDatum``: * ``tom_dataproducts.view_reduceddatum`` + +``PhotometryReducedDatum``: + +* ``tom_dataproducts.view_photometryreduceddatum`` + +``SpectroscopyReducedDatum``: + +* ``tom_dataproducts.view_spectroscopyreduceddatum`` + +``AstrometryReducedDatum``: + +* ``tom_dataproducts.view_astrometryreduceddatum`` diff --git a/docs/customization/customize_template_tags.rst b/docs/customization/customize_template_tags.rst index 557189f64..b9e88f04c 100644 --- a/docs/customization/customize_template_tags.rst +++ b/docs/customization/customize_template_tags.rst @@ -154,8 +154,8 @@ approach here: You can see that we’ll eventually be returning a dictionary, but first we need to add our logic. We’ll need to use the ``Target`` passed in to -get all ``ReducedDatum`` objects for that ``Target`` with a -``data_type`` of ``photometry``. Then we’ll need to order by +get all ``PhotometryReducedDatum`` objects for that ``Target`` +Then we’ll need to order by ``timestamp`` descending, and slice just the first few. Make sure to take note of the imports in this step! @@ -165,7 +165,7 @@ take note of the imports in this step! from django import template - from tom_dataproducts.models import ReducedDatum + from tom_dataproducts.models import PhotometryReducedDatum register = template.Library() @@ -173,8 +173,8 @@ take note of the imports in this step! @register.inclusion_tag('custom_code/partials/recent_photometry.html') def recent_photometry(target, num_points=1): - photometry = ReducedDatum.objects.filter(data_type='photometry').order_by('-timestamp')[:num_points] - return {'recent_photometry': [(datum.timestamp, json.loads(datum.value)['magnitude']) for datum in photometry]} + photometry = PhotometryReducedDatum.objects.order_by('-timestamp')[:num_points] + return {'recent_photometry': [(datum.timestamp, datum.brightness) for datum in photometry]} It’s only a couple of lines, but there’s a lot going on here. The first line does the aforemention database query and slices the first point of @@ -330,4 +330,4 @@ As far as this template tag goes, as of this tutorial, it’s now a part of the base TOM Toolkit, but all of the information here should provide you with the ability to write your own. -.. |image0| image:: /_static/customize_template_tags_doc/Templatetags.png \ No newline at end of file +.. |image0| image:: /_static/customize_template_tags_doc/Templatetags.png diff --git a/docs/introduction/tomarchitecture.rst b/docs/introduction/tomarchitecture.rst index c109124d8..0bf49f940 100644 --- a/docs/introduction/tomarchitecture.rst +++ b/docs/introduction/tomarchitecture.rst @@ -52,7 +52,7 @@ they are left with a functioning but generic TOM. It is then up to the developer to implement the specific features that their science case requires. The toolkit tries to facilitate this as efficiently as possible and provides :doc:`documentation ` in areas of customization from :doc:`changing the HTML layout of a page ` -to :doc:`customizing an OCS facility and forms ` and even +to :doc:`customizing an OCS facility and forms ` and even :doc:`creating a new alert broker `. Django, and by extension the toolkit, rely heavily on object oriented @@ -95,7 +95,7 @@ each other. This means a TOM developer can easily change the layout and style of any page without modifying the underlying framework's code directly. Entire pages may be replaced, or only "blocks" within a template. -Compare these screenshots of the `standard target detail page <../../../_static/architecture/snex2layout.png>`_ and the +Compare these screenshots of the `standard target detail page <../../../_static/architecture/snex2layout.png>`_ and the `Global Supernova Project's target detail page <../../../_static/architecture/snex2layout.png>`_, the latter taking heavy advantage of template inheritance. @@ -266,28 +266,46 @@ ReducedDatum ------------ A ``ReducedDatum`` is a single point of data associated with a ``Target`` and optionally a -``DataProduct``. The single data point is typically a single point of photometry or an individual -spectrum. The ``ReducedDatum`` model has the following fields, in addition to its aforementioned +``DataProduct``. +There are three classes of ReducedDatum for the common data types: +``PhotometryReducedDatum``, ``SpectroscopyReducedDatum``, and ``AstrometryReducedDatum``. +The ``ReducedDatum`` is a general model meant to be flexible enough to allow for other data types as well. + +The ``ReducedDatum`` model has the following fields, in addition to its aforementioned foreign key relationships: - ``data_type`` is maintained on both the ``ReducedDatum`` and ``DataProduct`` for the case when data is brought in from another source, such as a broker - The ``source_name`` optionally refers to the original source of the data. The intent of this field was to track data ingested from brokers, but could potentially be used for other purposes. - ``source_location`` optionally gives a hard location to the source--for a broker, it would be a link to the original alert. - The ``timestamp`` time at which the datum was produced. -- ``value`` is a ``TextField`` that can take any series of data. As implemented, photometry is stored as JSON with keys for magnitude and error, but the ``TextField`` provides flexibility for additional photometry values on the datum. Spectroscopy is also stored as JSON, with keys for ``magnitude`` and ``flux``. - -Feedback and bug reporting -========================== +- ``value`` is a ``JSONField`` that can take any series of data. +- ``telescope`` and ``instrument`` are optional fields that can be used to track additional metadata. -We hope the TOM Toolkit is helpful to you and your project. If you have any -concerns about implementation details, or questions about your own needs, please -don't hesitate to `reach out `_. Issues and pull requests -are also welcome on the project's `GitHub page `_. +The ``PhotometryReducedDatum`` model has the following Photometry specific fields: +- ``brightness`` and ``brightness_error`` are float fields that track the magnitude and error, respectively. +- ``bandpass`` is a char field that tracks the bandpass/filter of the photometry. +- ``limit`` optional float field that tracks the limiting magnitude of the photometry +- ``unit`` optional char field that tracks the unit of the photometry +- ``exposure_time`` optional float field that tracks the exposure time of the photometry +The ``SpectroscopyReducedDatum`` model has the following Spectroscopy specific fields: +- ``wavelength``, ``flux`` and ``error`` are all FloatArrayFields that track the wavelength, flux, and error of the spectroscopy, respectively +- ``unit`` optional char field that tracks the unit of the spectroscopy +- ``setup`` optional text field for arbitrary metadata about the spectroscopic setup +- ``exposure_time`` optional float field that tracks the exposure time of the spectroscopy +The ``AstrometryReducedDatum`` model has the following Astrometry specific fields: +- ``ra``, ``dec``, ``ra_error`` and ``dec_error`` are all float fields for tracking coordinates and error. Errors are optional. +- ``ra_error_units`` and ``dec_error_units`` optional char fields that track the units of the errors +Feedback and bug reporting +========================== +We hope the TOM Toolkit is helpful to you and your project. If you have any +concerns about implementation details, or questions about your own needs, please +don't hesitate to `reach out `_. Issues and pull requests +are also welcome on the project's `GitHub page `_. diff --git a/docs/managing_data/customizing_data_processing.rst b/docs/managing_data/customizing_data_processing.rst index 86bcec121..3f728e74d 100644 --- a/docs/managing_data/customizing_data_processing.rst +++ b/docs/managing_data/customizing_data_processing.rst @@ -21,12 +21,12 @@ tom_dataproducts app in the TOM Toolkit: Let’s start with a quick overview of ``models.py``. The file contains the Django models for the dataproducts app–in our case, ``DataProduct`` -and ``ReducedDatum``. The ``DataProduct`` contains information about +, ``ReducedDatum`` and the three specialized reduced data product classes: +``PhotometryReducedDatum``, ``SpectroscopyReducedDatum`` and ``AstrometryReducedDatum``. +The ``DataProduct`` contains information about uploaded or saved ``DataProducts``, such as the file name, file path, -and what kind of file it is. The ``ReducedDatum`` contains individual +and what kind of file it is. The ``*ReducedDatum`` classes contain individual science data points that are taken from the ``DataProduct`` files. -Examples of ``ReducedDatum`` points would be individual photometry -points or individual spectra. Each ``DataProduct`` also has a ``data_product_type``. The ``data_product_type`` is simply a description of what the file is, more diff --git a/docs/managing_data/plotting_data.rst b/docs/managing_data/plotting_data.rst index 22bac5722..30300d2bf 100644 --- a/docs/managing_data/plotting_data.rst +++ b/docs/managing_data/plotting_data.rst @@ -128,7 +128,7 @@ Next, add the function body: # x axis: target names. y axis: datum count data = [go.Bar( x=[target.name for target in targets], - y=[target.reduceddatum_set.count() for target in targets] + y=[target.photometryreduceddatum_set.count() for target in targets] )] # Create the plot figure = offline.plot(go.Figure(data=data), output_type='div', show_link=False) diff --git a/docs/managing_data/tom_direct_sharing.rst b/docs/managing_data/tom_direct_sharing.rst index bc0edb2ca..9ab67abfc 100644 --- a/docs/managing_data/tom_direct_sharing.rst +++ b/docs/managing_data/tom_direct_sharing.rst @@ -46,7 +46,7 @@ Receiving Shared Data: Reduced Datums: --------------- When your TOM receives a new ``ReducedDatum`` from another TOM it will be saved to your TOM's database with its source -set to the name of the TOM that submitted it. Currently, only Photometry data can be directly shared between +set to the name of the TOM that submitted it. Currently, only ``PhotometryReducedDatum`` can be directly shared between TOMS and a ``Target`` with a matching name or alias must exist in both TOMS for sharing to take place. Data Products: @@ -66,9 +66,3 @@ Target Lists: When your TOM receives a new ``TargetList`` from another TOM it will be saved to your TOM's database. If the targets in the ``TargetList`` are also shared, but already exist in the destination TOM, they will be added to the new ``TargetList``. - - - - - - diff --git a/tom_alerts/brokers/alerce.py b/tom_alerts/brokers/alerce.py index 41a49e923..b9a4568da 100644 --- a/tom_alerts/brokers/alerce.py +++ b/tom_alerts/brokers/alerce.py @@ -10,7 +10,7 @@ from tom_alerts.alerts import GenericAlert, GenericBroker, GenericQueryForm from tom_targets.models import Target -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum logger = logging.getLogger(__name__) @@ -573,35 +573,31 @@ def process_reduced_data(self, target, alert=None): for detection in lightcurve['detections']: mjd = Time(detection['mjd'], format='mjd', scale='utc') - value = { - 'filter': FILTERS[detection['fid']], - 'magnitude': detection['magpsf'], - 'error': detection['sigmapsf'], - 'telescope': 'ZTF', - } - ReducedDatum.objects.get_or_create( + PhotometryReducedDatum.objects.get_or_create( + target=target, + bandpass=FILTERS[detection['fid']], timestamp=mjd.to_datetime(TimezoneInfo()), - value=value, - source_name=self.name, - source_location=oid, - data_type='photometry', - target=target + defaults={ + 'brightness': detection['magpsf'], + 'brightness_error': detection['sigmapsf'], + 'telescope': 'ZTF', + 'source_name': self.name, + 'source_location': oid, + } ) for non_detection in lightcurve['non_detections']: mjd = Time(non_detection['mjd'], format='mjd', scale='utc') - value = { - 'filter': FILTERS[non_detection['fid']], - 'limit': non_detection['diffmaglim'], - 'telescope': 'ZTF', - } - ReducedDatum.objects.get_or_create( + PhotometryReducedDatum.objects.get_or_create( + target=target, + bandpass=FILTERS[non_detection['fid']], timestamp=mjd.to_datetime(TimezoneInfo()), - value=value, - source_name=self.name, - source_location=oid, - data_type='photometry', - target=target + defaults={ + 'limit': non_detection['diffmaglim'], + 'telescope': 'ZTF', + 'source_name': self.name, + 'source_location': oid, + } ) def to_target(self, alert): diff --git a/tom_alerts/brokers/gaia.py b/tom_alerts/brokers/gaia.py index e1385fabc..e4d902a03 100644 --- a/tom_alerts/brokers/gaia.py +++ b/tom_alerts/brokers/gaia.py @@ -12,7 +12,7 @@ from django import forms from tom_alerts.alerts import GenericAlert, GenericBroker, GenericQueryForm -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum BASE_BROKER_URL = 'http://gsaweb.ast.cam.ac.uk' @@ -181,17 +181,16 @@ def process_reduced_data(self, target, alert=None): jd = Time(float(phot_data[1]), format='jd', scale='utc') jd.to_datetime(timezone=TimezoneInfo()) - value = { - 'magnitude': float(phot_data[2]), - 'filter': 'G' - } - - rd, _ = ReducedDatum.objects.get_or_create( + rd, _ = PhotometryReducedDatum.objects.get_or_create( + target=target, + bandpass='G', timestamp=jd.to_datetime(timezone=TimezoneInfo()), - value=value, - source_name=self.name, - source_location=alert_url, - data_type='photometry', - target=target) + defaults={ + 'brightness': float(phot_data[2]), + 'source_name': self.name, + 'source_location': alert_url, + + } + ) return diff --git a/tom_alerts/tests/brokers/test_alerce.py b/tom_alerts/tests/brokers/test_alerce.py index 8e3187050..041482474 100644 --- a/tom_alerts/tests/brokers/test_alerce.py +++ b/tom_alerts/tests/brokers/test_alerce.py @@ -7,7 +7,7 @@ from faker import Faker from tom_alerts.brokers.alerce import ALeRCEBroker, ALeRCEQueryForm -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum from tom_targets.models import Target from tom_targets.tests.factories import SiderealTargetFactory @@ -401,7 +401,7 @@ def test_process_reduced_datum(self, mock_fetch_lightcurve): mock_fetch_lightcurve.return_value = test_data target = SiderealTargetFactory() ALeRCEBroker().process_reduced_data(target) - self.assertEqual(ReducedDatum.objects.count(), 2) + self.assertEqual(PhotometryReducedDatum.objects.count(), 2) def test_to_generic_alert(self): """Test to_generic_alert broker method.""" diff --git a/tom_alerts/tests/brokers/test_gaia.py b/tom_alerts/tests/brokers/test_gaia.py index 803032306..7c208b0cc 100644 --- a/tom_alerts/tests/brokers/test_gaia.py +++ b/tom_alerts/tests/brokers/test_gaia.py @@ -9,7 +9,7 @@ from tom_alerts.brokers.gaia import GaiaQueryForm from tom_alerts.brokers.gaia import GaiaBroker from tom_targets.models import Target -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum @override_settings(TOM_ALERT_CLASSES=['tom_alerts.brokers.gaia.GaiaBroker']) @@ -102,13 +102,12 @@ def setUp(self): "rvs": 'false'} ] self.test_target = Target.objects.create(name=self.alert_list[0]['name']) - ReducedDatum.objects.create( + PhotometryReducedDatum.objects.create( source_name='Gaia', source_location=111111, target=self.test_target, - data_type='photometry', timestamp=timezone.now(), - value=12345.6789 + brightness=12345.6789 ) @mock.patch('tom_alerts.brokers.gaia.requests.get') @@ -141,7 +140,7 @@ def test_process_reduced_data_with_alert(self, mock_requests_get): GaiaBroker().process_reduced_data(self.test_target, alert=self.alert_list[0]) - reduced_data = ReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') + reduced_data = PhotometryReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') self.assertGreater(reduced_data.count(), 1) self.assertEqual(reduced_data.count(), 3) # one from setUp and two from this test @@ -158,7 +157,7 @@ def test_process_reduced_data_without_alert(self, mock_fetch_alerts, mock_reques GaiaBroker().process_reduced_data(self.test_target) - reduced_data = ReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') + reduced_data = PhotometryReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') self.assertGreater(reduced_data.count(), 1) self.assertEqual(reduced_data.count(), 3) # one from setUp and two from this test @@ -183,6 +182,6 @@ def test_rewrite_process_reduced_data_with_alert(self): except ValidationError as e: self.fail(f'This test should have created two UNIQUE ReducedDatum objects, but {e}') - reduced_data = ReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') + reduced_data = PhotometryReducedDatum.objects.filter(target=self.test_target, source_name='Gaia') self.assertGreater(reduced_data.count(), 1) self.assertEqual(reduced_data.count(), 3) # one from setUp and two from this test diff --git a/tom_dataproducts/admin.py b/tom_dataproducts/admin.py index 2cdcb3f87..1be7d6b38 100644 --- a/tom_dataproducts/admin.py +++ b/tom_dataproducts/admin.py @@ -1,7 +1,12 @@ from django.contrib import admin -from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum +from tom_dataproducts.models import (DataProduct, DataProductGroup, ReducedDatum, + PhotometryReducedDatum, SpectroscopyReducedDatum, + AstrometryReducedDatum) admin.site.register(DataProduct) admin.site.register(DataProductGroup) admin.site.register(ReducedDatum) +admin.site.register(PhotometryReducedDatum) +admin.site.register(SpectroscopyReducedDatum) +admin.site.register(AstrometryReducedDatum) diff --git a/tom_dataproducts/alertstreams/hermes.py b/tom_dataproducts/alertstreams/hermes.py index d3cd8b2c7..55fe6ef86 100644 --- a/tom_dataproducts/alertstreams/hermes.py +++ b/tom_dataproducts/alertstreams/hermes.py @@ -9,7 +9,7 @@ from tom_alerts.models import AlertStreamMessage from tom_targets.models import Target, TargetList -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum, SpectroscopyReducedDatum import requests @@ -74,56 +74,34 @@ def get_hermes_photometry(self, datum): phot_table_row = { 'target_name': datum.target.name, 'date_obs': datum.timestamp.isoformat(), - 'telescope': datum.value.get('telescope'), - 'instrument': datum.value.get('instrument'), - 'bandpass': datum.value.get('filter', ''), + 'telescope': datum.telescope, + 'instrument': datum.instrument, + 'bandpass': datum.bandpass, } - brightness_unit = convert_astropy_brightness_to_hermes(datum.value.get('unit')) + brightness_unit = convert_astropy_brightness_to_hermes(datum.unit) if brightness_unit: phot_table_row['brightness_unit'] = brightness_unit - if datum.value.get('magnitude', None): - phot_table_row['brightness'] = datum.value['magnitude'] + if datum.brightness is not None: + phot_table_row['brightness'] = datum.brightness else: - phot_table_row['limiting_brightness'] = datum.value.get('limit', None) - error_value = datum.value.get('error', datum.value.get('magnitude_error', None)) - if error_value is not None: - phot_table_row['brightness_error'] = error_value + phot_table_row['limiting_brightness'] = datum.limit + if datum.brightness_error is not None: + phot_table_row['brightness_error'] = datum.brightness_error return phot_table_row def get_hermes_spectroscopy(self, datum): - """Build a row for a Hermes Spectroscopy Table using a TOM Spectroscopy datum - The datum is assumed to have is json value be of the form {1: {flux: 1, wavelength:200}, 2: {},...} - Or the form {'flux': [1,2,3,...], 'wavelength': [1,2,3,...]} - """ - flux_list = [] - flux_error_list = [] - wavelength_list = [] - if 'flux' in datum.value and 'wavelength' in datum.value: - flux_list = datum.value['flux'] - wavelength_list = datum.value['wavelength'] - flux_error_list = datum.value.get('flux_error', datum.value.get('error', [])) - else: - for entry in datum.value.values(): - if 'flux' in entry: - flux_list.append(entry['flux']) - if 'wavelength' in entry: - wavelength_list.append(entry['wavelength']) - if 'error' in entry: - flux_error_list.append(entry['error']) - if 'flux_error' in entry: - flux_error_list.append(entry['flux_error']) + """Build a row for a Hermes Spectroscopy Table using a SpectroscopyReducedDatum.""" if self.validate: - if len(flux_list) != len(wavelength_list): + if len(datum.flux) != len(datum.wavelength): msg = f"Spectroscopy Datum {datum.id} has mismatched flux and wavelength values" logger.error(msg) raise HermesMessageException(msg) - elif len(flux_list) == 0 or len(wavelength_list) == 0: - msg = f"Spectroscopy Datum {datum.id} has spectrum data in unknown format." - msg += "Please implement a custom HermesDatumConverter to support your data format." + elif len(datum.flux) == 0 or len(datum.wavelength) == 0: + msg = f"Spectroscopy Datum {datum.id} has no spectral data." logger.error(msg) raise HermesMessageException(msg) - if flux_error_list and len(flux_error_list) != len(flux_list): + if datum.error and len(datum.error) != len(datum.flux): msg = f"Spectroscopy Datum {datum.id} must have the same number of flux and flux error datapoints" logger.error(msg) raise HermesMessageException(msg) @@ -131,17 +109,15 @@ def get_hermes_spectroscopy(self, datum): spectroscopy_table_row = { 'target_name': datum.target.name, 'date_obs': datum.timestamp.isoformat(), - 'telescope': datum.value.get('telescope'), - 'instrument': datum.value.get('instrument'), - 'reducer': datum.value.get('reducer'), - 'observer': datum.value.get('observer'), - 'flux': flux_list, - 'wavelength': wavelength_list, - 'flux_units': datum.value.get('flux_units'), + 'telescope': datum.telescope, + 'instrument': datum.instrument, + 'flux': datum.flux, + 'wavelength': datum.wavelength, + 'flux_units': datum.flux_unit, 'wavelength_units': convert_astropy_wavelength_to_hermes(datum.value.get('wavelength_units')), } - if flux_error_list: - spectroscopy_table_row['flux_error'] = flux_error_list + if datum.error: + spectroscopy_table_row['flux_error'] = datum.error return spectroscopy_table_row @@ -249,9 +225,9 @@ def create_hermes_alert(message_info, datums, targets=Target.objects.none(), **k for datum in datums: if datum.target.name not in hermes_target_dict: hermes_target_dict[datum.target.name] = hermes_data_converter.get_hermes_target(datum.target) - if datum.data_type == 'photometry': + if isinstance(datum, PhotometryReducedDatum): hermes_photometry_data.append(hermes_data_converter.get_hermes_photometry(datum)) - elif datum.data_type == 'spectroscopy': + elif isinstance(datum, SpectroscopyReducedDatum): hermes_spectroscopy_data.append(hermes_data_converter.get_hermes_spectroscopy(datum)) # Now go through the targets queryset and ensure we have all of them in the table @@ -330,15 +306,23 @@ def hermes_alert_handler(alert, metadata): except ValueError: continue - datum = { - 'target': target, - 'data_type': 'photometry', - 'source_name': alert_as_dict['topic'], - 'source_location': 'Hermes via HOP', # TODO Add message URL here once message ID's exist - 'timestamp': obs_date, - 'value': get_hermes_phot_value(row) - } - new_rd, created = ReducedDatum.objects.get_or_create(**datum) + value = get_hermes_phot_value(row) + new_rd, created = PhotometryReducedDatum.objects.get_or_create( + target=target, + timestamp=obs_date, + bandpass=value.get('filter', ''), + defaults={ + 'brightness': value.get('magnitude', None), + 'brightness_error': value.get('error', None), + 'unit': value.get('unit', None), + 'limit': value.get('limit', None), + 'source_name': alert_as_dict['topic'], + 'source_location': 'Hermes via HOP', # TODO Add message URL here once message ID's exist + 'timestamp': obs_date, + 'telescope': value.get('telescope', ''), + 'instrument': value.get('instrument', ''), + } + ) if created: hermes_alert.save() new_rd.message.add(hermes_alert) diff --git a/tom_dataproducts/api_views.py b/tom_dataproducts/api_views.py index ff2df3729..d875cb402 100644 --- a/tom_dataproducts/api_views.py +++ b/tom_dataproducts/api_views.py @@ -1,4 +1,5 @@ from django.conf import settings +from django.http import Http404 from django_filters import rest_framework as drf_filters from guardian.mixins import PermissionListMixin from guardian.shortcuts import assign_perm, get_objects_for_user @@ -11,11 +12,21 @@ from tom_common.hooks import run_hook from tom_dataproducts.data_processor import run_data_processor from tom_dataproducts.filters import DataProductFilter, ReducedDatumFilter -from tom_dataproducts.models import DataProduct, ReducedDatum +from tom_dataproducts.models import (DataProduct, ReducedDatum, PhotometryReducedDatum, + SpectroscopyReducedDatum, AstrometryReducedDatum, + REDUCED_DATUM_MODELS) from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer from tom_targets.models import Target +# Maps the data_type query param to the concrete model that holds those rows. +_DATA_TYPE_MODEL_MAP = { + 'photometry': PhotometryReducedDatum, + 'spectroscopy': SpectroscopyReducedDatum, + 'astrometry': AstrometryReducedDatum, +} + + class DataProductViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, GenericViewSet, PermissionListMixin): """ Viewset for DataProduct objects. Supports list, create, and delete. @@ -50,7 +61,8 @@ def create(self, request, *args, **kwargs): assign_perm('tom_dataproducts.delete_dataproduct', group, dp) assign_perm('tom_dataproducts.view_reduceddatum', group, reduced_data) except Exception: - ReducedDatum.objects.filter(data_product=dp).delete() + for model in REDUCED_DATUM_MODELS: + model.objects.filter(data_product=dp).delete() dp.delete() return Response({'Data processing error': '''There was an error in processing your DataProduct into \ individual ReducedDatum objects.'''}, @@ -77,6 +89,9 @@ class ReducedDatumViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, G Viewset for ReducedDatum objects. Supports list, create, and delete. To view supported query parameters, please use the OPTIONS endpoint, which can be accessed through the web UI. + + The list endpoint queries all concrete ReducedDatum and returns them in the legacy json format. + TODO: Deprecate the legacy format and have seperate enpoints for each type? """ queryset = ReducedDatum.objects.all() serializer_class = ReducedDatumSerializer @@ -85,6 +100,58 @@ class ReducedDatumViewSet(CreateModelMixin, DestroyModelMixin, ListModelMixin, G permission_required = 'tom_dataproducts.view_reduceddatum' parser_classes = [FormParser, JSONParser] + def get_object(self): + pk = self.kwargs.get(self.lookup_field) + for model in REDUCED_DATUM_MODELS: + try: + obj = model.objects.get(pk=pk) + self.check_object_permissions(self.request, obj) + return obj + except model.DoesNotExist: + pass + raise Http404 + + def _base_queryset_for_model(self, model): + qs = model.objects.all() + if settings.TARGET_PERMISSIONS_ONLY: + qs = qs.filter( + target__in=get_objects_for_user(self.request.user, f'{Target._meta.app_label}.view_target') + ) + return qs + + def list(self, request, *args, **kwargs): + params = request.query_params + requested_data_type = params.get('data_type', '').lower() + + # Determine which models to query + if requested_data_type in _DATA_TYPE_MODEL_MAP: + # A typed data_type + models_to_query = [_DATA_TYPE_MODEL_MAP[requested_data_type]] + elif requested_data_type: + # An unmapped data_type is a generic ReducedDatum + models_to_query = [ReducedDatum] + else: + models_to_query = REDUCED_DATUM_MODELS + + # Strip data_type before passing to ReducedDatumFilter it doesn't exist on concrete types + filter_params = {k: v for k, v in params.items() if k != 'data_type'} + + all_instances = [] + for model in models_to_query: + qs = self._base_queryset_for_model(model) + if model is ReducedDatum and requested_data_type: + qs = qs.filter(data_type=requested_data_type) + qs = ReducedDatumFilter(data=filter_params, queryset=qs).qs + all_instances.extend(list(qs)) + + all_instances.sort(key=lambda x: x.timestamp, reverse=True) + + page = self.paginate_queryset(all_instances) + if page is not None: + return self.get_paginated_response(self.get_serializer(page, many=True).data) + + return Response(self.get_serializer(all_instances, many=True).data) + def create(self, request, *args, **kwargs): response = super().create(request, *args, **kwargs) diff --git a/tom_dataproducts/data_processor.py b/tom_dataproducts/data_processor.py index 1bf7eaac5..36aa6f266 100644 --- a/tom_dataproducts/data_processor.py +++ b/tom_dataproducts/data_processor.py @@ -1,11 +1,10 @@ -import json import logging import mimetypes from django.conf import settings from importlib import import_module -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import try_parse_reduced_datum from tom_targets.sharing import continuous_share_data logger = logging.getLogger(__name__) @@ -26,8 +25,8 @@ def run_data_processor(dp, dp_type_override=None): type from the `dp` object is used. :type dp_type_override: str, optional - :returns: QuerySet of `ReducedDatum` objects created by the `run_data_processor` call - :rtype: `QuerySet` of `ReducedDatum` + :returns: List of typed ReducedDatum objects created by the `run_data_processor` call + :rtype: list """ data_type = dp_type_override or dp.data_product_type try: @@ -43,39 +42,35 @@ def run_data_processor(dp, dp_type_override=None): raise ImportError('Could not import {}. Did you provide the correct path?'.format(processor_class)) data_processor = clazz() - # data returned by process_data is a list of 3-tuples: (timestamp, datum, source) + # 1. data returned by process_data is a list of 3-tuples: (timestamp, datum, source) data = data_processor.process_data(dp) data_type = data_processor.data_type_override() or data_type - # Add only the new (non-duplicate) ReducedDatum objects to the database - - # 1. For quick O(1) lookup, create a hash table of existing ReducedDatum objects - - # Extract exising ReducedDatums for this target, and create a hash table (dict) - # (We make the reduced_dataum.value JSONField dict hashable by converting it to a json string). - # This is so we can do O(1) lookups below as we check for duplicate data. - existing_reduced_datum_values = {json.dumps(rd.value, sort_keys=True, skipkeys=True): 1 - for rd in ReducedDatum.objects.filter(target=dp.target)} - - # 2. Create the list of new ReducedDatum objects (ready for bulk_create) + # 2. Build typed ReducedDatum instances from the raw data. + # try_parse_reduced_datum inspects the data_type and field names to determine the + # correct concrete subclass (photometry, spectroscopy, astrometry, or generic). new_reduced_datums = [] - skipped_data = [] for datum in data: - # Check if the value is already in the ReducedDatum table - # (via lookup in the hash table created above for this purpose) - if json.dumps(datum[1], sort_keys=True, skipkeys=True) in existing_reduced_datum_values: - skipped_data.append(datum) - else: - new_reduced_datums.append( - ReducedDatum(target=dp.target, data_product=dp, data_type=data_type, - timestamp=datum[0], value=datum[1], source_name=datum[2])) - - # prior to checking for duplicates, we created the (yet-to-be-inserted) ReducedDatum list like this: - # reduced_datums = [ReducedDatum(target=dp.target, data_product=dp, data_type=data_type, - # timestamp=datum[0], value=datum[1], source_name=datum[2]) for datum in data] - - # 3. Finally, insert the new ReducedDatum objects into the database - reduced_datums = ReducedDatum.objects.bulk_create(new_reduced_datums) + instance = try_parse_reduced_datum({ + 'target': dp.target, + 'data_product': dp, + 'timestamp': datum[0], + 'source_name': datum[2], + 'data_type': data_type, + **datum[1], + }) + new_reduced_datums.append(instance) + + # 3. bulk_create requires a uniform model type, so group instances by their concrete class. + # Then create them + by_type: dict[type, list] = {} + for instance in new_reduced_datums: + by_type.setdefault(type(instance), []).append(instance) + + reduced_datums = [] + for model_class, instances in by_type.items(): + # ignore_conflicts uses DB level cosntraints for deduplication + reduced_datums.extend(model_class.objects.bulk_create(instances, ignore_conflicts=True)) # 4. Trigger any sharing you may have set to occur when new data comes in # Encapsulate this in a try/catch so sharing failure doesn't prevent dataproduct ingestion @@ -85,12 +80,9 @@ def run_data_processor(dp, dp_type_override=None): logger.warning(f"Failed to share new dataproduct {dp.product_id}: {repr(e)}") # log what happened - if skipped_data: - logger.warning(f'{len(skipped_data)} of {len(data)} skipped as duplicates') - logger.info(f'{len(new_reduced_datums)} of {len(data)} new ReducedDatums ' - f'added for DataProduct: {dp.product_id}') + logger.info(f'{len(reduced_datums)} of {len(data)} new ReducedDatums added for DataProduct: {dp.product_id}') - return ReducedDatum.objects.filter(data_product=dp) + return reduced_datums class DataProcessor(): diff --git a/tom_dataproducts/fields.py b/tom_dataproducts/fields.py new file mode 100644 index 000000000..6806013ea --- /dev/null +++ b/tom_dataproducts/fields.py @@ -0,0 +1,44 @@ +from django.core.exceptions import ValidationError +from django.db import models + + +class FloatArrayField(models.JSONField): + """ + Stores a list of floats as a JSON array. + All values are coerced to float on assignment and validated. + """ + + @staticmethod + def _coerce_to_floats(value): + if not isinstance(value, list): + raise ValidationError("FloatArrayField value must be a list.") + result = [] + for i, item in enumerate(value): + try: + result.append(float(item)) + except (TypeError, ValueError): + raise ValidationError( + f"Element at index {i} cannot be converted to float: {item!r}" + ) + return result + + def from_db_value(self, value, expression, connection): + value = super().from_db_value(value, expression, connection) + if value is None: + return value + return [float(x) for x in value] + + def to_python(self, value): + value = super().to_python(value) + if value is None or value == []: + return value + return self._coerce_to_floats(value) + + def get_prep_value(self, value): + if value is None: + return super().get_prep_value(value) + return super().get_prep_value(self._coerce_to_floats(value)) + + def validate(self, value, model_instance): + super().validate(value, model_instance) + self._coerce_to_floats(value) diff --git a/tom_dataproducts/management/commands/migrateReducedDatums.py b/tom_dataproducts/management/commands/migrateReducedDatums.py new file mode 100644 index 000000000..744cd0a68 --- /dev/null +++ b/tom_dataproducts/management/commands/migrateReducedDatums.py @@ -0,0 +1,141 @@ +from django.core.management.base import BaseCommand + +from tom_dataproducts.models import (ReducedDatum, PhotometryReducedDatum, + SpectroscopyReducedDatum, AstrometryReducedDatum) + +BATCH_SIZE = 1000 + +BRIGHTNESS_FIELDS = ('brightness', 'magnitude', 'mag') +BRIGHTNESS_ERROR_FIELDS = ('brightness_error', 'error', 'magnitude_error', 'mag_error') +BANDPASS_FIELDS = ('bandpass', 'filter', 'band', 'f') + + +def _pop_first(d, keys, default=None): + for key in keys: + if key in d and d[key] is not None: + return d[key] + return default + + +def _run_batched(queryset, build_fn, TypedModel, dry_run, stdout): + to_create = [] + pks_to_delete = [] + total_created = 0 + total_deleted = 0 + + for rd in queryset.iterator(chunk_size=BATCH_SIZE): + instance = build_fn(rd) + to_create.append(instance) + pks_to_delete.append(rd.pk) + + if len(to_create) >= BATCH_SIZE: + if not dry_run: + TypedModel.objects.bulk_create(to_create, ignore_conflicts=True) + ReducedDatum.objects.filter(pk__in=pks_to_delete).delete() + total_created += len(to_create) + total_deleted += len(pks_to_delete) + to_create = [] + pks_to_delete = [] + + if to_create: + if not dry_run: + TypedModel.objects.bulk_create(to_create, ignore_conflicts=True) + ReducedDatum.objects.filter(pk__in=pks_to_delete).delete() + total_created += len(to_create) + total_deleted += len(pks_to_delete) + + label = '[DRY RUN] Would migrate' if dry_run else 'Migrated' + stdout.write(f' {label} {total_created} {TypedModel.__name__} rows, deleted {total_deleted} ReducedDatum rows.') + + +def _build_photometry(rd): + value = rd.value or {} + return PhotometryReducedDatum( + target_id=rd.target_id, + data_product_id=rd.data_product_id, + source_name=rd.source_name, + source_location=rd.source_location, + timestamp=rd.timestamp, + telescope=rd.telescope or value.get('telescope', ''), + instrument=rd.instrument or value.get('instrument', ''), + brightness=_pop_first(value, BRIGHTNESS_FIELDS), + brightness_error=_pop_first(value, BRIGHTNESS_ERROR_FIELDS), + limit=value.get('limit'), + unit=value.get('unit', ''), + bandpass=_pop_first(value, BANDPASS_FIELDS, default=''), + ) + + +def _build_spectroscopy(rd): + value = rd.value or {} + return SpectroscopyReducedDatum( + target_id=rd.target_id, + data_product_id=rd.data_product_id, + source_name=rd.source_name, + source_location=rd.source_location, + timestamp=rd.timestamp, + telescope=rd.telescope or value.get('telescope', ''), + instrument=rd.instrument or value.get('instrument', ''), + flux=value.get('flux', []), + wavelength=value.get('wavelength', []), + error=value.get('error', value.get('flux_error', [])), + flux_unit=value.get('flux_units', value.get('flux_unit', '')), + ) + + +def _build_astrometry(rd): + value = rd.value or {} + return AstrometryReducedDatum( + target_id=rd.target_id, + data_product_id=rd.data_product_id, + source_name=rd.source_name, + source_location=rd.source_location, + timestamp=rd.timestamp, + telescope=rd.telescope or value.get('telescope', ''), + instrument=rd.instrument or value.get('instrument', ''), + ra=value.get('ra'), + dec=value.get('dec'), + ra_error=value.get('ra_error'), + dec_error=value.get('dec_error'), + ra_error_units=value.get('ra_error_units', ''), + dec_error_units=value.get('dec_error_units', ''), + ) + + +class Command(BaseCommand): + help = """ + Migrates generic ReducedDatum rows into their concrete typed models + (PhotometryReducedDatum, SpectroscopyReducedDatum, AstrometryReducedDatum) + and deletes the originals. Run this once after deploying the reduceddatum refactor. + Only necessary for TOMs that existed prior to v3. + """ + + def add_arguments(self, parser): + parser.add_argument( + '--dry-run', + action='store_true', + help='Report what would be migrated without writing any changes.', + ) + + def handle(self, *args, **options): + dry_run = options['dry_run'] + + if dry_run: + self.stdout.write('Dry run — no changes will be written.\n') + + steps = [ + ('photometry', ReducedDatum.objects.filter(data_type='photometry'), + _build_photometry, PhotometryReducedDatum), + ('spectroscopy', ReducedDatum.objects.filter(data_type='spectroscopy'), + _build_spectroscopy, SpectroscopyReducedDatum), + ('astrometry', ReducedDatum.objects.filter(data_type='astrometry'), + _build_astrometry, AstrometryReducedDatum), + ] + + for name, queryset, build_fn, TypedModel in steps: + count = queryset.count() + self.stdout.write(f'{name}: {count} rows to migrate.') + if count: + _run_batched(queryset, build_fn, TypedModel, dry_run, self.stdout) + + self.stdout.write(self.style.SUCCESS('Done.')) diff --git a/tom_dataproducts/management/commands/updatereduceddata.py b/tom_dataproducts/management/commands/updatereduceddata.py index dbaa6b521..34ae98099 100644 --- a/tom_dataproducts/management/commands/updatereduceddata.py +++ b/tom_dataproducts/management/commands/updatereduceddata.py @@ -4,7 +4,7 @@ from tom_alerts import alerts from tom_targets.models import Target -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import REDUCED_DATUM_MODELS class Command(BaseCommand): @@ -21,18 +21,28 @@ def handle(self, *args, **options): for broker in brokers: broker_classes[broker] = alerts.get_service_class(broker)() - target = None - sources = [s.source_name for s in ReducedDatum.objects.filter(source_name__in=broker_classes.keys()).distinct()] + all_source_names = set() + for model in REDUCED_DATUM_MODELS: + all_source_names.update( + model.objects.filter(source_name__in=broker_classes.keys()) + .values_list('source_name', flat=True).distinct() + ) + sources = list(all_source_names) + + all_target_ids = set() + for model in REDUCED_DATUM_MODELS: + all_target_ids.update( + model.objects.filter(source_name__in=sources) + .values_list('target', flat=True).distinct() + ) + if options['target_id']: try: targets = [Target.objects.get(pk=options['target_id'])] except ObjectDoesNotExist: raise Exception('Invalid target id provided') else: - targets = Target.objects.filter( - id__in=ReducedDatum.objects.filter( - source_name__in=sources - ).values_list('target').distinct()) + targets = Target.objects.filter(id__in=all_target_ids) failed_records = {} for target in targets: diff --git a/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py b/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py new file mode 100644 index 000000000..8b27e2201 --- /dev/null +++ b/tom_dataproducts/migrations/0015_reduceddatum_instrument_reduceddatum_telescope_and_more.py @@ -0,0 +1,105 @@ +# Generated by Django 5.2.13 on 2026-04-16 21:38 + +import django.db.models.deletion +import django.utils.timezone +import tom_dataproducts.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tom_alerts', '0007_alter_alertstreammessage_message_id'), + ('tom_dataproducts', '0014_alter_reduceddatum_timestamp'), + ('tom_targets', '0030_alter_basetarget_slope'), + ] + + operations = [ + migrations.AddField( + model_name='reduceddatum', + name='instrument', + field=models.CharField(blank=True, default='', max_length=255), + ), + migrations.AddField( + model_name='reduceddatum', + name='telescope', + field=models.CharField(blank=True, default='', max_length=255), + ), + migrations.AlterField( + model_name='reduceddatum', + name='value', + field=models.JSONField(blank=True, default=dict, verbose_name='extra data'), + ), + migrations.CreateModel( + name='AstrometryReducedDatum', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now)), + ('value', models.JSONField(blank=True, default=dict, verbose_name='extra data')), + ('telescope', models.CharField(blank=True, default='', max_length=255)), + ('instrument', models.CharField(blank=True, default='', max_length=255)), + ('source_name', models.CharField(blank=True, default='', max_length=100)), + ('source_location', models.CharField(blank=True, default='', max_length=200)), + ('ra', models.FloatField()), + ('dec', models.FloatField()), + ('ra_error', models.FloatField(blank=True, default=None, null=True)), + ('dec_error', models.FloatField(blank=True, default=None, null=True)), + ('ra_error_units', models.CharField(blank=True, default='', max_length=32)), + ('dec_error_units', models.CharField(blank=True, default='', max_length=32)), + ('data_product', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tom_dataproducts.dataproduct')), + ('message', models.ManyToManyField(blank=True, to='tom_alerts.alertstreammessage')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tom_targets.basetarget')), + ], + options={ + 'constraints': [models.UniqueConstraint(fields=('target', 'timestamp', 'telescope', 'instrument'), name='unique_astrometry')], + }, + ), + migrations.CreateModel( + name='PhotometryReducedDatum', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now)), + ('value', models.JSONField(blank=True, default=dict, verbose_name='extra data')), + ('telescope', models.CharField(blank=True, default='', max_length=255)), + ('instrument', models.CharField(blank=True, default='', max_length=255)), + ('source_name', models.CharField(blank=True, default='', max_length=100)), + ('source_location', models.CharField(blank=True, default='', max_length=200)), + ('brightness', models.FloatField(blank=True, null=True)), + ('brightness_error', models.FloatField(blank=True, null=True)), + ('limit', models.FloatField(blank=True, null=True)), + ('unit', models.CharField(blank=True, default='', max_length=32)), + ('bandpass', models.CharField(max_length=32)), + ('exposure_time', models.FloatField(blank=True, null=True)), + ('data_product', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tom_dataproducts.dataproduct')), + ('message', models.ManyToManyField(blank=True, to='tom_alerts.alertstreammessage')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tom_targets.basetarget')), + ], + options={ + 'constraints': [models.UniqueConstraint(fields=('target', 'bandpass', 'timestamp'), name='unique_photometry')], + }, + ), + migrations.CreateModel( + name='SpectroscopyReducedDatum', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(db_index=True, default=django.utils.timezone.now)), + ('value', models.JSONField(blank=True, default=dict, verbose_name='extra data')), + ('telescope', models.CharField(blank=True, default='', max_length=255)), + ('instrument', models.CharField(blank=True, default='', max_length=255)), + ('source_name', models.CharField(blank=True, default='', max_length=100)), + ('source_location', models.CharField(blank=True, default='', max_length=200)), + ('setup', models.CharField(blank=True, default='', max_length=2000)), + ('exposure_time', models.FloatField(blank=True, null=True)), + ('wavelength', tom_dataproducts.fields.FloatArrayField(blank=True, default=list)), + ('flux', tom_dataproducts.fields.FloatArrayField(blank=True, default=list)), + ('error', tom_dataproducts.fields.FloatArrayField(blank=True, default=list)), + ('flux_unit', models.TextField(blank=True, default='')), + ('data_product', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tom_dataproducts.dataproduct')), + ('message', models.ManyToManyField(blank=True, to='tom_alerts.alertstreammessage')), + ('target', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='tom_targets.basetarget')), + ], + options={ + 'constraints': [models.UniqueConstraint(fields=('target', 'timestamp', 'telescope', 'instrument'), name='unique_spectroscopy')], + }, + ), + ] diff --git a/tom_dataproducts/models.py b/tom_dataproducts/models.py index 1419a917d..371681deb 100644 --- a/tom_dataproducts/models.py +++ b/tom_dataproducts/models.py @@ -1,20 +1,22 @@ import logging import os import tempfile +from importlib import import_module from astropy.io import fits from django.conf import settings +from django.core.exceptions import ValidationError from django.core.files import File from django.db import models -from django.utils import timezone, text -from django.core.exceptions import ValidationError +from django.utils import text, timezone from fits2image.conversions import fits_to_jpg from PIL import Image -from importlib import import_module -from tom_targets.base_models import BaseTarget from tom_alerts.models import AlertStreamMessage from tom_observations.models import ObservationRecord +from tom_targets.base_models import BaseTarget + +from .fields import FloatArrayField logger = logging.getLogger(__name__) @@ -335,55 +337,40 @@ def get_queryset(self): return ReducedDatumQuerySet(self.model) -class ReducedDatum(models.Model): +class ReducedDatumCommon(models.Model): """ - Class representing a datum in a TOM. + Abstract base class for all reduced datum models. A ``ReducedDatum`` generally refers to a single piece of data--e.g., a spectrum, or a photometry point. It is associated with a target, and optionally with the data product it came from. An example of a ``ReducedDatum`` - without an associated data product would be photometry ingested from a broker. + without an associated data product would be photometry ingested from a broker. There are + concrete implementations of Photometry, Spectroscopy and Astronmetry ReducedDatum models. :param target: The ``Target`` with which this object is associated. :param data_product: The ``DataProduct`` with which this object is optionally associated. - :param data_type: The type of data this datum represents. Default choices are the default values found in - DATA_PRODUCT_TYPES in settings.py. - :type data_type: str - - :param source_name: The original source of this datum. The current major use of this field is to track the broker a - datum came from, but can be used for other sources. - :type source_name: str - - :param source_location: A reference to the location that this datum was originally sourced from. The current major - use of this field is the URL path to the alert that this datum came from. - :type source_name: str - :param timestamp: The timestamp of this datum. :type timestamp: datetime - :param value: The value of the datum. This is a dict, intended to store data with a variety of - scopes. As an example, a photometry value might contain the following: + :param value: Freeform data. This is a dict, intended to store extra data with a variety of + scopes. As an example, one might want to store the originating survey: - :: + :: { - 'magnitude': 18.5, - 'error': .5 + 'survey': 'lsst', } - but could also contain a filter, a telescope, an instrument, and/or a unit: + :type value: dict - :: + :param source_name: The original source of this datum. The current major use of this field is to track the broker a + datum came from, but can be used for other sources. + :type source_name: str - { - 'magnitude': 18.5, - 'error': .5, - 'filter': 'r', - 'telescope': 'ELP.domeA.1m0a', - 'instrument': 'fa07', - } - :type value: dict + :param source_location: A reference to the location that this datum was originally sourced from. The current major + use of this field is the URL path to the alert that this datum came from. + :type source_location: str :param message: Set of ``AlertStreamMessage`` objects this object is associated with. :type message: ManyRelatedManager object @@ -391,34 +378,40 @@ class ReducedDatum(models.Model): """ target = models.ForeignKey(BaseTarget, null=False, on_delete=models.CASCADE) - data_product = models.ForeignKey(DataProduct, null=True, blank=True, on_delete=models.CASCADE) - data_type = models.CharField( - max_length=100, - default='' + data_product = models.ForeignKey( + DataProduct, null=True, blank=True, on_delete=models.CASCADE + ) + timestamp = models.DateTimeField( + null=False, blank=False, default=timezone.now, db_index=True + ) + value = models.JSONField( + null=False, blank=True, default=dict, verbose_name="extra data" ) - source_name = models.CharField(max_length=100, default='', blank=True) - source_location = models.CharField(max_length=200, default='', blank=True) - timestamp = models.DateTimeField(null=False, blank=False, default=timezone.now, db_index=True) - value = models.JSONField(null=False, blank=False) + telescope = models.CharField(max_length=255, blank=True, default="") + instrument = models.CharField(max_length=255, blank=True, default="") + source_name = models.CharField(max_length=100, default="", blank=True) + source_location = models.CharField(max_length=200, default="", blank=True) message = models.ManyToManyField(AlertStreamMessage, blank=True) objects = ReducedDatumManager() class Meta: - get_latest_by = ('timestamp',) + abstract = True - def save(self, *args, **kwargs): - # Validate data_type based on options in settings.py or default types: (type, display) - for dp_type, _ in DATA_TYPE_CHOICES: - if self.data_type and self.data_type == dp_type: - break - else: - raise ValidationError('Not a valid DataProduct type.') - # because we have a custom way of validating the uniqueness of the ReducedDatum, - # we need to call full_clean() here to invoke our validate_unique() method. - self.full_clean() - return super().save() +class ReducedDatum(ReducedDatumCommon): + """ + Class representing a generic datum in a TOM that isn't represented by any of the existing data types. + + :param data_type: The type of data this datum represents. Default choices are the default values found in + DATA_PRODUCT_TYPES in settings.py. + :type data_type: str + """ + + data_type = models.CharField(max_length=100, default="") + + class Meta: + get_latest_by = ("timestamp",) def validate_unique(self, *args, **kwargs): """ @@ -426,22 +419,225 @@ def validate_unique(self, *args, **kwargs): on standard validation. Also, We do not want to repeat identical data from two different sources. Do nothing if the uniqueness test passes. Otherwise, raise a ValidationError. - see https://docs.djangoproject.com/en/5.0/ref/models/instances/#validating-objects """ super().validate_unique(*args, **kwargs) - # Check if the Reduced Datum exists in the database try: - existing_reduced_datum = ReducedDatum.objects.get(target=self.target, - data_type=self.data_type, - timestamp=self.timestamp, - value=self.value) - if existing_reduced_datum and existing_reduced_datum.id != self.id: # not the same object - existing_source = existing_reduced_datum.__dict__.get('source_name', 'Unknown Source') + existing_reduced_datum = ReducedDatum.objects.get( + target=self.target, + data_type=self.data_type, + timestamp=self.timestamp, + value=self.value, + ) + if ( + existing_reduced_datum and existing_reduced_datum.id != self.id + ): # not the same object + existing_source = existing_reduced_datum.__dict__.get( + "source_name", "Unknown Source" + ) # found ReducedDatum with the same values. Don't save this duplicate ReducedDatum. - raise ValidationError(f'ReducedDatum already exists: Identical {self.data_type} data ' - f'found for {self.target} from {existing_source}.') + raise ValidationError( + f"ReducedDatum already exists: Identical {self.data_type} data " + f"found for {self.target} from {existing_source}." + ) except ReducedDatum.DoesNotExist: # this means that our check for uniqueness passed: so do not raise ValidationError pass + + +class PhotometryReducedDatum(ReducedDatumCommon): + brightness = models.FloatField(blank=True, null=True) + brightness_error = models.FloatField(blank=True, null=True) + limit = models.FloatField(blank=True, null=True) + unit = models.CharField(max_length=32, blank=True, default="") + bandpass = models.CharField(max_length=32) + exposure_time = models.FloatField(blank=True, null=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["target", "bandpass", "timestamp"], name="unique_photometry" + ) + ] + + +class SpectroscopyReducedDatum(ReducedDatumCommon): + setup = models.CharField(max_length=2000, blank=True, default="") + exposure_time = models.FloatField(blank=True, null=True) + wavelength = FloatArrayField(blank=True, default=list) + flux = FloatArrayField(blank=True, default=list) + error = FloatArrayField(blank=True, default=list) + flux_unit = models.TextField(blank=True, default="") + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["target", "timestamp", "telescope", "instrument"], + name="unique_spectroscopy", + ) + ] + + +class AstrometryReducedDatum(ReducedDatumCommon): + ra = models.FloatField() + dec = models.FloatField() + ra_error = models.FloatField(null=True, blank=True, default=None) + dec_error = models.FloatField(null=True, blank=True, default=None) + ra_error_units = models.CharField(max_length=32, blank=True, default="") + dec_error_units = models.CharField(max_length=32, blank=True, default="") + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["target", "timestamp", "telescope", "instrument"], + name="unique_astrometry", + ) + ] + + +REDUCED_DATUM_MODELS = ( + ReducedDatum, + PhotometryReducedDatum, + SpectroscopyReducedDatum, + AstrometryReducedDatum, +) + + +def _pop_find_field(possible_fields: set, value: dict): + """ + Helper function to find a field in a dict given a set of possible field names. + Pops the value of the first found field, or returns None if no field is found. + """ + for field in possible_fields: + if field in value: + return value.pop(field) + + return None + + +def _extract_extra_fields(data: dict, model: type[models.Model]) -> dict: + """ + Helper function to extract fields from the value dict that are not part of the model. + Pops the extra fields from the value dict and returns them as a new dict to be saved + in the models `value` field. + """ + model_fields = set(f.name for f in model._meta.get_fields()) + extra_fields = {} + for field in list(data.keys()): + if field not in model_fields: + extra_fields[field] = data.pop(field) + + return extra_fields + + +def _build_photometry_reduced_datum(data: dict) -> PhotometryReducedDatum: + BRIGHTNESS_FIELDS = {"brightness", "magnitude", "mag"} + BRIGHTNESS_ERROR_FIELDS = { + "error", + "brightness_error", + "magnitude_error", + "mag_err", + } + BANDPASS_FIELDS = {"bandpass", "filter", "band", "f"} + + brightness = _pop_find_field(BRIGHTNESS_FIELDS, data) + brightness_error = _pop_find_field(BRIGHTNESS_ERROR_FIELDS, data) + bandpass = _pop_find_field(BANDPASS_FIELDS, data) or "" + + extra_fields = _extract_extra_fields(data, PhotometryReducedDatum) + + return PhotometryReducedDatum( + brightness=brightness, + brightness_error=brightness_error, + bandpass=bandpass, + value=extra_fields, + **data, + ) + + +def _build_spectroscopy_reduced_datum(data: dict) -> SpectroscopyReducedDatum: + FLUX_FIELDS = {"flux", "f"} + WAVELENGTH_FIELDS = {"wavelength", "wave", "wl"} + ERROR_FIELDS = {"error", "err", "flux_error", "f_error"} + FLUX_UNIT_FIELDS = {"flux_unit", "f_unit", "flux_units", "f_units"} + + flux = _pop_find_field(FLUX_FIELDS, data) or [] + wavelength = _pop_find_field(WAVELENGTH_FIELDS, data) or [] + error = _pop_find_field(ERROR_FIELDS, data) or [] + flux_unit = _pop_find_field(FLUX_UNIT_FIELDS, data) or "" + + extra_fields = _extract_extra_fields(data, SpectroscopyReducedDatum) + + return SpectroscopyReducedDatum( + wavelength=wavelength, + flux=flux, + error=error, + flux_unit=flux_unit, + value=extra_fields, + **data, + ) + + +def _build_astrometry_reduced_datum(data: dict) -> AstrometryReducedDatum: + RA_FIELDS = {"ra", "right_ascension", "ra_deg"} + DEC_FIELDS = {"dec", "declination", "dec_deg"} + RA_ERROR_FIELDS = {"ra_error", "right_ascension_error", "ra_err"} + DEC_ERROR_FIELDS = {"dec_error", "declination_error", "dec_err"} + RA_ERROR_UNIT_FIELDS = {"ra_error_units", "ra_err_units"} + DEC_ERROR_UNIT_FIELDS = {"dec_error_units", "dec_err_units"} + + ra = _pop_find_field(RA_FIELDS, data) + dec = _pop_find_field(DEC_FIELDS, data) + ra_error = _pop_find_field(RA_ERROR_FIELDS, data) + dec_error = _pop_find_field(DEC_ERROR_FIELDS, data) + ra_error_units = _pop_find_field(RA_ERROR_UNIT_FIELDS, data) or "" + dec_error_units = _pop_find_field(DEC_ERROR_UNIT_FIELDS, data) or "" + + extra_fields = _extract_extra_fields(data, AstrometryReducedDatum) + + return AstrometryReducedDatum( + ra=ra, + dec=dec, + ra_error=ra_error, + dec_error=dec_error, + ra_error_units=ra_error_units, + dec_error_units=dec_error_units, + value=extra_fields, + **data, + ) + + +def _build_generic_reduced_datum(data: dict, data_type: str) -> ReducedDatum: + extra_fields = _extract_extra_fields(data, ReducedDatum) + + return ReducedDatum(value=extra_fields, data_type=data_type, **data) + + +def try_parse_reduced_datum( + data: dict, +) -> ( + ReducedDatum + | PhotometryReducedDatum + | SpectroscopyReducedDatum + | AstrometryReducedDatum +): + """ + Accepts unstructured data and attempts to create the correct ReducedDatum sublcass. + If the heuristics fail, returns a generic ReducedDatum. + """ + if data.get("value") and isinstance(data["value"], dict): + # `value` is the existing free-form field. Pull those values to the top of the dict + # so we can use them to determine which reduced datum type to create + value = data.pop("value") + data = {**data, **value} + + match data_type := data.pop("data_type", "").lower(): + case "photometry": + return _build_photometry_reduced_datum(data) + case "spectroscopy": + return _build_spectroscopy_reduced_datum(data) + case "astrometry": + return _build_astrometry_reduced_datum(data) + case _: + return _build_generic_reduced_datum(data, data_type) diff --git a/tom_dataproducts/serializers.py b/tom_dataproducts/serializers.py index 7c1d92c0f..87e32fbf9 100644 --- a/tom_dataproducts/serializers.py +++ b/tom_dataproducts/serializers.py @@ -5,6 +5,8 @@ from tom_common.serializers import GroupSerializer from tom_dataproducts.models import DataProductGroup, DataProduct, ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum, try_parse_reduced_datum +from tom_dataproducts.models import SpectroscopyReducedDatum, AstrometryReducedDatum from tom_observations.models import ObservationRecord from tom_observations.serializers import ObservationRecordFilteredPrimaryKeyRelatedField from tom_targets.models import Target @@ -32,13 +34,65 @@ class Meta: 'target' ) + def to_representation(self, instance): + if isinstance(instance, (PhotometryReducedDatum, SpectroscopyReducedDatum, AstrometryReducedDatum)): + return { + 'data_product': None, + 'data_type': self._get_data_type(instance), + 'source_name': instance.source_name, + 'source_location': instance.source_location, + 'timestamp': self.fields['timestamp'].to_representation(instance.timestamp), + 'value': self._get_typed_value(instance), + 'target': instance.target_id, + } + return super().to_representation(instance) + + def _get_data_type(self, instance): + if isinstance(instance, PhotometryReducedDatum): + return 'photometry' + if isinstance(instance, SpectroscopyReducedDatum): + return 'spectroscopy' + if isinstance(instance, AstrometryReducedDatum): + return 'astrometry' + + def _get_typed_value(self, instance): + if isinstance(instance, PhotometryReducedDatum): + return { + 'brightness': instance.brightness, + 'brightness_error': instance.brightness_error, + 'bandpass': instance.bandpass, + 'unit': instance.unit, + 'telescope': instance.telescope, + 'instrument': instance.instrument, + } + if isinstance(instance, SpectroscopyReducedDatum): + return { + 'flux': instance.flux, + 'wavelength': instance.wavelength, + 'error': instance.error, + 'flux_unit': instance.flux_unit, + 'telescope': instance.telescope, + 'instrument': instance.instrument, + } + if isinstance(instance, AstrometryReducedDatum): + return { + 'ra': instance.ra, + 'dec': instance.dec, + 'ra_error': instance.ra_error, + 'dec_error': instance.dec_error, + 'ra_error_units': instance.ra_error_units, + 'dec_error_units': instance.dec_error_units, + 'telescope': instance.telescope, + 'instrument': instance.instrument, + } + def create(self, validated_data): """DRF requires explicitly handling writeable nested serializers, here we pop the groups data and save it using its serializer. """ groups = validated_data.pop('groups', []) - rd = ReducedDatum(**validated_data) + rd = try_parse_reduced_datum(validated_data) rd.full_clean() rd.save() diff --git a/tom_dataproducts/sharing.py b/tom_dataproducts/sharing.py index 58ce87088..6067adfcc 100644 --- a/tom_dataproducts/sharing.py +++ b/tom_dataproducts/sharing.py @@ -13,7 +13,7 @@ from tom_targets.models import Target -from tom_dataproducts.models import DataProduct, ReducedDatum +from tom_dataproducts.models import DataProduct, PhotometryReducedDatum, REDUCED_DATUM_MODELS from tom_dataproducts.alertstreams.hermes import publish_to_hermes, BuildHermesMessage, get_hermes_topics from tom_dataproducts.serializers import DataProductSerializer, ReducedDatumSerializer @@ -33,9 +33,9 @@ def share_target_list_with_hermes(share_destination, form_data, selected_targets title_name = f"{target_list.name} target list" targets = Target.objects.filter(id__in=selected_targets) if include_all_data: - reduced_datums = ReducedDatum.objects.filter(target__id__in=selected_targets, data_type='photometry') + reduced_datums = PhotometryReducedDatum.objects.filter(target__id__in=selected_targets) else: - reduced_datums = ReducedDatum.objects.none() + reduced_datums = PhotometryReducedDatum.objects.none() return _share_with_hermes(share_destination, form_data, title_name, reduced_datums, targets) @@ -50,29 +50,27 @@ def share_data_with_hermes(share_destination, form_data, product_id=None, target :return: json response for the sharing """ # Query relevant Reduced Datums Queryset - accepted_data_types = ['photometry', 'spectroscopy'] if product_id: product = DataProduct.objects.get(pk=product_id) target = product.target - reduced_datums = ReducedDatum.objects.filter(data_product=product) + reduced_datums = PhotometryReducedDatum.objects.filter(data_product=product) elif selected_data: - reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) + reduced_datums = PhotometryReducedDatum.objects.filter(pk__in=selected_data) target = reduced_datums[0].target elif target_id: target = Target.objects.get(pk=target_id) - reduced_datums = ReducedDatum.objects.none() + reduced_datums = PhotometryReducedDatum.objects.none() else: - reduced_datums = ReducedDatum.objects.none() + reduced_datums = PhotometryReducedDatum.objects.none() target = Target.objects.none() title_name = target.name if target else '' - reduced_datums.filter(data_type__in=accepted_data_types) return _share_with_hermes( share_destination, form_data, title_name, reduced_datums, targets=Target.objects.filter(pk=target.pk) ) def _share_with_hermes(share_destination, form_data, title_name, - reduced_datums=ReducedDatum.objects.none(), + reduced_datums=PhotometryReducedDatum.objects.none(), targets=Target.objects.none()): """ Helper method to serialize and share data with hermes @@ -127,7 +125,7 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id dataproducts_url = destination_tom_base_url + 'api/dataproducts/' targets_url = destination_tom_base_url + 'api/targets/' reduced_datums_url = destination_tom_base_url + 'api/reduceddatums/' - reduced_datums = ReducedDatum.objects.none() + reduced_datums = PhotometryReducedDatum.objects.none() # If a DataProduct is provided, share that DataProduct if product_id: @@ -151,7 +149,7 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id elif selected_data or target_id: # If ReducedDatums are provided, share those ReducedDatums if selected_data: - reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) + reduced_datums = PhotometryReducedDatum.objects.filter(pk__in=selected_data) targets = set(reduced_datum.target for reduced_datum in reduced_datums) target_dict = {} for target in targets: @@ -166,7 +164,9 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id # If Target is provided, share all ReducedDatums for that Target # (Will not create New Target in Destination TOM) target = Target.objects.get(pk=target_id) - reduced_datums = ReducedDatum.objects.filter(target=target) + reduced_datums = [] + for model in REDUCED_DATUM_MODELS: + reduced_datums.extend(model.objects.filter(target=target)) destination_target_id, _ = get_destination_target(target, targets_url, headers, auth) if destination_target_id is None: return {'message': 'ERROR: No matching target found.'} @@ -181,7 +181,6 @@ def share_data_with_tom(share_destination, form_data, product_id=None, target_id if target_dict[datum.target.name]: serialized_data = ReducedDatumSerializer(datum).data serialized_data['target'] = target_dict[datum.target.name] - serialized_data['data_product'] = '' if not serialized_data['source_name']: serialized_data['source_name'] = settings.TOM_NAME serialized_data['source_location'] = f"ReducedDatum shared from " \ @@ -313,35 +312,25 @@ def sharing_feedback_handler(response, request): return -def process_spectro_data_for_download(serialized_datum): - """ Turns a serialized spectrograph datum into a list of serialized datums with the - spectrograph info expanded one piece per line +def process_spectro_data_for_download(datum): + """ Turns a SpectroscopyReducedDatum into a list of row dicts with the spectrum + expanded to one row per wavelength/flux point. """ download_datums = [] - spectra_data = serialized_datum.pop('value') - if ('flux' in spectra_data and isinstance(spectra_data['flux'], list) - and 'wavelength' in spectra_data and isinstance(spectra_data['wavelength'], list) - and len(spectra_data['flux']) == len(spectra_data['wavelength'])): - datum_to_copy = serialized_datum.copy() - # If its a data dict with certain array or dict fields, then first copy the scalar fields over - for key, value in spectra_data.items(): - if not isinstance(value, (list, dict)) and key not in datum_to_copy: - datum_to_copy[key] = value - # And then iterate over the expected array fields to build output rows - for i, flux in enumerate(spectra_data['flux']): - expanded_datum = datum_to_copy.copy() - expanded_datum['flux'] = flux - expanded_datum['wavelength'] = spectra_data['wavelength'][i] - if 'flux_error' in spectra_data and isinstance(spectra_data['flux_error'], list): - expanded_datum['flux_error'] = spectra_data['flux_error'][i] - download_datums.append(expanded_datum) - else: - for entry in spectra_data.values(): - if isinstance(entry, dict): - expanded_datum = serialized_datum.copy() - # If its an "array" of dicts, just expand each dict into the output - expanded_datum.update(entry) - download_datums.append(expanded_datum) + if datum.flux and datum.wavelength and len(datum.flux) == len(datum.wavelength): + scalar_fields = { + 'timestamp': datum.timestamp, + 'telescope': datum.telescope, + 'instrument': datum.instrument, + 'setup': datum.setup, + 'flux_unit': datum.flux_unit, + 'source_name': datum.source_name, + } + for i, (wavelength, flux) in enumerate(zip(datum.wavelength, datum.flux)): + row = {**scalar_fields, 'wavelength': wavelength, 'flux': flux} + if datum.error and i < len(datum.error): + row['flux_error'] = datum.error[i] + download_datums.append(row) return download_datums @@ -354,18 +343,24 @@ def download_data(form_data, selected_data): :param selected_data: ReducucedDatums selected via the checkboxes in the DataShareForm :return: CSV photometry or spectroscopy table as a StreamingHttpResponse """ - reduced_datums = ReducedDatum.objects.filter(pk__in=selected_data) - serialized_data = [ReducedDatumSerializer(rd).data for rd in reduced_datums] + # TODO: selected_data can only contain photometry PKs as of now. the share-box checkboxes are + # only rendered in photometry_datalist_for_target.html. + # There is no spectroscopy share UI. + phot_datums = PhotometryReducedDatum.objects.filter(pk__in=selected_data) data_to_save = [] sort_fields = ['timestamp'] - for datum in serialized_data: - if datum.get('data_type') == 'photometry': - datum.update(datum.pop('value')) - data_to_save.append(datum) - elif datum.get('data_type') == 'spectroscopy': - sort_fields = ['timestamp', 'wavelength'] - # Attempt to expand the photometry table stored in the .value into multiple entries in serialized data - data_to_save.extend(process_spectro_data_for_download(datum)) + for datum in phot_datums: + data_to_save.append({ + 'timestamp': datum.timestamp, + 'telescope': datum.telescope, + 'instrument': datum.instrument, + 'bandpass': datum.bandpass, + 'brightness': datum.brightness, + 'brightness_error': datum.brightness_error, + 'limit': datum.limit, + 'unit': datum.unit, + 'source_name': datum.source_name, + }) table = Table(data_to_save) if form_data.get('share_message'): table.meta['comments'] = [form_data['share_message']] diff --git a/tom_dataproducts/templatetags/dataproduct_extras.py b/tom_dataproducts/templatetags/dataproduct_extras.py index a7085eb09..b1999de20 100644 --- a/tom_dataproducts/templatetags/dataproduct_extras.py +++ b/tom_dataproducts/templatetags/dataproduct_extras.py @@ -1,4 +1,5 @@ import logging +from collections import defaultdict from urllib.parse import urlencode from django import template @@ -18,8 +19,7 @@ import numpy as np from tom_dataproducts.forms import DataProductUploadForm, DataShareForm -from tom_dataproducts.models import DataProduct, ReducedDatum -from tom_dataproducts.processors.data_serializers import SpectrumSerializer +from tom_dataproducts.models import DataProduct, PhotometryReducedDatum, SpectroscopyReducedDatum from tom_dataproducts.single_target_data_service.single_target_data_service import get_service_classes, \ get_service_class from tom_observations.models import ObservationRecord @@ -134,12 +134,7 @@ def recent_photometry(target, limit=1): """ Displays a table of the most recent photometric points for a target. """ - photometry = ReducedDatum.objects.filter(data_type='photometry', target=target).order_by('-timestamp')[:limit] - - # Possibilities for reduced_datums from ZTF/MARS: - # reduced_datum.value: {'error': 0.0929680392146111, 'filter': 'r', 'magnitude': 18.2364940643311} - # reduced_datum.value: {'limit': 20.1023998260498, 'filter': 'g'} - + photometry = PhotometryReducedDatum.objects.filter(target=target).order_by('-timestamp')[:limit] # for limit magnitudes, set the value of the limit key to True and # the value of the magnitude key to the limit so the template and # treat magnitudes as such and prepend a '>' to the limit magnitudes @@ -147,11 +142,11 @@ def recent_photometry(target, limit=1): data = [] for reduced_datum in photometry: rd_data = {'timestamp': reduced_datum.timestamp} - if 'limit' in reduced_datum.value.keys(): - rd_data['magnitude'] = reduced_datum.value['limit'] + if reduced_datum.limit is not None: + rd_data['magnitude'] = reduced_datum.limit rd_data['limit'] = True else: - rd_data['magnitude'] = reduced_datum.value['magnitude'] + rd_data['magnitude'] = reduced_datum.brightness rd_data['limit'] = False data.append(rd_data) @@ -164,38 +159,31 @@ def get_photometry_data(context, target, target_share=False): """ Displays a table of the all photometric points for a target. """ - photometry = ReducedDatum.objects.filter(data_type='photometry', target=target).order_by('-timestamp') + photometry = PhotometryReducedDatum.objects.filter(target=target).order_by('-timestamp') if not settings.TARGET_PERMISSIONS_ONLY: photometry = get_objects_for_user( context["request"].user, - "tom_dataproducts.view_reduceddatum", + "tom_dataproducts.view_photometryreduceddatum", klass=photometry, ) - # Possibilities for reduced_datums from ZTF/MARS: - # reduced_datum.value: {'error': 0.0929680392146111, 'filter': 'r', 'magnitude': 18.2364940643311} - # reduced_datum.value: {'limit': 20.1023998260498, 'filter': 'g'} - - # for limit magnitudes, set the value of the limit key to True and - # the value of the magnitude key to the limit so the template and - # treat magnitudes as such and prepend a '>' to the limit magnitudes - # see recent_photometry.html + # Non detections have limit set and brightness null, detections are the reverse. data = [] for reduced_datum in photometry: rd_data = {'id': reduced_datum.pk, 'timestamp': reduced_datum.timestamp, 'source': reduced_datum.source_name, - 'filter': reduced_datum.value.get('filter', ''), - 'telescope': reduced_datum.value.get('telescope', ''), - 'error': reduced_datum.value.get('error', reduced_datum.value.get('magnitude_error', '')) + 'filter': reduced_datum.bandpass, + 'telescope': reduced_datum.telescope, } - - if 'limit' in reduced_datum.value.keys(): - rd_data['magnitude'] = reduced_datum.value['limit'] + if reduced_datum.limit is not None: + rd_data['magnitude'] = reduced_datum.limit rd_data['limit'] = True + rd_data['error'] = '' else: - rd_data['magnitude'] = reduced_datum.value['magnitude'] + rd_data['magnitude'] = reduced_datum.brightness rd_data['limit'] = False + rd_data['error'] = reduced_datum.brightness_error or '' data.append(rd_data) initial = {'submitter': context['request'].user, @@ -223,9 +211,6 @@ def photometry_for_target(context, target, width=700, height=600, background=Non """ Renders a photometric plot for a target. - This templatetag requires all ``ReducedDatum`` objects with a data_type of ``photometry`` to be structured with the - following keys in the JSON representation: magnitude, error, filter - :param width: Width of generated plot :type width: int @@ -248,28 +233,20 @@ def photometry_for_target(context, target, width=700, height=600, background=Non 'i': 'black' } - try: - photometry_data_type = settings.DATA_PRODUCT_TYPES['photometry'][0] - except (AttributeError, KeyError): - photometry_data_type = 'photometry' - photometry_data = {} + photometry_data = defaultdict(lambda: {'time': [], 'magnitude': [], 'error': [], 'limit': []}) if settings.TARGET_PERMISSIONS_ONLY: - datums = ReducedDatum.objects.filter(target=target, data_type=photometry_data_type) + datums = PhotometryReducedDatum.objects.filter(target=target) else: datums = get_objects_for_user(context['request'].user, - 'tom_dataproducts.view_reduceddatum', - klass=ReducedDatum.objects.filter( - target=target, - data_type=photometry_data_type)) + 'tom_dataproducts.view_photometryreduceddatum', + klass=PhotometryReducedDatum.objects.filter(target=target)) for datum in datums: - if (isinstance(datum.value.get('magnitude', 0), float) and isinstance(datum.value.get('error', 0), float)) \ - or isinstance(datum.value.get('limit', 0), float): - photometry_data.setdefault(datum.value['filter'], {}) - photometry_data[datum.value['filter']].setdefault('time', []).append(datum.timestamp) - photometry_data[datum.value['filter']].setdefault('magnitude', []).append(datum.value.get('magnitude')) - photometry_data[datum.value['filter']].setdefault('error', []).append(datum.value.get('error')) - photometry_data[datum.value['filter']].setdefault('limit', []).append(datum.value.get('limit')) + if datum.brightness is not None or datum.limit is not None: + photometry_data[datum.bandpass]['time'].append(datum.timestamp) + photometry_data[datum.bandpass]['magnitude'].append(datum.brightness) + photometry_data[datum.bandpass]['error'].append(datum.brightness_error) + photometry_data[datum.bandpass]['limit'].append(datum.limit) plot_data = [] all_ydata = [] @@ -390,16 +367,16 @@ def spectroscopy_for_target(context, target, dataproduct=None): plot_data = [] if settings.TARGET_PERMISSIONS_ONLY: - datums = ReducedDatum.objects.filter(data_product__in=spectral_dataproducts) + datums = SpectroscopyReducedDatum.objects.filter(data_product__in=spectral_dataproducts) else: datums = get_objects_for_user(context['request'].user, - 'tom_dataproducts.view_reduceddatum', - klass=ReducedDatum.objects.filter(data_product__in=spectral_dataproducts)) + 'tom_dataproducts.view_spectroscopyreduceddatum', + klass=SpectroscopyReducedDatum.objects.filter( + data_product__in=spectral_dataproducts)) for datum in datums: - deserialized = SpectrumSerializer().deserialize(datum.value) plot_data.append(go.Scatter( - x=deserialized.wavelength.value, - y=deserialized.flux.value, + x=datum.wavelength, + y=datum.flux, name=datetime.strftime(datum.timestamp, '%Y%m%d-%H:%M:%s') )) @@ -471,29 +448,29 @@ def reduceddatum_sparkline(target, height, spacing=5, color_map=None, limit_y=Tr 'i': (0, 0, 0) } - vals = target.reduceddatum_set.filter( + vals = target.photometryreduceddatum_set.filter( timestamp__gte=datetime.utcnow() - timedelta(days=days) - ).values('value', 'timestamp') + ).values('brightness', 'limit', 'bandpass', 'timestamp') if len(vals) < 1: return {'sparkline': None} - vals = [v for v in vals if v['value']] + vals = [v for v in vals if v['brightness'] is not None or v['limit'] is not None] - min_mag = min([val['value']['magnitude'] for val in vals if val['value'].get('magnitude')]) - max_mag = max([val['value']['magnitude'] for val in vals if val['value'].get('magnitude')]) + min_mag = min([val['brightness'] for val in vals if val.get('brightness')]) + max_mag = max([val['brightness'] for val in vals if val.get('brightness')]) if not limit_y: # The following values are used if we want the graph's y range to extend to the values of non-detections - min_mag = min([min_mag, *[val['value']['limit'] for val in vals if val['value'].get('limit')]]) - max_mag = max([max_mag, *[val['value']['limit'] for val in vals if val['value'].get('limit')]]) + min_mag = min([min_mag, *[val['limit'] for val in vals if val.get('limit')]]) + max_mag = max([max_mag, *[val['limit'] for val in vals if val.get('limit')]]) - distinct_filters = set([val['value']['filter'] for val in vals]) + distinct_filters = set([val['bandpass'] for val in vals]) by_filter = {f: [(None, None)] * days for f in distinct_filters} for val in vals: day_index = (val['timestamp'].replace(tzinfo=timezone.utc) - timezone.now()).days - by_filter[val['value']['filter']][day_index] = (val['value'].get('magnitude'), val['value'].get('limit')) + by_filter[val['bandpass']][day_index] = (val.get('brightness'), val.get('limit')) val_range = max_mag - min_mag image_width = (spacing + 1) * (days - 1) diff --git a/tom_dataproducts/tests/test_api.py b/tom_dataproducts/tests/test_api.py index a8ad6dc69..bac7e868c 100644 --- a/tom_dataproducts/tests/test_api.py +++ b/tom_dataproducts/tests/test_api.py @@ -1,12 +1,18 @@ from django.contrib.auth.models import Group, User +from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse -from django.core.exceptions import ValidationError from guardian.shortcuts import assign_perm from rest_framework import status from rest_framework.test import APITestCase -from tom_dataproducts.models import DataProduct, ReducedDatum +from tom_dataproducts.models import ( + AstrometryReducedDatum, + DataProduct, + PhotometryReducedDatum, + ReducedDatum, + SpectroscopyReducedDatum, +) from tom_observations.tests.factories import ObservingRecordFactory from tom_targets.tests.factories import SiderealTargetFactory @@ -39,7 +45,7 @@ def test_data_product_upload_for_target(self): response = self.client.post(reverse('api:dataproducts-list'), self.dp_data, format='multipart') self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(DataProduct.objects.count(), 1) - self.assertEqual(ReducedDatum.objects.count(), 3) + self.assertEqual(PhotometryReducedDatum.objects.count(), 3) dp = DataProduct.objects.get(pk=response.data['id']) self.assertEqual(dp.target_id, self.st.id) @@ -56,7 +62,7 @@ def test_data_product_upload_for_observation(self): self.assertEqual(response.status_code, status.HTTP_201_CREATED) self.assertEqual(DataProduct.objects.count(), 1) - self.assertEqual(ReducedDatum.objects.count(), 3) + self.assertEqual(PhotometryReducedDatum.objects.count(), 3) dp = DataProduct.objects.get(pk=response.data['id']) self.assertEqual(dp.target_id, self.st.id) self.assertEqual(dp.observation_record_id, self.obsr.id) @@ -139,7 +145,7 @@ def test_upload_same_reduced_datum_twice(self): self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') self.rd_data['value'] = {'magnitude': 15.582, 'filter': 'B', 'error': 0.005} self.client.post(reverse('api:reduceddatums-list'), self.rd_data, format='json') - rd_queryset = ReducedDatum.objects.all() + rd_queryset = PhotometryReducedDatum.objects.all() self.assertEqual(rd_queryset.count(), 2) def test_upload_reduced_datum_no_sharing_location(self): @@ -170,28 +176,182 @@ def test_reduced_datum_list(self): self.assertContains(response, rd.data_type, status_code=status.HTTP_200_OK) def test_reduced_datum_filter(self): - rd1 = ReducedDatum.objects.create( + rd1 = PhotometryReducedDatum.objects.create( target=self.st, - data_type='photometry', source_name='TOM Toolkit', - value={'magnitude': 15.582, 'filter': 'r', 'error': 0.005}, + brightness=15.582, + brightness_error=0.005, + bandpass='r', ) - rd2 = ReducedDatum.objects.create( + rd2 = SpectroscopyReducedDatum.objects.create( target=self.st, - data_type='spectroscopy', source_name='TOM Toolkit', - value={'wavelength': 150, 'flux': 12, 'error': 0.005}, + wavelength=[150.0], + flux=[12.0], + error=[0.005] ) # test filter for one object response = self.client.get(reverse('api:reduceddatums-list'), QUERY_STRING='data_type=photometry') - self.assertContains(response, rd1.data_type, status_code=status.HTTP_200_OK, count=1) + self.assertContains(response, rd1.brightness, status_code=status.HTTP_200_OK, count=1) # test filter for both objects response2 = self.client.get(reverse('api:reduceddatums-list'), QUERY_STRING=f'target_name={self.st.name}') - self.assertContains(response2, rd2.data_type, status_code=status.HTTP_200_OK, count=2) + self.assertContains(response2, rd2.flux, status_code=status.HTTP_200_OK, count=2) # test filter for no objects response3 = self.client.get(reverse('api:reduceddatums-list'), QUERY_STRING='source_name=thin_air') self.assertEqual(response3.data['count'], 0) self.assertEqual(response3.data['results'], []) + + def test_upload_reduced_photometry_datum(self): + payload = { + "data_product": "", + "data_type": "photometry", + "value": {"magnitude": 15.582, "filter": "r", "error": 0.005}, + "target": self.st.id, + "timestamp": "2012-02-12T01:40:47Z", + } + response = self.client.post( + reverse("api:reduceddatums-list"), payload, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(PhotometryReducedDatum.objects.count(), 1) + rd = PhotometryReducedDatum.objects.first() + self.assertEqual(rd.brightness, payload["value"]["magnitude"]) + self.assertEqual(rd.target.id, payload["target"]) + + def test_upload_spectroscopy_datum(self): + payload = { + "data_product": "", + "data_type": "spectroscopy", + "value": {"flux": [123.4, 4.321], "wavelength": [150, 151], "error": [0.005], "flux_unit": "s"}, + "target": self.st.id, + "timestamp": "2012-02-12T01:40:47Z", + } + response = self.client.post( + reverse("api:reduceddatums-list"), payload, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(SpectroscopyReducedDatum.objects.count(), 1) + rd = SpectroscopyReducedDatum.objects.first() + self.assertEqual(rd.flux, [123.4, 4.321]) + self.assertEqual(rd.target.id, payload["target"]) + + def test_upload_astrometry_reduced_datum(self): + payload = { + "data_product": "", + "data_type": "astrometry", + "value": {"ra": 11.2, "dec": 30.0, "error": 0.005}, + "target": self.st.id, + "timestamp": "2012-02-12T01:40:47Z", + } + response = self.client.post( + reverse("api:reduceddatums-list"), payload, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(AstrometryReducedDatum.objects.count(), 1) + rd = AstrometryReducedDatum.objects.first() + self.assertEqual(rd.ra, 11.2) + self.assertEqual(rd.target.id, payload["target"]) + + def test_upload_generic_reduced_datum(self): + payload = { + "data_product": "", + "data_type": "image_file", # can be any custom data type + "value": {"ra": 11.2, "dec": 30.0, "foobar": 0.005}, + "target": self.st.id, + "timestamp": "2012-02-12T01:40:47Z", + } + response = self.client.post( + reverse("api:reduceddatums-list"), payload, format="json" + ) + self.assertEqual(response.status_code, status.HTTP_201_CREATED) + self.assertEqual(ReducedDatum.objects.count(), 1) + rd = ReducedDatum.objects.first() + self.assertEqual(rd.value["ra"], 11.2) + self.assertEqual(rd.value["foobar"], 0.005) + self.assertEqual(rd.target.id, payload["target"]) + + def test_list_includes_all_typed_models(self): + """GET /api/reduceddatums/ returns rows from all concrete model tables.""" + PhotometryReducedDatum.objects.create(target=self.st, brightness=15.0, bandpass='r') + SpectroscopyReducedDatum.objects.create(target=self.st, flux=[1.0], wavelength=[6000.0]) + AstrometryReducedDatum.objects.create(target=self.st, ra=10.0, dec=20.0) + + response = self.client.get(reverse('api:reduceddatums-list')) + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(response.data['count'], 3) + + def test_photometry_representation(self): + """PhotometryReducedDatum is serialized to the legacy wire format.""" + PhotometryReducedDatum.objects.create( + target=self.st, brightness=15.582, brightness_error=0.005, + bandpass='r', unit='AB', telescope='tst' + ) + + result = self.client.get(reverse('api:reduceddatums-list')).data['results'][0] + self.assertEqual(result['data_type'], 'photometry') + self.assertIsNone(result['data_product']) + self.assertEqual(result['value']['brightness'], 15.582) + self.assertEqual(result['value']['brightness_error'], 0.005) + self.assertEqual(result['value']['bandpass'], 'r') + self.assertEqual(result['value']['telescope'], 'tst') + + def test_spectroscopy_representation(self): + """SpectroscopyReducedDatum is serialized to the legacy wire format.""" + SpectroscopyReducedDatum.objects.create( + target=self.st, flux=[1.0, 2.0], wavelength=[6000.0, 6001.0], + error=[0.1, 0.1], flux_unit='erg/cm2/s/A' + ) + + result = self.client.get(reverse('api:reduceddatums-list')).data['results'][0] + self.assertEqual(result['data_type'], 'spectroscopy') + self.assertIsNone(result['data_product']) + self.assertEqual(result['value']['flux'], [1.0, 2.0]) + self.assertEqual(result['value']['wavelength'], [6000.0, 6001.0]) + self.assertEqual(result['value']['error'], [0.1, 0.1]) + self.assertEqual(result['value']['flux_unit'], 'erg/cm2/s/A') + + def test_astrometry_representation(self): + """AstrometryReducedDatum is serialized to the legacy wire format.""" + AstrometryReducedDatum.objects.create( + target=self.st, ra=10.5, dec=20.3, ra_error=0.001, dec_error=0.002 + ) + + result = self.client.get(reverse('api:reduceddatums-list')).data['results'][0] + self.assertEqual(result['data_type'], 'astrometry') + self.assertIsNone(result['data_product']) + self.assertEqual(result['value']['ra'], 10.5) + self.assertEqual(result['value']['dec'], 20.3) + self.assertEqual(result['value']['ra_error'], 0.001) + self.assertEqual(result['value']['dec_error'], 0.002) + + def test_data_type_filter_routes_to_correct_model(self): + """?data_type= returns only rows from the matching concrete model table.""" + PhotometryReducedDatum.objects.create(target=self.st, brightness=15.0, bandpass='r') + SpectroscopyReducedDatum.objects.create(target=self.st, flux=[1.0], wavelength=[6000.0]) + + response = self.client.get(reverse('api:reduceddatums-list'), QUERY_STRING='data_type=photometry') + self.assertEqual(response.data['count'], 1) + self.assertEqual(response.data['results'][0]['data_type'], 'photometry') + + response = self.client.get(reverse('api:reduceddatums-list'), QUERY_STRING='data_type=spectroscopy') + self.assertEqual(response.data['count'], 1) + self.assertEqual(response.data['results'][0]['data_type'], 'spectroscopy') + + def test_round_trip_post_then_get(self): + """Data POSTed in legacy wire format is stored as a typed model and returned in the same format.""" + payload = { + 'data_type': 'photometry', + 'value': {'magnitude': 15.582, 'filter': 'r', 'error': 0.005}, + 'target': self.st.id, + 'timestamp': '2012-02-12T01:40:47Z', + } + self.client.post(reverse('api:reduceddatums-list'), payload, format='json') + self.assertEqual(PhotometryReducedDatum.objects.count(), 1) + + result = self.client.get(reverse('api:reduceddatums-list')).data['results'][0] + self.assertEqual(result['data_type'], 'photometry') + self.assertEqual(result['value']['brightness'], 15.582) + self.assertEqual(result['value']['bandpass'], 'r') diff --git a/tom_dataproducts/tests/test_sharing.py b/tom_dataproducts/tests/test_sharing.py index 8aac8bb30..2e9144c80 100644 --- a/tom_dataproducts/tests/test_sharing.py +++ b/tom_dataproducts/tests/test_sharing.py @@ -1,7 +1,7 @@ from django.test import TestCase, override_settings from tom_dataproducts.alertstreams.hermes import create_hermes_alert, BuildHermesMessage, HermesMessageException -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum, SpectroscopyReducedDatum from tom_observations.tests.utils import FakeRoboticFacility from tom_observations.tests.factories import SiderealTargetFactory, ObservingRecordFactory from django.contrib.auth.models import User @@ -28,47 +28,43 @@ def setUp(self): parameters={} ) self.user = User.objects.create_user(username='test', email='test@example.com') - self.rd1 = ReducedDatum.objects.create( + self.rd1 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 18.5, 'error': .5, 'filter': 'V', 'telescope': 'tst'} + brightness=18.5, + brightness_error=0.5, + bandpass='V', + telescope='tst', ) - self.rd2 = ReducedDatum.objects.create( + self.rd2 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 19.5, 'error': .5, 'filter': 'B', 'telescope': 'tst'} + brightness=19.5, + brightness_error=0.5, + bandpass='B', + telescope='tst', ) - self.rd3 = ReducedDatum.objects.create( + self.rd3 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 17.5, 'error': .5, 'filter': 'R', 'telescope': 'tst'} + brightness=17.5, + brightness_error=0.5, + bandpass='R', + telescope='tst', ) - self.rd4 = ReducedDatum.objects.create( + self.rd4 = SpectroscopyReducedDatum.objects.create( target=self.target, - data_type='spectroscopy', - value={ - 'flux': [1, 2, 3], - 'wavelength': [6000, 5999, 5998], - 'error': [0.11, 0.22, 0.33], - 'telescope': 'SpectraTelescope' - } + flux=[1, 2, 3], + wavelength=[6000, 5999, 5998], + error=[0.11, 0.22, 0.33], + telescope='SpectraTelescope', ) - self.rd5 = ReducedDatum.objects.create( + self.rd5 = SpectroscopyReducedDatum.objects.create( target=self.target, - data_type='spectroscopy', - value={ - '1': {'flux': 20, 'wavelength': 3000}, - '2': {'flux': 21, 'wavelength': 3001}, - '3': {'flux': 22, 'wavelength': 3002}, - } + flux=[20, 21, 22], + wavelength=[3000, 3001, 3002], ) - self.bad_rd = ReducedDatum.objects.create( + self.bad_rd = SpectroscopyReducedDatum.objects.create( target=self.target, - data_type='spectroscopy', - value={ - 'myflux': [1, 2, 3], - 'wavelength_function': 'lambda_xyz' - } + flux=[1, 2, 3], + wavelength=[6000, 5999], # mismatched length — triggers HermesMessageException ) self.message_info = BuildHermesMessage( title='Test Title', @@ -100,38 +96,30 @@ def _check_alert(self, alert, message_info, datums, targets): photometry_count = 0 spectroscopy_count = 0 self.assertEqual(photometry_datums + spectroscopy_datums, len(datums)) - # These should line up for datum in datums: - if datum.data_type == 'photometry': + if isinstance(datum, PhotometryReducedDatum): hermes_datum = alert['data']['photometry'][photometry_count] self.assertEqual(hermes_datum['target_name'], datum.target.name) self.assertEqual(hermes_datum['date_obs'], datum.timestamp.isoformat()) - self.assertEqual(hermes_datum['telescope'], datum.value.get('telescope')) - self.assertEqual(hermes_datum['brightness'], datum.value.get('magnitude')) - self.assertEqual(hermes_datum['brightness_error'], datum.value.get('error')) - self.assertEqual(hermes_datum['bandpass'], datum.value.get('filter')) + self.assertEqual(hermes_datum['telescope'], datum.telescope) + self.assertEqual(hermes_datum['brightness'], datum.brightness) + self.assertEqual(hermes_datum['brightness_error'], datum.brightness_error) + self.assertEqual(hermes_datum['bandpass'], datum.bandpass) photometry_count += 1 - elif datum.data_type == 'spectroscopy': + elif isinstance(datum, SpectroscopyReducedDatum): hermes_datum = alert['data']['spectroscopy'][spectroscopy_count] self.assertEqual(hermes_datum['target_name'], datum.target.name) self.assertEqual(hermes_datum['date_obs'], datum.timestamp.isoformat()) - if 'flux' in datum.value and 'wavelength' in datum.value: - self.assertEqual(hermes_datum['flux'], datum.value.get('flux')) - self.assertEqual(hermes_datum['wavelength'], datum.value.get('wavelength')) - if 'error' in datum.value: - self.assertEqual(hermes_datum['flux_error'], datum.value.get('error')) - else: - for i, entry in enumerate(datum.value.values()): - if 'flux' in entry: - self.assertEqual(hermes_datum['flux'][i], entry['flux']) - self.assertEqual(hermes_datum['wavelength'][i], entry['wavelength']) + self.assertEqual(hermes_datum['flux'], datum.flux) + self.assertEqual(hermes_datum['wavelength'], datum.wavelength) + if datum.error: + self.assertEqual(hermes_datum['flux_error'], datum.error) spectroscopy_count += 1 def test_convert_to_hermes_format(self): datums = [self.rd1, self.rd2, self.rd3] targets = [self.target] alert = create_hermes_alert(self.message_info, datums, targets) - # Now check the alerts formatting is correct self._check_alert(alert, self.message_info, datums, targets) def test_convert_to_hermes_format_extra_target(self): @@ -139,32 +127,27 @@ def test_convert_to_hermes_format_extra_target(self): datums = [self.rd1, self.rd2, self.rd3] targets = [target2, self.target] alert = create_hermes_alert(self.message_info, datums, targets) - # Now check the alerts formatting is correct self._check_alert(alert, self.message_info, datums, targets) def test_convert_to_hermes_format_only_targets(self): target2 = SiderealTargetFactory.create() targets = [target2, self.target] alert = create_hermes_alert(self.message_info, [], targets) - # Now check the alerts formatting is correct self._check_alert(alert, self.message_info, [], targets) def test_convert_to_hermes_format_only_datums(self): datums = [self.rd1, self.rd2, self.rd3] alert = create_hermes_alert(self.message_info, datums, []) - # Now check the alerts formatting is correct self._check_alert(alert, self.message_info, datums, [self.target]) def test_convert_to_hermes_format_spectro_datums(self): datums = [self.rd4, self.rd5] alert = create_hermes_alert(self.message_info, datums, []) - # Now check the alerts formatting is correct self._check_alert(alert, self.message_info, datums, [self.target]) def test_convert_to_hermes_format_mixed_datums(self): datums = [self.rd1, self.rd2, self.rd3, self.rd4, self.rd5] alert = create_hermes_alert(self.message_info, datums, []) - # Now check the alerts formatting is correct self._check_alert(alert, self.message_info, datums, [self.target]) def test_convert_to_hermes_format_bad_spectro_datum_fails(self): diff --git a/tom_dataproducts/tests/tests.py b/tom_dataproducts/tests/tests.py index b53fd58e9..1a0f0f0c4 100644 --- a/tom_dataproducts/tests/tests.py +++ b/tom_dataproducts/tests/tests.py @@ -1,34 +1,44 @@ -import datetime -from http import HTTPStatus import os import tempfile -import responses +from datetime import date, time +from http import HTTPStatus +from unittest.mock import patch +import numpy as np +import responses from astropy import units from astropy.io import fits from astropy.table import Table -from datetime import date, time -from django.test import TestCase, override_settings from django.conf import settings from django.contrib.auth.models import Group, User -from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile +from django.core.exceptions import ValidationError +from django.db import IntegrityError +from django.test import TestCase, override_settings from django.urls import reverse -from django.utils import timezone, text +from django.utils import text, timezone from guardian.shortcuts import assign_perm -import numpy as np from specutils import Spectrum -from unittest.mock import patch from tom_dataproducts.exceptions import InvalidFileFormatException from tom_dataproducts.forms import DataProductUploadForm -from tom_dataproducts.models import DataProduct, is_fits_image_file, ReducedDatum, data_product_path +from tom_dataproducts.models import ( + DataProduct, + PhotometryReducedDatum, + ReducedDatum, + SpectroscopyReducedDatum, + data_product_path, + is_fits_image_file, +) from tom_dataproducts.processors.data_serializers import SpectrumSerializer from tom_dataproducts.processors.photometry_processor import PhotometryProcessor from tom_dataproducts.processors.spectroscopy_processor import SpectroscopyProcessor from tom_dataproducts.utils import create_image_dataproduct +from tom_observations.tests.factories import ( + ObservingRecordFactory, + SiderealTargetFactory, +) from tom_observations.tests.utils import FakeRoboticFacility -from tom_observations.tests.factories import SiderealTargetFactory, ObservingRecordFactory def mock_fits2image(file1, file2, width, height): @@ -538,6 +548,47 @@ def test_create_thumbnail(self, mock_is_fits_image_file): self.assertIn(expected, logs.output) +class TestCustomFields(TestCase): + def test_create_spectra_with_flux_data(self): + flux = [7.427265572723272, 3399.697753906248] + wavelength = [3399.697753906248, 3401.383788108824] + rd = SpectroscopyReducedDatum.objects.create( + target=SiderealTargetFactory.create(), + timestamp=timezone.now(), + exposure_time=1000.0, + flux=flux, + wavelength=wavelength, + flux_unit="Å", + ) + rd.refresh_from_db() # ensure we round trip to the database + self.assertEqual(flux, rd.flux) + + def test_create_spectra_with_error_data(self): + error = [0.0001, 0.0002, 0.0003, 0.0004] + rd = SpectroscopyReducedDatum.objects.create( + target=SiderealTargetFactory.create(), + timestamp=timezone.now(), + exposure_time=1000.0, + flux=[1.0, 2.0, 3.0, 4.0], + wavelength=[1, 2, 3, 4], + error=error, + flux_unit="Å", + ) + rd.refresh_from_db() + self.assertEqual(error, rd.error) + + def test_create_spectra_bad_flux(self): + with self.assertRaises(ValidationError): + SpectroscopyReducedDatum.objects.create( + target=SiderealTargetFactory.create(), + timestamp=timezone.now(), + exposure_time=1000.0, + flux=[1.0, 2.0, 3.0, "asd"], + wavelength=[1, 2, 3, 4], + flux_unit="cm^2", + ) + + class TestReducedDatumModel(TestCase): def setUp(self): # set up a ReducedDatum instance to test against @@ -577,43 +628,22 @@ def test_create_reduced_datum(self): self.assertEqual(2, ReducedDatum.objects.count()) def test_create_reduced_datum_duplicate(self): - """Test that we cannot add a second ReducedDatum with the same target, data_type, - timestamp, and value dict""" - # in this case ALL fields are the same as the self.existing_reduced_datum - with self.assertRaises(ValidationError): - ReducedDatum.objects.create( - target=self.target, - data_type=self.data_type, - source_name=self.source_name, - timestamp=self.timestamp, - value=self.existing_reduced_datum_value) - - # in this case only the target, data_type and value fields - # are the same as the self.existing_reduced_datum - # so an exception should NOT be raised - try: - ReducedDatum.objects.create( - target=self.target, - data_type=self.data_type, - source_name='new_source_name', - timestamp=(self.timestamp - datetime.timedelta(days=1)), # different timestamp - value=self.existing_reduced_datum_value) - except ValidationError: - self.fail("ValidationError raised when it should not have been (timestamps differ)") - - # by NOT raising ValidationError, this shows that - # ReducedDatum.objects.bulk_create() bypasses the ReducedDatum.save() - # method which validated uniqueness!! - # (this is a duplicate ReducedDatum that we are trying to add here - unsaved_reduced_datum = ReducedDatum( + """Test that we cannot add a second PhotometryReducedDatum with the same target, + timestamp, and bandpass""" + PhotometryReducedDatum.objects.create( target=self.target, - data_type=self.data_type, - source_name=self.source_name, timestamp=self.timestamp, - value=self.existing_reduced_datum_value) - # does bulk_create bypass the ReducedDatum.save() method which validated uniqueness? - # (this is a duplicate ReducedDatum that we are trying to add here - ReducedDatum.objects.bulk_create([unsaved_reduced_datum]) + brightness=1.0, + bandpass="r" + ) + + with self.assertRaises(IntegrityError): + PhotometryReducedDatum.objects.create( + target=self.target, + timestamp=self.timestamp, + brightness=2.0, + bandpass="r" + ) @override_settings(TOM_FACILITY_CLASSES=['tom_observations.tests.utils.FakeRoboticFacility'], @@ -639,20 +669,23 @@ def setUp(self): assign_perm('tom_targets.view_target', self.user, self.target) self.client.force_login(self.user) - self.rd1 = ReducedDatum.objects.create( + self.rd1 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 18.5, 'error': .5, 'filter': 'V'} + brightness=18.5, + brightness_error=0.5, + bandpass='V', ) - self.rd2 = ReducedDatum.objects.create( + self.rd2 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 19.5, 'error': .5, 'filter': 'B'} + brightness=19.5, + brightness_error=0.5, + bandpass='B', ) - self.rd3 = ReducedDatum.objects.create( + self.rd3 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + brightness=17.5, + brightness_error=0.5, + bandpass='R', ) @responses.activate @@ -749,7 +782,7 @@ def test_share_reduced_datums_no_valid_responses(self): 'share_destination': [share_destination], 'share_title': ['Updated data for thingy.'], 'share_message': ['test_message'], - 'share-box': [1, 2] + 'share-box': [self.rd1.pk, self.rd2.pk] }, follow=True ) @@ -867,7 +900,7 @@ def test_share_reduced_datums_valid_responses(self): 'share_destination': [share_destination], 'share_title': ['Updated data for thingy.'], 'share_message': ['test_message'], - 'share-box': [1, 2] + 'share-box': [self.rd1.pk, self.rd2.pk] }, follow=True ) @@ -899,7 +932,7 @@ def test_share_reduced_datums_invalid_responses(self): 'share_destination': [share_destination], 'share_title': ['Updated data for thingy.'], 'share_message': ['test_message'], - 'share-box': [1, 2] + 'share-box': [self.rd1.pk, self.rd2.pk] } # Check 500 error responses.add( diff --git a/tom_dataproducts/views.py b/tom_dataproducts/views.py index d5787ea73..cc9961698 100644 --- a/tom_dataproducts/views.py +++ b/tom_dataproducts/views.py @@ -23,7 +23,7 @@ from tom_common.hooks import run_hook from tom_common.hints import add_hint from tom_common.mixins import Raise403PermissionRequiredMixin -from tom_dataproducts.models import DataProduct, DataProductGroup, ReducedDatum +from tom_dataproducts.models import DataProduct, DataProductGroup, REDUCED_DATUM_MODELS from tom_dataproducts.exceptions import InvalidFileFormatException from tom_dataproducts.forms import AddProductToGroupForm, DataProductUploadForm, DataShareForm from tom_dataproducts.filters import DataProductFilter @@ -39,6 +39,11 @@ logger.setLevel(logging.DEBUG) +def _delete_reduced_datums_for_product(dp): + for model in REDUCED_DATUM_MODELS: + model.objects.filter(data_product=dp).delete() + + class DataProductSaveView(LoginRequiredMixin, View): """ View that handles saving a ``DataProduct`` generated by an observation. Requires authentication. @@ -213,17 +218,19 @@ def form_valid(self, form): for group in form.cleaned_data['groups']: assign_perm('tom_dataproducts.view_dataproduct', group, dp) assign_perm('tom_dataproducts.delete_dataproduct', group, dp) - assign_perm('tom_dataproducts.view_reduceddatum', group, reduced_data) + for datum in reduced_data: + perm = f'tom_dataproducts.view_{type(datum).__name__.lower()}' + assign_perm(perm, group, datum) successful_uploads.append(str(dp)) except InvalidFileFormatException as iffe: - ReducedDatum.objects.filter(data_product=dp).delete() + _delete_reduced_datums_for_product(dp) dp.delete() messages.error( self.request, f'File format invalid for file {str(dp)} -- error was {iffe}' ) except Exception as e: - ReducedDatum.objects.filter(data_product=dp).delete() + _delete_reduced_datums_for_product(dp) dp.delete() messages.error(self.request, f'There was a problem processing your file: {str(dp)} -- Error: {e}') if successful_uploads: @@ -288,7 +295,7 @@ def form_valid(self, form): data_product = self.get_object() # Delete associated ReducedDatum objects - ReducedDatum.objects.filter(data_product=data_product).delete() + _delete_reduced_datums_for_product(data_product) # Delete the file reference. data_product.data.delete() diff --git a/tom_targets/merge.py b/tom_targets/merge.py index 11831cbbc..1ffcf1f68 100644 --- a/tom_targets/merge.py +++ b/tom_targets/merge.py @@ -1,7 +1,7 @@ from django.contrib import messages from django.core.exceptions import ValidationError from tom_targets.models import TargetName, TargetExtra -from tom_dataproducts.models import ReducedDatum, DataProduct +from tom_dataproducts.models import DataProduct, REDUCED_DATUM_MODELS from tom_observations.models import ObservationRecord @@ -61,15 +61,15 @@ def target_merge(primary_target, secondary_target): dataproduct.save() # take secondary_target reduceddatums and save them as primary_target reduceddatums - st_reduceddatums = ReducedDatum.objects.filter(target=secondary_target) - for reduceddatum in st_reduceddatums: - reduceddatum.target = primary_target - try: - reduceddatum.validate_unique() - except ValidationError: - reduceddatum.delete() # delete what would become a duplicate reduceddatum - else: - reduceddatum.save() + for model in REDUCED_DATUM_MODELS: + for reduceddatum in model.objects.filter(target=secondary_target): + reduceddatum.target = primary_target + try: + reduceddatum.validate_unique() + except ValidationError: + reduceddatum.delete() # delete what would become a duplicate reducedatum + else: + reduceddatum.save() # take secondary target extras without repeated keys and save them as primary target extras pt_targetextra_keys = list(TargetExtra.objects.filter(target=primary_target).values_list("key", flat=True)) diff --git a/tom_targets/sharing.py b/tom_targets/sharing.py index ebeba11f9..815ea5bde 100644 --- a/tom_targets/sharing.py +++ b/tom_targets/sharing.py @@ -7,7 +7,7 @@ from tom_targets.models import PersistentShare from tom_dataproducts.sharing import (check_for_share_safe_datums, share_data_with_tom, get_destination_target, sharing_feedback_converter) -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum from tom_dataproducts.alertstreams.hermes import publish_to_hermes, BuildHermesMessage @@ -22,7 +22,7 @@ def share_target_and_all_data(share_destination, target): hermes_topic = share_destination.split(':')[1] destination = share_destination.split(':')[0] filtered_reduced_datums = check_for_share_safe_datums( - destination, ReducedDatum.objects.filter(target=target), topic=hermes_topic) + destination, PhotometryReducedDatum.objects.filter(target=target), topic=hermes_topic) sharing = getattr(settings, "DATA_SHARING", {}) tom_name = f"{getattr(settings, 'TOM_NAME', 'TOM Toolkit')}" message = BuildHermesMessage(title=f"Setting up continuous sharing for {target.name} from " @@ -56,7 +56,7 @@ def continuous_share_data(target, reduced_datums): hermes_topic = share_destination.split(':')[1] destination = share_destination.split(':')[0] filtered_reduced_datums = check_for_share_safe_datums( - destination, ReducedDatum.objects.filter(pk__in=reduced_datum_pks), topic=hermes_topic) + destination, PhotometryReducedDatum.objects.filter(pk__in=reduced_datum_pks), topic=hermes_topic) sharing = getattr(settings, "DATA_SHARING", {}) tom_name = f"{getattr(settings, 'TOM_NAME', 'TOM Toolkit')}" message = BuildHermesMessage(title=f"Updated data for {target.name} from " diff --git a/tom_targets/signals/handlers.py b/tom_targets/signals/handlers.py index 8563d3b27..8ad4312a6 100644 --- a/tom_targets/signals/handlers.py +++ b/tom_targets/signals/handlers.py @@ -2,12 +2,11 @@ from django.db.models.signals import post_save, post_delete from guardian.utils import clean_orphan_obj_perms -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import REDUCED_DATUM_MODELS from tom_targets.sharing import continuous_share_data from tom_targets.models import Target -@receiver(post_save, sender=ReducedDatum) def cb_reduceddatum_post_save(sender, instance, *args, **kwargs): # When a new ReducedDatum is created or updated, check for any persistentshare instances on that target # and if they exist, attempt to share the new data @@ -15,6 +14,10 @@ def cb_reduceddatum_post_save(sender, instance, *args, **kwargs): continuous_share_data(target, reduced_datums=[instance]) +for model in REDUCED_DATUM_MODELS: + post_save.connect(cb_reduceddatum_post_save, sender=model) + + @receiver(post_delete, sender=Target) def cb_target_post_delete(sender, instance, *args, **kwargs): # When a Target is deleted, clean up orphaned permissions. diff --git a/tom_targets/tests/tests.py b/tom_targets/tests/tests.py index 522b09b49..c0f4222ec 100644 --- a/tom_targets/tests/tests.py +++ b/tom_targets/tests/tests.py @@ -21,7 +21,7 @@ from tom_targets.base_models import BaseTarget from tom_targets.templatetags.targets_extras import target_table_headers, target_table_row from tom_targets.permissions import targets_for_user -from tom_dataproducts.models import ReducedDatum, DataProduct +from tom_dataproducts.models import PhotometryReducedDatum, ReducedDatum, DataProduct from tom_observations.models import ObservationRecord from guardian.shortcuts import assign_perm, get_perms @@ -1466,20 +1466,23 @@ def setUp(self): self.user = User.objects.create_user(username='test', email='test@example.com') assign_perm('tom_targets.view_target', self.user, self.target) self.client.force_login(self.user) - self.rd1 = ReducedDatum.objects.create( + self.rd1 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 18.5, 'error': .5, 'filter': 'V'} + brightness=18.5, + brightness_error=0.5, + bandpass='V', ) - self.rd2 = ReducedDatum.objects.create( + self.rd2 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 19.5, 'error': .5, 'filter': 'B'} + brightness=19.5, + brightness_error=0.5, + bandpass='B', ) - self.rd3 = ReducedDatum.objects.create( + self.rd3 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + brightness=17.5, + brightness_error=0.5, + bandpass='R', ) @responses.activate @@ -1616,7 +1619,7 @@ def test_share_reduceddatums_target_valid_responses(self): 'submitter': ['test_submitter'], 'target': self.target.id, 'share_destination': [share_destination], - 'share-box': [1, 2] + 'share-box': [self.rd1.pk, self.rd2.pk] }, follow=True ) @@ -1646,25 +1649,29 @@ def setUp(self): self.user = User.objects.create_user(username='test', email='test@example.com') assign_perm('tom_targets.view_target', self.user, self.target) self.client.force_login(self.user) - self.rd1 = ReducedDatum.objects.create( + self.rd1 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 18.5, 'error': .5, 'filter': 'V'} + brightness=18.5, + brightness_error=0.5, + bandpass='V', ) - self.rd2 = ReducedDatum.objects.create( + self.rd2 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 19.5, 'error': .5, 'filter': 'B'} + brightness=19.5, + brightness_error=0.5, + bandpass='B', ) - self.rd3 = ReducedDatum.objects.create( + self.rd3 = PhotometryReducedDatum.objects.create( target=self.target, - data_type='photometry', - value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + brightness=17.5, + brightness_error=0.5, + bandpass='R', ) - self.rd4 = ReducedDatum.objects.create( + self.rd4 = PhotometryReducedDatum.objects.create( target=self.target2, - data_type='photometry', - value={'magnitude': 17.5, 'error': .5, 'filter': 'R'} + brightness=17.5, + brightness_error=0.5, + bandpass='R', ) @responses.activate diff --git a/tom_targets/views.py b/tom_targets/views.py index 18c0b2024..b256725f1 100644 --- a/tom_targets/views.py +++ b/tom_targets/views.py @@ -48,7 +48,7 @@ from tom_targets.merge import target_merge from tom_dataproducts.sharing import (share_data_with_hermes, share_data_with_tom, sharing_feedback_handler, share_target_list_with_hermes) -from tom_dataproducts.models import ReducedDatum +from tom_dataproducts.models import PhotometryReducedDatum from tom_targets.groups import ( add_all_to_grouping, add_selected_to_grouping, remove_all_from_grouping, remove_selected_from_grouping, move_all_to_grouping, move_selected_to_grouping @@ -554,7 +554,7 @@ def post(self, request, *args, **kwargs): message=request.POST.get('share_message', ''), authors=sharing['hermes'].get('DEFAULT_AUTHORS') ) - reduced_datums = ReducedDatum.objects.filter(pk__in=request.POST.getlist('share-box', [])) + reduced_datums = PhotometryReducedDatum.objects.filter(pk__in=request.POST.getlist('share-box', [])) preload_key = preload_to_hermes(hermes_message, reduced_datums, [target]) load_url = sharing['hermes']['BASE_URL'] + f'submit-message?id={preload_key}' return HttpResponseRedirect(load_url) @@ -923,9 +923,9 @@ def post(self, request, *args, **kwargs): ) targets = Target.objects.filter(pk__in=request.POST.getlist('selected-target', [])) if request.POST.get('dataSwitch', '') == 'on': - reduced_datums = ReducedDatum.objects.filter(target__in=targets, data_type='photometry') + reduced_datums = PhotometryReducedDatum.objects.filter(target__in=targets) else: - reduced_datums = ReducedDatum.objects.none() + reduced_datums = PhotometryReducedDatum.objects.none() preload_key = preload_to_hermes(hermes_message, reduced_datums, targets) load_url = sharing['hermes']['BASE_URL'] + f'submit-message?id={preload_key}' return HttpResponseRedirect(load_url)