Reducedddatum refactor

docs/common/latex_generation.rst

@@ -100,7 +100,7 @@ to access the object for which we’re generating data.
 
    from django import forms
 
-   from tom_dataproducts.models import ReducedDatum
+   from tom_dataproducts.models import PhotometryReducedDatum, SpectroscopyReducedDatum
    from tom_publications.latex import GenericLatexProcessor, GenericLatexForm
    from tom_targets.models import Target
 
@@ -110,18 +110,17 @@ to access the object for which we’re generating data.
      form_class = TargetDataLatexForm
 
      def create_latex_table_data(self, cleaned_data):
-       target = Target.objects.get(pk=cleaned_data.get('model_pk'))
-       data = ReducedDatum.objects.filter(target=target, data_type=cleaned_data.get('data_type'))
-
-       table_data = {}
-       if cleaned_data.get('data_type') == 'photometry':
-         for datum in data:
-           for key, value in json.loads(datum.value).items():
-             table_data.setdefault(key, []).append(value)
-       elif cleaned_data.get('data_type') == 'spectroscopy':
-         ...
-
-       return table_data
+        target = Target.objects.get(pk=cleaned_data.get('model_pk'))
+        table_data = {}
+        if cleaned_data.get('data_type') == 'photometry':
+            data = PhotometryReducedDatum.objects.filter(target=target)
+            for brightness, bandpass in data.values_list('brightness', 'bandpass'):
+                table_data.setdefault('brightness', []).append(brightness)
+                table_data.setdefault('bandpass', []).append(bandpass)
+        elif cleaned_data.get('data_type') == 'spectroscopy':
+            ...
+
+        return table_data
 
 The above example only shows the photometric table generation, but
 spectroscopic can be left as an exercise to the reader.
@@ -184,7 +183,7 @@ TOM. Here’s our final ``target_data_latex_processor.py``:
 
    from django import forms
 
-   from tom_dataproducts.models import ReducedDatum
+   from tom_dataproducts.models import PhotometryReducedDatum, SpectroscopyReducedDatum
    from tom_publications.latex import GenericLatexProcessor, GenericLatexForm
    from tom_targets.models import Target
 
@@ -202,13 +201,12 @@ TOM. Here’s our final ``target_data_latex_processor.py``:
 
        def create_latex_table_data(self, cleaned_data):
            target = Target.objects.get(pk=cleaned_data.get('model_pk'))
-           data = ReducedDatum.objects.filter(target=target, data_type=cleaned_data.get('data_type'))
-
            table_data = {}
            if cleaned_data.get('data_type') == 'photometry':
-               for datum in data:
-                   for key, value in json.loads(datum.value).items():
-                       table_data.setdefault(key, []).append(value)
+               data = PhotometryReducedDatum.objects.filter(target=target)
+               for brightness, bandpass in data.values_list('brightness', 'bandpass'):
+                   table_data.setdefault('brightness', []).append(brightness)
+                   table_data.setdefault('bandpass', []).append(bandpass)
            elif cleaned_data.get('data_type') == 'spectroscopy':
                ...
 

docs/common/permissions.rst

@@ -139,3 +139,15 @@ The above code will allow all users in the groups that the example user belongs
 ``ReducedDatum``:
 
 * ``tom_dataproducts.view_reduceddatum``
+
+``PhotometryReducedDatum``:
+
+* ``tom_dataproducts.view_photometryreduceddatum``
+
+``SpectroscopyReducedDatum``:
+
+* ``tom_dataproducts.view_spectroscopyreduceddatum``
+
+``AstrometryReducedDatum``:
+
+* ``tom_dataproducts.view_astrometryreduceddatum``

docs/customization/customize_template_tags.rst

@@ -154,8 +154,8 @@ approach here:
 
 You can see that we’ll eventually be returning a dictionary, but first
 we need to add our logic. We’ll need to use the ``Target`` passed in to
-get all ``ReducedDatum`` objects for that ``Target`` with a
-``data_type`` of ``photometry``. Then we’ll need to order by
+get all ``PhotometryReducedDatum`` objects for that ``Target``
+Then we’ll need to order by
 ``timestamp`` descending, and slice just the first few. Make sure to
 take note of the imports in this step!
 
@@ -165,16 +165,16 @@ take note of the imports in this step!
 
    from django import template
 
-   from tom_dataproducts.models import ReducedDatum
+   from tom_dataproducts.models import PhotometryReducedDatum
 
 
    register = template.Library()
 
 
    @register.inclusion_tag('custom_code/partials/recent_photometry.html')
    def recent_photometry(target, num_points=1):
-       photometry = ReducedDatum.objects.filter(data_type='photometry').order_by('-timestamp')[:num_points]
-       return {'recent_photometry': [(datum.timestamp, json.loads(datum.value)['magnitude']) for datum in photometry]}
+       photometry = PhotometryReducedDatum.objects.order_by('-timestamp')[:num_points]
+       return {'recent_photometry': [(datum.timestamp, datum.brightness) for datum in photometry]}
 
 It’s only a couple of lines, but there’s a lot going on here. The first
 line does the aforemention database query and slices the first point of
@@ -330,4 +330,4 @@ As far as this template tag goes, as of this tutorial, it’s now a part
 of the base TOM Toolkit, but all of the information here should provide
 you with the ability to write your own.
 
-.. |image0| image:: /_static/customize_template_tags_doc/Templatetags.png
+.. |image0| image:: /_static/customize_template_tags_doc/Templatetags.png

docs/introduction/tomarchitecture.rst

@@ -52,7 +52,7 @@ they are left with a functioning but generic TOM. It is then up to the developer
 to implement the specific features that their science case requires. The toolkit
 tries to facilitate this as efficiently as possible and provides
 :doc:`documentation <index>` in areas of customization from :doc:`changing the HTML layout of a page </customization/customize_templates>`
-to :doc:`customizing an OCS facility and forms </observing/customize_ocs_facility>` and even 
+to :doc:`customizing an OCS facility and forms </observing/customize_ocs_facility>` and even
 :doc:`creating a new alert broker </brokers/create_broker>`.
 
 Django, and by extension the toolkit, rely heavily on object oriented
@@ -95,7 +95,7 @@ each other. This means a TOM developer can easily change the layout and style of
 any page without modifying the underlying framework's code directly. Entire pages
 may be replaced, or only "blocks" within a template.
 
-Compare these screenshots of the `standard target detail page <../../../_static/architecture/snex2layout.png>`_ and the 
+Compare these screenshots of the `standard target detail page <../../../_static/architecture/snex2layout.png>`_ and the
 `Global Supernova Project's target detail page <../../../_static/architecture/snex2layout.png>`_, the
 latter taking heavy advantage of template inheritance.
 
@@ -266,28 +266,46 @@ ReducedDatum
 ------------
 
 A ``ReducedDatum`` is a single point of data associated with a ``Target`` and optionally a
-``DataProduct``. The single data point is typically a single point of photometry or an individual
-spectrum. The ``ReducedDatum`` model has the following fields, in addition to its aforementioned
+``DataProduct``.
+There are three classes of ReducedDatum for the common data types:
+``PhotometryReducedDatum``, ``SpectroscopyReducedDatum``, and ``AstrometryReducedDatum``.
+The ``ReducedDatum`` is a general model meant to be flexible enough to allow for other data types as well.
+
+The ``ReducedDatum`` model has the following fields, in addition to its aforementioned
 foreign key relationships:
 
 - ``data_type`` is maintained on both the ``ReducedDatum`` and ``DataProduct`` for the case when data is brought in from another source, such as a broker
 - The ``source_name`` optionally refers to the original source of the data. The intent of this field was to track data ingested from brokers, but could potentially be used for other purposes.
 - ``source_location`` optionally gives a hard location to the source--for a broker, it would be a link to the original alert.
 - The ``timestamp`` time at which the datum was produced.
-- ``value`` is a ``TextField`` that can take any series of data. As implemented, photometry is stored as JSON with keys for magnitude and error, but the ``TextField`` provides flexibility for additional photometry values on the datum. Spectroscopy is also stored as JSON, with keys for ``magnitude`` and ``flux``.
-
-Feedback and bug reporting
-==========================
+- ``value`` is a ``JSONField`` that can take any series of data.
+- ``telescope`` and ``instrument`` are optional fields that can be used to track additional metadata.
 
-We hope the TOM Toolkit is helpful to you and your project. If you have any
-concerns about implementation details, or questions about your own needs, please
-don't hesitate to `reach out <mailto:dcollom@lco.global>`_. Issues and pull requests
-are also welcome on the project's `GitHub page <https://github.com/TOMToolkit/>`_.
 
+The ``PhotometryReducedDatum`` model has the following Photometry specific fields:
 
+- ``brightness`` and ``brightness_error`` are float fields that track the magnitude and error, respectively.
+- ``bandpass`` is a char field that tracks the bandpass/filter of the photometry.
+- ``limit`` optional float field that tracks the limiting magnitude of the photometry
+- ``unit`` optional char field that tracks the unit of the photometry
+- ``exposure_time`` optional float field that tracks the exposure time of the photometry
 
+The ``SpectroscopyReducedDatum`` model has the following Spectroscopy specific fields:
 
+- ``wavelength``, ``flux`` and ``error`` are all FloatArrayFields that track the wavelength, flux, and error of the spectroscopy, respectively
+- ``unit`` optional char field that tracks the unit of the spectroscopy
+- ``setup`` optional text field for arbitrary metadata about the spectroscopic setup
+- ``exposure_time`` optional float field that tracks the exposure time of the spectroscopy
 
+The ``AstrometryReducedDatum`` model has the following Astrometry specific fields:
 
+- ``ra``, ``dec``, ``ra_error`` and ``dec_error``  are all float fields for tracking coordinates and error. Errors are optional.
+- ``ra_error_units`` and ``dec_error_units`` optional char fields that track the units of the errors
 
+Feedback and bug reporting
+==========================
 
+We hope the TOM Toolkit is helpful to you and your project. If you have any
+concerns about implementation details, or questions about your own needs, please
+don't hesitate to `reach out <mailto:dcollom@lco.global>`_. Issues and pull requests
+are also welcome on the project's `GitHub page <https://github.com/TOMToolkit/>`_.

docs/managing_data/customizing_data_processing.rst

@@ -21,12 +21,12 @@ tom_dataproducts app in the TOM Toolkit:
 
 Let’s start with a quick overview of ``models.py``. The file contains
 the Django models for the dataproducts app–in our case, ``DataProduct``
-and ``ReducedDatum``. The ``DataProduct`` contains information about
+, ``ReducedDatum`` and the three specialized reduced data product classes:
+``PhotometryReducedDatum``, ``SpectroscopyReducedDatum`` and ``AstrometryReducedDatum``.
+The ``DataProduct`` contains information about
 uploaded or saved ``DataProducts``, such as the file name, file path,
-and what kind of file it is. The ``ReducedDatum`` contains individual
+and what kind of file it is. The ``*ReducedDatum`` classes contain individual
 science data points that are taken from the ``DataProduct`` files.
-Examples of ``ReducedDatum`` points would be individual photometry
-points or individual spectra.
 
 Each ``DataProduct`` also has a ``data_product_type``. The
 ``data_product_type`` is simply a description of what the file is, more

docs/managing_data/plotting_data.rst

@@ -128,7 +128,7 @@ Next, add the function body:
        # x axis: target names. y axis: datum count
        data = [go.Bar(
            x=[target.name for target in targets],
-           y=[target.reduceddatum_set.count() for target in targets]
+           y=[target.photometryreduceddatum_set.count() for target in targets]
        )]
        # Create the plot
        figure = offline.plot(go.Figure(data=data), output_type='div', show_link=False)

docs/managing_data/tom_direct_sharing.rst

@@ -46,7 +46,7 @@ Receiving Shared Data:
 Reduced Datums:
 ---------------
 When your TOM receives a new ``ReducedDatum`` from another TOM it will be saved to your TOM's database with its source
-set to the name of the TOM that submitted it. Currently, only Photometry data can be directly shared between
+set to the name of the TOM that submitted it. Currently, only ``PhotometryReducedDatum`` can be directly shared between
 TOMS and a ``Target`` with a matching name or alias must exist in both TOMS for sharing to take place.
 
 Data Products:
@@ -66,9 +66,3 @@ Target Lists:
 When your TOM receives a new ``TargetList`` from another TOM it will be saved to your TOM's database. If the targets in
 the ``TargetList`` are also shared, but already exist in the destination TOM, they will be added to the new
 ``TargetList``.
-
-
-
-
-
-

tom_alerts/brokers/alerce.py

@@ -10,7 +10,7 @@
 
 from tom_alerts.alerts import GenericAlert, GenericBroker, GenericQueryForm
 from tom_targets.models import Target
-from tom_dataproducts.models import ReducedDatum
+from tom_dataproducts.models import PhotometryReducedDatum
 
 logger = logging.getLogger(__name__)
 
@@ -573,35 +573,31 @@ def process_reduced_data(self, target, alert=None):
 
         for detection in lightcurve['detections']:
             mjd = Time(detection['mjd'], format='mjd', scale='utc')
-            value = {
-                'filter': FILTERS[detection['fid']],
-                'magnitude': detection['magpsf'],
-                'error': detection['sigmapsf'],
-                'telescope': 'ZTF',
-            }
-            ReducedDatum.objects.get_or_create(
+            PhotometryReducedDatum.objects.get_or_create(
+                target=target,
+                bandpass=FILTERS[detection['fid']],
                 timestamp=mjd.to_datetime(TimezoneInfo()),
-                value=value,
-                source_name=self.name,
-                source_location=oid,
-                data_type='photometry',
-                target=target
+                defaults={
+                    'brightness': detection['magpsf'],
+                    'brightness_error': detection['sigmapsf'],
+                    'telescope': 'ZTF',
+                    'source_name': self.name,
+                    'source_location': oid,
+                }
             )
 
         for non_detection in lightcurve['non_detections']:
             mjd = Time(non_detection['mjd'], format='mjd', scale='utc')
-            value = {
-                'filter': FILTERS[non_detection['fid']],
-                'limit': non_detection['diffmaglim'],
-                'telescope': 'ZTF',
-            }
-            ReducedDatum.objects.get_or_create(
+            PhotometryReducedDatum.objects.get_or_create(
+                target=target,
+                bandpass=FILTERS[non_detection['fid']],
                 timestamp=mjd.to_datetime(TimezoneInfo()),
-                value=value,
-                source_name=self.name,
-                source_location=oid,
-                data_type='photometry',
-                target=target
+                defaults={
+                    'limit': non_detection['diffmaglim'],
+                    'telescope': 'ZTF',
+                    'source_name': self.name,
+                    'source_location': oid,
+                }
             )
 
     def to_target(self, alert):

tom_alerts/brokers/gaia.py

@@ -12,7 +12,7 @@
 from django import forms
 
 from tom_alerts.alerts import GenericAlert, GenericBroker, GenericQueryForm
-from tom_dataproducts.models import ReducedDatum
+from tom_dataproducts.models import PhotometryReducedDatum
 
 BASE_BROKER_URL = 'http://gsaweb.ast.cam.ac.uk'
 
@@ -181,17 +181,16 @@ def process_reduced_data(self, target, alert=None):
                     jd = Time(float(phot_data[1]), format='jd', scale='utc')
                     jd.to_datetime(timezone=TimezoneInfo())
 
-                    value = {
-                        'magnitude': float(phot_data[2]),
-                        'filter': 'G'
-                    }
-
-                    rd, _ = ReducedDatum.objects.get_or_create(
+                    rd, _ = PhotometryReducedDatum.objects.get_or_create(
+                        target=target,
+                        bandpass='G',
                         timestamp=jd.to_datetime(timezone=TimezoneInfo()),
-                        value=value,
-                        source_name=self.name,
-                        source_location=alert_url,
-                        data_type='photometry',
-                        target=target)
+                        defaults={
+                            'brightness': float(phot_data[2]),
+                            'source_name': self.name,
+                            'source_location': alert_url,
+
+                        }
+                    )
 
         return

tom_alerts/tests/brokers/test_alerce.py

@@ -7,7 +7,7 @@
 from faker import Faker
 
 from tom_alerts.brokers.alerce import ALeRCEBroker, ALeRCEQueryForm
-from tom_dataproducts.models import ReducedDatum
+from tom_dataproducts.models import PhotometryReducedDatum
 from tom_targets.models import Target
 from tom_targets.tests.factories import SiderealTargetFactory
 
@@ -401,7 +401,7 @@ def test_process_reduced_datum(self, mock_fetch_lightcurve):
         mock_fetch_lightcurve.return_value = test_data
         target = SiderealTargetFactory()
         ALeRCEBroker().process_reduced_data(target)
-        self.assertEqual(ReducedDatum.objects.count(), 2)
+        self.assertEqual(PhotometryReducedDatum.objects.count(), 2)
 
     def test_to_generic_alert(self):
         """Test to_generic_alert broker method."""