diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 68722b67..3f1c856d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-ver: ['3.9', '3.13'] + python-ver: ['3.10', '3.13'] steps: - uses: actions/checkout@v6 diff --git a/.github/workflows/docker-deploy.yml b/.github/workflows/docker-deploy.yml index b7a3ea5b..2ea29a36 100644 --- a/.github/workflows/docker-deploy.yml +++ b/.github/workflows/docker-deploy.yml @@ -14,7 +14,7 @@ jobs: with: submodules: 'recursive' - name: Log in to Docker Hub - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: username: ${{ vars.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} @@ -22,4 +22,6 @@ jobs: - name: Build image and push to Docker Hub run: | docker build -t ${{ vars.DOCKER_USERNAME }}/neurocaps:${{ github.ref_name }} -f docker/Dockerfile . && - docker push ${{ vars.DOCKER_USERNAME }}/neurocaps:${{ github.ref_name }} + docker push ${{ vars.DOCKER_USERNAME }}/neurocaps:${{ github.ref_name }} && + docker tag ${{ vars.DOCKER_USERNAME }}/neurocaps:${{ github.ref_name }} ${{ vars.DOCKER_USERNAME }}/neurocaps:latest && + docker push ${{ vars.DOCKER_USERNAME }}/neurocaps:latest diff --git a/.github/workflows/minimum_dependencies.yml b/.github/workflows/minimum_dependencies.yml index 1fffe2ce..e850b154 100644 --- a/.github/workflows/minimum_dependencies.yml +++ b/.github/workflows/minimum_dependencies.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] - python-ver: ['3.9'] + python-ver: ['3.10'] name: Python ${{ matrix.python-ver }} check on ${{ matrix.os }} steps: diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml index bab171c1..9cf25a11 100644 --- a/.github/workflows/precommit.yml +++ b/.github/workflows/precommit.yml @@ -17,7 +17,7 @@ jobs: fetch-depth: 0 - uses: actions/setup-python@v6 with: - python-version: '3.9' + python-version: '3.10' - name: Install precommit run: | diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index e7ef6868..7f47401e 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 with: - python-version: '3.9' + python-version: '3.10' - name: Install dependencies run: | diff --git a/.github/workflows/testing.yaml b/.github/workflows/testing.yaml index 68af7881..85f64015 100644 --- a/.github/workflows/testing.yaml +++ b/.github/workflows/testing.yaml @@ -44,7 +44,7 @@ jobs: shell: bash - name: Install pytest-forked for Ubuntu Python 3.11+ - if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-ver != '3.9' && matrix.python-ver != '3.10' }} + if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-ver != '3.10' }} run: pip install pytest-forked shell: bash @@ -64,19 +64,12 @@ jobs: run: pytest tests/ --cov neurocaps -v - name: Run tests without coverage for Ubuntu on Python 3.11-3.14 (forked) - if: ${{ matrix.os == 'ubuntu-latest' && (matrix.python-ver == '3.11' || matrix.python-ver == '3.12' || matrix.python-ver == '3.13' || matrix.python-ver == '3.14') }} + if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-ver != '3.10' }} uses: coactions/setup-xvfb@v1 with: options: -screen 0 1600x1200x24 run: pytest tests/ --forked -v - - name: Run tests without coverage for Ubuntu on Python 3.9 - if: ${{ matrix.os == 'ubuntu-latest' && (matrix.python-ver == '3.9') }} - uses: coactions/setup-xvfb@v1 - with: - options: -screen 0 1600x1200x24 - run: pytest tests/ -v - - name: Run tests without coverage for Mac if: ${{ matrix.os == 'macos-latest' }} run: pytest . -v @@ -91,6 +84,6 @@ jobs: - name: Upload coverage reports to Codecov for Ubuntu if: ${{ matrix.os == 'ubuntu-latest' && matrix.python-ver == '3.10' }} - uses: codecov/codecov-action@v5.5.2 + uses: codecov/codecov-action@v6.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1891c067..b7a9c368 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -12,7 +12,7 @@ repos: - id: mixed-line-ending - repo: https://github.com/psf/black - rev: 25.1.0 + rev: 26.3.1 hooks: - id: black args: [--line-length=100] @@ -21,15 +21,16 @@ repos: - black[jupyter] - repo: https://github.com/adamchainz/blacken-docs - rev: 1.19.1 + rev: 1.20.0 hooks: - id: blacken-docs args: [--line-length=90] + exclude: 'archives/' additional_dependencies: - black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.6 + rev: v0.15.8 hooks: - id: ruff # Ensure no print statements are in the codebase; only logging allowed diff --git a/CHANGELOG.md b/CHANGELOG.md index cb67a9af..728a6f9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,15 @@ noted in the changelog (e.g., new functions or parameters, changes in parameter improvements/enhancements. All fixes and modifications are backwards compatible. - *.postN* : Consists of documentation changes or metadata-related updates, such as modifications to type hints. +## [0.37.4] - 2026-05-09 +### ♻ Changed +- ``high_pass`` is now automatically set to ``None`` with a warning when cosine regressors are detected in user-specified `confound_names`, preventing multicollinearity between nilearn's DCT basis functions and fMRIPrep's cosine regressors. + +## [0.37.3] - 2026-04-16 +### ♻ Changed +- Added support for the ``NEUROCAPS_DATA`` environment variable to override the default data directory location used by fetch_preset_parcel_approach. Defaults to "~/neurocaps_data" when unset. +- Dropped Python 3.9 support + ## [0.37.2] - 2026-02-19 ### 💻 Metadata - Update metadata on Pypi diff --git a/README.md b/README.md index 789d04e7..6c064312 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Latest Version](https://img.shields.io/pypi/v/neurocaps.svg)](https://pypi.python.org/pypi/neurocaps/) [![Python Versions](https://img.shields.io/pypi/pyversions/neurocaps.svg)](https://pypi.python.org/pypi/neurocaps/) -[![DOI](https://img.shields.io/badge/DOI-10.5281%2Fzenodo.11642615-teal)](https://doi.org/10.5281/zenodo.18529846) +[![DOI](https://img.shields.io/badge/DOI-10.5281%2Fzenodo.11642615-teal)](https://doi.org/10.5281/zenodo.19602764) [![Test Status](https://github.com/donishadsmith/neurocaps/actions/workflows/testing.yaml/badge.svg)](https://github.com/donishadsmith/neurocaps/actions/workflows/testing.yaml) [![Documentation Status](https://readthedocs.org/projects/neurocaps/badge/?version=stable)](http://neurocaps.readthedocs.io/en/stable/?badge=stable) [![codecov](https://codecov.io/github/donishadsmith/neurocaps/branch/main/graph/badge.svg?token=WS2V7I16WF)](https://codecov.io/github/donishadsmith/neurocaps) @@ -18,7 +18,7 @@ k-means clustering on BOLD timeseries data [^1].

## Installation -**Requires Python 3.9-3.14.** +**Requires Python 3.10-3.14.** ### Standard Installation ```bash diff --git a/archives/CHANGELOG-v0.md b/archives/CHANGELOG-v0.md index 7c767153..45b24aa9 100644 --- a/archives/CHANGELOG-v0.md +++ b/archives/CHANGELOG-v0.md @@ -1,1504 +1,1500 @@ -# Changelog - -All notable future changes to neurocaps will be documented in this file. - -*Note*: All versions in this file are deployed on PyPi. - -## [Versioning] - -**Beyond version 0.10.0, versioning for the 0.x.x series for this package will work as:** - -`0.minor.patch.postN` - -- *.minor* : Introduces new features and may include potential breaking changes. Any breaking changes will be explicitly -noted in the changelog (i.e new functions or parameters, changes in parameter defaults or function names, etc). -- *.patch* : Will contain fixes for any identified bugs, may include modifications or an added parameter for -improvements/enhancements. Fixes and modifications will be backwards compatible. -- *.postN* : Consists of only metadata-related changes, such as updates to type hints or doc strings/documentation. - -## [0.18.11] - 2024-11-27 -### 🐛 Fixes -- An error in a setter method that did not use `raise`. - -## [0.18.10] - 2024-11-27 -### 🚀 New/Added -- Added deleter method for `subject_timeseries` and `concatenated_timeseries` properties - -## [0.18.9] - 2024-11-25 -### 🚀 New/Added -- Custom error to warn about querying issues -- Add optional dependency for demo -### 🐛 Fixes -- Documentation rendering issues -- Restrict from downloading the latest vtk 9.4.0 for `caps2surf` to work. - -## [0.18.8.post0] - 2024-11-23 -### 📖 Documentation -- Update to documentation to show example directory structure. - -## [0.18.8] - 2024-11-22 -### 🚀 New/Added -- Added "use_sample_mask" key for `fd_threshold` parameter, which if set to True, generates a sample mask to pass -to nilearn's `NiftiLabelsMasker` for censoring prior to nuisance regression. -Internally, the ``clean__extrapolate`` parameter is used to set extrapolate to False. If condition filtering is -requested, when "use_sample_mask" key for `fd_threshold` parameter is True, then the truncated timeseries is -temporarily padded to ensure the correct indices corresponding to the condition are obtained. -- Added new property. - -## [0.18.7] - 2024-11-18 -### 🐛 Fixes -- Fixes projection of CAPs onto NiFTI atlas by preventing in-place modification. Previously, if a new CAP value matched -a subsequent atlas label ID, it could cause incorrect coordinate assignments. - -## [0.18.6] - 2024-11-18 -- Minor code cleaning -### 📖 Documentation -- Readme example fix - -## [0.18.5] - 2024-11-16 -### ♻ Changed -- Updated Dependencies: - - NumPy: version 2.0 and above can be installed. - - BrainSpace: requires version 0.1.16 and above. - -## [0.18.4] - 2024-11-15 -### 🐛 Fixes -- Corrected region names for version "3v2" of the AAL atlas. -### ♻ Changed -- Added a specific logged warning when no confound names are found. If some confound names are missing, -they will still be listed accordingly. -- Added a specific logged warnings for methods in `CAP` that use the `runs` parameter. Warnings are issued if a subject -is missing any requested run, with an additional warning if all runs are missing. - -## [0.18.3.post0] - 2024-11-14 -### 📖 Documentation -- Added reference for `merge_dicts`. - -## [0.18.3.post0] - 2024-11-10 -### 📖 Documentation -- Significant documentation revisions for clarity and precision. -- Also, updates outdated documentation for `CAP.get_caps` to clarify that figure generation and saving parameters can -be used with any `cluster_selection_method`, not just specific ones as previously implied. - -## [0.18.3] - 2024-11-10 -### 🐛 Fixes -- More conservative maxsizes for `@lru_cache`, change `@cache` in `TimeseriesExtractor` to `@lru_cache`. -- Clean unused import. - -## [0.18.2] - 2024-11-09 -- A simple pre commit hook added to remove a few trailing whitespace, add new lines, etc. - -### ♻ Changed -- Add's `_get_target_indices` and `_build_tree` to the init file for a shorter import path if cache needs to be cleared or -assessed. essentially allows: - -New import: -```python -from neurocaps._utils import _build_tree, _get_target_indices -``` - -Previous import: -```python -from neurocaps._utils.analysis.cap2statmap import ( - _build_tree, - _get_target_indices, -) -``` - -### 📖 Documentation -- Very minor doc fix. - -## [0.18.1] - 2024-11-08 -### 🚀 New/Added -- Added "cbarlabels_size" kwarg to several plots to allow the font size of the colorbar labels to increase or decrease. -### ♻ Changed -- For `CAP.caps2radar`, when the `legend` kwarg is set to None, the legend will be removed entirely. - -## [0.18.0] - 2024-11-07 -### ♻ Changed -- In `TimeseriesExtractor.get_bold`, location of `parallel_log_config` parameter in function signature moved -from being the last parameter to underneath `n_cores`. Additionally, `exclude_niftis` moved from being the second to last -parameter to being underneath `exclude_subjects`. - -*Changes related to `knn_dict`, which is only relevant for certain atlases that project poorly to surface space or -has a sparsity issue* - -- Added a "reference_atlas" key to allow Schaefer or AAL to be used as the reference atlas. -- The "remove_subcortical" key changed to "remove_labels". -- Default "k" from 1 to 3. - -### 🐛 Fixes -*Fixes only related to `knn_dict`* - -- "remove_labels" now only removes the labels from being interpolated as opposed to removing the label from being -interpolated in addition to removing the corresponding indices from the atlas entirely. -- Certain internal helper functions - `_get_target_indices` and `_build_tree` - from -`neurocaps._utils.analyis.cap2statmap` now use functool's `lru_cache` decorator so that the indices that the -non-background coordinated that need interpolation as well as the indices that don't need interpolation, based on -"remove_labels" are only computed once per session for every unique parameter combination. -- Logged information related to `knn_dict` appears once per call of `CAP.caps2niftis` or `CAP.caps2surf` instead of -for every iteration performed within these functions. - -### 📖 Documentation -- Some documentation revisions. - -## [0.17.11.post1] - 2024-11-04 -### 📖 Documentation -- Documentation clarification - -## [0.17.11.post0] - 2024-11-03 -### 📖 Documentation -- Minor fix to a versionchanged directive - -## [0.17.11] - 2024-11-03 -### ♻ Changed -- In `CAP.calculate_metrics`, if `continuous_runs` used, then in the "Run" column, the labels for these runs will -now follow the "run-" format and has changed from "continuous_runs" to run-continuous" -- Also in `CAP.calculate_metrics`, the group labels in the dataframe will no longer replace whitespace with underline. -Do names such as "High ADHD" wil no longer change to "High_ADHD" in the dataframe. -- The averaged transition matrix dataframe from `transition_matrix` now contains to index name "From\To" to show that -the index CAPs are "From" and the column CAPs are "To". -### 🐛 Fixes -- More robust error checking to ensure that the ``subject_timeseries`` follows the correct format when the -``subject_timeseries`` setter property is used in ``TimeseriesExtractor``. -- For ``space`` setter property is used in ``TimeseriesExtractor``, checks to ensure it is a string. - -## [0.17.10] - 2024-11-02 -### ♻ Changed -- Logger names now use `__name__` instead of `__name__.split(".")[-1]` -- Module folder and file naming in `neurocaps._utils` changed, only `_utils` has the leading underscore. -- Default logger now includes logger name in logged message. -### 🐛 Fixes -- Prevents logging duplication in certain user-defined logging scenarios when logs redirected. - -## [0.17.9.post0] - 2024-11-02 -### 📖 Documentation -- Minor doc changes - -## [0.17.9] - 2024-10-31 -### 📖 Documentation -- Enhanced documentation -- Documentation renders properly with Pylance in VSCode. -### ♻ Changed -- The ``parcel_approach`` stored as an property will remove the initial configuration sub-keys parameters ("n_rois", -"resolution_mm", "yeo_networks" for Schaefer and "version" for AAL) after the information from these sub-keys are used -internally to retrieve the appropriate parcellation. Thus, the dictionary will retain the primary keys: -"maps", "nodes", and "regions". However, similar to previous behavior, additional sub-keys can still be added, such as -adding a sub-key consisting of metadata, but they will be ignored and won't affect core operation. -- When ``parcel_approach`` is included as a setter, or the required keys ("maps", "nodes", and "regions") -are detected, it will no longer reset the "maps", "nodes", and "regions" for "Schaefer" and "AAL". This allows the -"maps", "nodes", and "regions" to be modified for "Schaefer" and "AAL" if needed. - -## [0.17.8.post1] - 2024-10-30 -### 💻 Metadata -- Readme revisions on Pypi. - -## [0.17.8.post0] - 2024-10-30 -### 💻 Metadata -- Shortens Readme on Pypi. - -## [0.17.8] - 2024-10-29 -### 🚀 New/Added -- For `TimeseriesExtractor.get_bold()`, a new `parallel_log_config` parameter has been added to pass an instance of -`multiprocessing.Manager.Queue` to redirect logs if parallel processing is used. -### ♻ Changed -- For `TimeseriesExtractor.get_bold()`, logging of additional subject-specific messages are also controlled by `verbose`. - -## [0.17.7] - 2024-10-25 -### 🚀 New/Added -- Added `vmin` and `vmin` kwargs for `transition_matrix` and `CAP.caps2corr`. -### 🐛 Fixes -- Fix issue with x and y labels for `CAP.caps2plot` not changing in size when `xticklabels_size` and -`yticklabels_size` modified. - -## [0.17.6] - 2024-10-19 -### 🚀 New/Added -- Added "n_before" and "n_after" subkeys for when `fd_threshold` is a dictionary. This the frame exceeding the -fd threshold to be scrubbed including an additional "n" number of frames before or after the exceeding frame to also -be scrubbed. -### 🐛 Fixes -- Log message that specifies certain confounds not found is now set as a warning instead of info. Additionally, there -is no Nonetype error if zero confounds are found in the data frame. -- Fix that appears to have only affected Windows where the system defined a default root handler (stderr) after the -top level loggers were initialized. This caused loggers initialized inside functions, such as `_extract_timeseries` -to adopt this root handler and timeseries extraction logs had a slightly different formatting. This is now fixed with -the following internal code. - -```python -"""Internal logging function and class for flushing""" - -import logging, sys - -# Global variables to determine if a handler is user defined or defined by OS -_USER_ROOT_HANDLER = None -_USER_MODULE_HANDLERS = {} - - -class _Flush(logging.StreamHandler): - def emit(self, record): - super().emit(record) - self.flush() - - -def _logger(name, flush=False, top_level=True): - global _USER_ROOT_HANDLER, _USER_MODULE_HANDLERS - - logger = logging.getLogger(name.split(".")[-1]) - - # Windows appears to assign stderr has the root handler after the top level loggers are assigned, which causes - # any loggers not assigned at top level to adopt this handler. Global variable used to assess if the base root - # handler is user defined or assigned by the system - if top_level == True: - _USER_ROOT_HANDLER = logging.getLogger().hasHandlers() - _USER_MODULE_HANDLERS[logger.name] = logging.getLogger(logger.name).hasHandlers() - - if not logger.level: - logger.setLevel(logging.INFO) - - # Check if user defined root handler or assigned a specific handler for module - default_handlers = _USER_ROOT_HANDLER or _USER_MODULE_HANDLERS[logger.name] - - # Works to see if root has handler and propagate if it does - logger.propagate = True if _USER_ROOT_HANDLER else False - - # Add or messages will repeat several times due to multiple handlers if same name used - if not default_handlers and not (logger.name == "_extract_timeseries" and top_level): - # If no user specified default handler, any handler is assigned by OS and is cleared - if logger.name == "_extract_timeseries": - logger.handlers.clear() - - if flush: - handler = _Flush(sys.stdout) - else: - handler = logging.StreamHandler(sys.stdout) - - handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) - logger.addHandler(handler) - - return logger -``` - -## [0.17.5] - 2024-10-14 -### 🚀 New/Added -- Added `dtype` parameter to `TimeseriesExtractor` with default set to None. This parameter is passed to nilearn's -`load_img` function. -### ♻ Changed -- Some speed increase if calling `TimeseriesExtractor.get_bold` multiple times in the same script. New internal function -uses `functools` `@cache` when calling `BIDSLayout` now. - -```python -@staticmethod -@cache -def _call_layout(bids_dir, pipeline_name): - try: - import bids - except ModuleNotFoundError: - raise ModuleNotFoundError( - "This function relies on the pybids package to query subject-specific files. " - "If on Windows, pybids does not install by default to avoid long path error issues " - "during installation. Try using `pip install pybids` or `pip install neurocaps[windows]`." - ) - - bids_dir = os.path.normpath(bids_dir).rstrip(os.path.sep) - - if bids_dir.endswith("derivatives"): - bids_dir = os.path.dirname(bids_dir) - - if pipeline_name: - pipeline_name = ( - os.path.normpath(pipeline_name).lstrip(os.path.sep).rstrip(os.path.sep) - ) - if pipeline_name.startswith("derivatives"): - pipeline_name = pipeline_name[len("derivatives") :].lstrip(os.path.sep) - layout = bids.BIDSLayout( - bids_dir, - derivatives=os.path.join(bids_dir, "derivatives", pipeline_name), - ) - else: - layout = bids.BIDSLayout(bids_dir, derivatives=True) - - LG.info(f"{layout}") - - return layout -``` - -## [0.17.4] - 2024-10-12 -- `CAP.caps2radar` test for Github Actions added due to this [action](https://github.com/coactions/setup-xvfb) -resolving the VTK issue. Test only for Linux. Now all functions are tested on Github Actions. -### 🐛 Fixes -- `CAP.caps2radar` uses try except block to show figures and should be able to display and close figures in a -Jupyter notebook or python CL. -- In `CAP.caps2radar` "round" kwarg was never used in code and was removed. -- When tick values are not specified in the `radialaxis` kwargs in `CAP.caps2radar` function, default tick values -no longer produces an error and now does the following: - -```python -if "tickvals" not in plot_dict["radialaxis"] and "range" not in plot_dict["radialaxis"]: - default_ticks = [ - max_value / 4, - max_value / 2, - 3 * max_value / 4, - max_value, - ] - plot_dict["radialaxis"]["tickvals"] = [round(x, 2) for x in default_ticks] -``` -### ♻ Changed -- To prevent numerical stability issues when scaling, previous versions did the following when the standard deviation -for a column was essentially zero: - -```python -std = np.std(arr, axis=0, ddof=1) -eps = np.finfo(np.float64).eps -std[std < eps] = 1.0 -``` - -The same method is employed; however, the smallest positive float representation used now considers the dtype -instead of being set to the smallest representation for "float64": -```python -std = np.std(arr, axis=0, ddof=1) -eps = np.finfo(std.dtype).eps -std[std < eps] = 1.0 -``` - -## [0.17.3] - 2024-10-08 -### 🐛 Fixes -- Fixes specific error that occurs when using a suffix name and saving nifti in `CAP.caps2surf`. - -## [0.17.2.post0] - 2024-10-06 -### 💻 Metadata -- Minor clarification in `CAP.caps2radar` function - -## [0.17.2] - 2024-10-06 -### ♻ Changed -- Internal refactoring and minor change to saved filenames for some functions for consistency. - -## [0.17.1] - 2024-10-01 -### ♻ Changed -- The `CAP.caps2radar` function now calculates the cosine similarity to the positive and negative activations of -a CAP cluster centroid separately. Each region has two traces (one for cosine similarity with positive activation -and one for cosine similarity with negative calculations). The plots should be easier to interpret and aligns better -with visualizations in CAP research. - -## [0.17.0] - 2024-09-21 -### 🚀 New/Added -- In `CAP.caps2radar`, "round" (rounds to three decimal points by default) and "linewidth" kwargs added. -### ♻ Changed -- In `TimeseriesExtractor`, `parcel_approach` and `fwhm` have changed positions. `parcel_approach` is second in the -list and `fwhm` is seventh in the list. -- `flush_print` changed to `flush`. -- In `CAP.caps2radar`, both `method` and `alpha` removed. Only the traditional cosine similarity calculation is -computed. -- In `CAP.calculate_metrics`, calculation for counts changed to abide by the formula, -temporal fraction = (persistence * counts)/total volumes which can be found in the supplemental of -Yang et al., 2021](https://doi.org/10.1016/j.neuroimage.2021.118193). Counts is now the frequency of initiations -of a specific CAP. -### 💻 Metadata -- Version directives less than 0.16.0 removed. - -## [0.16.5] - 2024-09-16 -- This update exclusively relates to improving documentation as well as improving the language in the error and -information messages for clarity. For instance, when a subject is skipped during timeseries extraction, instead of -`"[SUBJECT: 01 | SESSION: 002 | TASK: rest] Processing skipped: {message}"` it is now -`"[SUBJECT: 01 | SESSION: 002 | TASK: rest] Timeseries Extraction Skipped: {message}"`. Language in primarily in some -function descriptions have also been included. - -## [0.16.4] - 2024-09-16 -### ♻ Changed -- All uses of `print` and `warnings.warn` in package replaced with `logging.info` and `logging.warning`. The internal -function that creates the logger: -```python -import logging, sys - - -class _Flush(logging.StreamHandler): - def emit(self, record): - super().emit(record) - self.flush() - - -def _logger(name, level=logging.INFO, flush=False): - logger = logging.getLogger(name.split(".")[-1]) - logger.setLevel(level) - # Works to see if root has handler and propagate if it does - logger.propagate = logging.getLogger().hasHandlers() - # Add or messages will repeat several times due to multiple handlers if same name used - if not logger.hasHandlers(): - if flush: - handler = _Flush(sys.stdout) - else: - handler = logging.StreamHandler(sys.stdout) - handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) - logger.addHandler(handler) - - return logger -``` -- **Note**: The logger is initialized within the [internal time series extraction function]((https://github.com/donishadsmith/neurocaps/blob/900bf7a89d3ff16a8dd91310c8d177c5b5d6de8a/neurocaps/_utils/_timeseriesextractor_internals/_extract_timeseries.py#L12)) to ensure that each child -process has its own independent logger. This guarantees that subject-level information and warnings will be properly -logged, regardless of whether parallel processing is used or not. - -For non-parallel processing, the logger can be configured by a user with a command like the following: - -```python -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - handlers=[ - logging.StreamHandler(sys.stdout), - logging.FileHandler("info.out"), - ], -) -``` - -- Subject-specific messages are now more compact. - -**OLD:** -``` -List of confound regressors that will be used during timeseries extraction if available in confound dataframe: Cosine*, Rot*. - -BIDS Layout: ...0.4_ses001-022/ds000031_R1.0.4 | Subjects: 1 | Sessions: 1 | Runs: 1 - -[SUBJECT: 01 | SESSION: 002 | TASK: rest | RUN: 001] ----------------------------------------------------- -Preparing for timeseries extraction using - [FILE: '/Users/runner/work/neurocaps/neurocaps/tests/ds000031_R1.0.4_ses001-022/ds000031_R1.0.4/derivatives/fmriprep_1.0.0/fmriprep/sub-01/ses-002/func/sub-01_ses-002_task-rest_run-001_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz'] - -[SUBJECT: 01 | SESSION: 002 | TASK: rest | RUN: 001] ----------------------------------------------------- -The following confounds will be for nuisance regression: Cosine00, Cosine01, Cosine02, Cosine03, Cosine04, Cosine05, Cosine06, RotX, RotY, RotZ, aCompCor02, aCompCor03, aCompCor04, aCompCor05. -``` - -**NEW:** -``` -2024-09-16 00:17:11,689 [INFO] List of confound regressors that will be used during timeseries extraction if available in confound dataframe: Cosine*, aComp*, Rot*. -2024-09-16 00:17:12,113 [INFO] BIDS Layout: ...0.4_ses001-022/ds000031_R1.0.4 | Subjects: 1 | Sessions: 1 | Runs: 1 -2024-09-16 00:17:13,914 [INFO] [SUBJECT: 01 | SESSION: 002 | TASK: rest | RUN: 001] Preparing for timeseries extraction using [FILE: sub-01_ses-002_task-rest_run-001_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz]. -2024-09-16 00:17:13,917 [INFO] [SUBJECT: 01 | SESSION: 002 | TASK: rest | RUN: 001] The following confounds will be for nuisance regression: Cosine00, Cosine01, Cosine02, Cosine03, Cosine04, Cosine05, Cosine06, aCompCor00, aCompCor01, aCompCor02, aCompCor03, aCompCor04, aCompCor05, RotX, RotY, RotZ. -``` -*Note that only the absolute path is no longer outputted, only the file's basename* -*Jupyter Notebook may show an additional space between the "[" and "INFO" for subject level info* - -## [0.16.3.post0] - 2024-09-14 -### 💻 Metadata -- Uploading fixed readme to Pypi - -## [0.16.3] - 2024-09-14 -- Internal refactoring was completed, primarily in `CAPs.caps2plot`, `TimeseriesExtractor.get_bold`, and an -internal function `_extract_timeseries`. -- All existing pytest tests passed following the refactoring. -### 🐛 Fixes -- Minor improvements were made to error messages for better clarity. -- Annotations can now be specified for `CAP.caps2plot` regional heatmap. - -## [0.16.2.post1] - 2024-08-23 -### 💻 Metadata -- Fix truncated table in README, which did not show all values correctly due to missing an additional row header. - -## [0.16.2] - 2024-08-22 -### 🚀 New/Added -- Transition probabilities has been added to `CAP.calculate_metrics`. Below is a snippet from the codebase -of how the calculation is done. -```python -if "transition_probability" in metrics: - temp_dict[group].loc[len(temp_dict[group])] = [ - subj_id, - group, - curr_run, - ] + [ - 0.0 - ] * (temp_dict[group].shape[-1] - 3) - # Get number of transitions - trans_dict = { - target: np.sum( - np.where( - predicted_subject_timeseries[subj_id][curr_run][:-1] == target, - 1, - 0, - ) - ) - for target in group_caps[group] - } - indx = temp_dict[group].index[-1] - # Iterate through products and calculate all symmetric pairs/off-diagonals - for prod in products_unique[group]: - target1, target2 = prod[0], prod[1] - trans_array = predicted_subject_timeseries[subj_id][curr_run].copy() - # Set all values not equal to target1 or target2 to zero - trans_array[(trans_array != target1) & (trans_array != target2)] = 0 - trans_array[np.where(trans_array == target1)] = 1 - trans_array[np.where(trans_array == target2)] = 3 - # 2 indicates forward transition target1 -> target2; -2 means reverse/backward transition target2 -> target1 - diff_array = np.diff(trans_array, n=1) - # Avoid division by zero errors and calculate both the forward and reverse transition - if trans_dict[target1] != 0: - temp_dict[group].loc[indx, f"{target1}.{target2}"] = float( - np.sum(np.where(diff_array == 2, 1, 0)) / trans_dict[target1] - ) - if trans_dict[target2] != 0: - temp_dict[group].loc[indx, f"{target2}.{target1}"] = float( - np.sum(np.where(diff_array == -2, 1, 0)) / trans_dict[target2] - ) - - # Calculate the probability for the self transitions/diagonals - for target in group_caps[group]: - if trans_dict[target] == 0: - continue - # Will include the {target}.{target} column, but the value is initially set to zero - columns = temp_dict[group].filter(regex=rf"^{target}\.").columns.tolist() - cumulative = temp_dict[group].loc[indx, columns].values.sum() - temp_dict[group].loc[indx, f"{target}.{target}"] = 1.0 - cumulative -``` -Below is a simplified version of the above snippet. -```python -import itertools, math, pandas as pd, numpy as np - -groups = [["101", "A", "1"], ["102", "B", "1"]] -timeseries_dict = { - "101": np.array([1, 1, 1, 1, 2, 2, 1, 4, 3, 5, 3, 3, 5, 5, 6, 7]), - "102": np.array([1, 2, 1, 1, 3, 3, 1, 4, 3, 5, 3, 3, 4, 5, 6, 8, 7]), -} -caps = list(range(1, 9)) -# Get all combinations of transitions -products = list(itertools.product(caps, caps)) -df = pd.DataFrame( - columns=["Subject_ID", "Group", "Run"] + [f"{x}.{y}" for x, y in products] -) -# Filter out all reversed products and products with the self transitions -products_unique = [] -for prod in products: - if prod[0] == prod[1]: - continue - # Include only the first instance of symmetric pairs - if (prod[1], prod[0]) not in products_unique: - products_unique.append(prod) - -for info in groups: - df.loc[len(df)] = info + [0.0] * (df.shape[-1] - 3) - timeseries = timeseries_dict[info[0]] - # Get number of transitions - trans_dict = { - target: np.sum(np.where(timeseries[:-1] == target, 1, 0)) for target in caps - } - indx = df.index[-1] - # Iterate through products and calculate all symmetric pairs/off-diagonals - for prod in products_unique: - target1, target2 = prod[0], prod[1] - trans_array = timeseries.copy() - # Set all values not equal to target1 or target2 to zero - trans_array[(trans_array != target1) & (trans_array != target2)] = 0 - trans_array[np.where(trans_array == target1)] = 1 - trans_array[np.where(trans_array == target2)] = 3 - # 2 indicates forward transition target1 -> target2; -2 means reverse/backward transition target2 -> target1 - diff_array = np.diff(trans_array, n=1) - # Avoid division by zero errors and calculate both the forward and reverse transition - if trans_dict[target1] != 0: - df.loc[indx, f"{target1}.{target2}"] = float( - np.sum(np.where(diff_array == 2, 1, 0)) / trans_dict[target1] - ) - if trans_dict[target2] != 0: - df.loc[indx, f"{target2}.{target1}"] = float( - np.sum(np.where(diff_array == -2, 1, 0)) / trans_dict[target2] - ) - - # Calculate the probability for the self transitions/diagonals - for target in caps: - if trans_dict[target] == 0: - continue - # Will include the {target}.{target} column, but the value is initially set to zero - columns = df.filter(regex=rf"^{target}\.").columns.tolist() - cumulative = df.loc[indx, columns].values.sum() - df.loc[indx, f"{target}.{target}"] = 1.0 - cumulative -``` -- Added new external function - ``transition_matrix``, which generates and visualizes the average transition probabilities -for all groups, using the transition probability dataframe outputted by `CAP.calculate_metrics` - -## [0.16.1.post3] - 2024-08-07 -### 💻 Metadata -- Minor change to clarify the language in the docstring referring to the Custom parcellation approach and update readme -on PyPi for the installation instructions. - -## [0.16.1.post2] - 2024-08-06 -### 💻 Metadata -- Correct output for example in readme. - -## [0.16.1.post1] - 2024-08-06 -### 💻 Metadata -- Update outdated example in readme. - -## [0.16.1] - 2024-08-06 -### ♻ Changed -- For `knn_dict`, cKdtree is replaced with Kdtree and scipy is restricted to 1.6.0 or later since that is the version -were Kdtree used the C implementation. -- `TimeseriesExtractor.get_bold` can now be used on Windows, pybids still does not install by default to prevent -long path error but `pip install neurocaps[windows]` can be used for installation. -- All instances of textwrap replaced with normal strings, printed warnings or messages will be longer in length now -and occupies less vertical screen space. - -## [0.16.0] - 2024-07-31 -### ♻ Changed -- In `CAP.caps2surf`, the `save_stat_map` parameter has been changed to `save_stat_maps`. -- Slight improvements in a few errors/exceptions to improve their informativeness. -- Now, when a subject's run is excluded due to exceeding the fd threshold, the percentage of their volumes -exceeding the threshold is given as opposed to simply stating that they have been excluded. -### 🐛 Fixes -- Fix a specific instance when `tr` is not specified for `TimseriesExtractor.get_bold`. When the `tr` is not specified, -the code attempts to check the the bold metadata/json file in the derivatives directory to extract the -repetition time. Now, it will check for this file in both the derivatives and root bids dir. The code will also -raise an error earlier if the tr isn't specified, cannot be extracted from the bold metadata file, and bandpass filtering -is requested. -- A warning check that is done to assess if indices for a certain condition is outside a possible range due to -duration mismatch, incorrect tr, etc is now also done before calculating the percentage of volumes exceeding the threshold -to not dilute calculations. Before this check was only done before extracting the condition from the timeseries array. -### 💻 Metadata -- Very minor documentation updates for `TimseriesExtractor.get_bold`. - -## [0.15.2] - 2024-07-23 -### ♻ Changed -- Created a specific message when dummy_scans = {"auto": True} and zero "non_steady_state_outlier_XX" are found -when `verbose=True`. -- Regardless if `parcel_approach`, whether used as a setter or input, accepts pickles. -### 🐛 Fixes -Fixed a reference before assignment issue in `merge_dicts`. This occurred when only the merged dictionary was requested -to be saved without saving the reduced dictionaries, and no user-provided file_names were given. In this scenario, -the default name for the merged dictionary is now correctly used. - -## [0.15.1] - 2024-07-23 -### 🚀 New/Added -- In `TimeseriesExtractor`, "min" and "max" sub-keys can now be used when `dummy_scans` is a dictionary and the -"auto" sub-key is True. The "min" sub-key is used to set the minimum dummy scans to remove if the number of -"non_steady_state_outlier_XX" columns detected is less than this value and the "max" sub-key is used to set the -maximum number of dummy scans to remove if the number of "non_steady_state_outlier_XX" columns detected exceeds this -value. - -## [0.15.0] - 2024-07-21 -### 🚀 New/Added -- `save_reduced_dicts` parameter to `merge_dicts` so that the reduced dictionaries can also be saved instead of only -being returned. - -### ♻ Changed -- Some parameter names, inputs, and outputs for non-class functions - `merge_dicts`, `change_dtype`, and `standardize` -have changed to improve consistency across these functions. - - `merge_dicts` - - `return_combined_dict` has been changed to `return_merged_dict`. - - `file_name` has been changed to `file_names` since the reduced dicts can also be saved now. - - Key in output dictionary containing the merged dictionary changed from "combined" to "merged". - - `standardize` & `change_dtypes` - - `subject_timeseries` has been changed to `subject_timeseries_list`, the same as in `merge_dicts`. - - `file_name` has been changed to `file_names`. - - `return_dict` has been changed to `return_dicts`. -- The returned dictionary for `merge_dicts`, `change_dtype`, and `standardize` is only -`dict[str, dict[str, dict[str, np.ndarray]]]` now. - -- In `CAP.calculate_metrics`, the metrics calculations, except for "temporal_fraction" have been refactored to remove an -import or use numpy operations to reduce needed to create the same calculation. - - **"counts"** - - Previous Code: - ```python - # Get frequency - frequency_dict = dict( - collections.Counter(predicted_subject_timeseries[subj_id][curr_run]) - ) - # Sort the keys - sorted_frequency_dict = {key: frequency_dict[key] for key in sorted(list(frequency_dict))} - # Add zero to missing CAPs for participants that exhibit zero instances of a certain CAP - if len(sorted_frequency_dict) != len(cap_numbers): - sorted_frequency_dict = { - cap_number: ( - sorted_frequency_dict[cap_number] - if cap_number in list(sorted_frequency_dict) - else 0 - ) - for cap_number in cap_numbers - } - # Replace zeros with nan for groups with less caps than the group with the max caps - if len(cap_numbers) > group_cap_counts[group]: - sorted_frequency_dict = { - cap_number: ( - sorted_frequency_dict[cap_number] - if cap_number <= group_cap_counts[group] - else float("nan") - ) - for cap_number in cap_numbers - } - ``` - - Refactored Code: - ```python - # Get frequency; - frequency_dict = { - key: np.where(predicted_subject_timeseries[subj_id][curr_run] == key, 1, 0).sum() - for key in range(1, group_cap_counts[group] + 1) - } - # Replace zeros with nan for groups with less caps than the group with the max caps - if max(cap_numbers) > group_cap_counts[group]: - for i in range(group_cap_counts[group] + 1, max(cap_numbers) + 1): - frequency_dict.update({i: float("nan")}) - ``` - - **"temporal_fraction"** - - Previous Code: - ```python - proportion_dict = { - key: item / (len(predicted_subject_timeseries[subj_id][curr_run])) - for key, item in sorted_frequency_dict.items() - } - ``` - - "Refactored Code": Nothing other than some parameter names have changed. - ```python - proportion_dict = { - key: value / (len(predicted_subject_timeseries[subj_id][curr_run])) - for key, value in frequency_dict.items() - } - ``` - - **"persistence"** - - Previous Code: - ```python - # Initialize variable - persistence_dict = {} - uninterrupted_volumes = [] - count = 0 - # Iterate through caps - for target in cap_numbers: - # Iterate through each element and count uninterrupted volumes that equal target - for index in range(0, len(predicted_subject_timeseries[subj_id][curr_run])): - if predicted_subject_timeseries[subj_id][curr_run][index] == target: - count += 1 - # Store count in list if interrupted and not zero - else: - if count != 0: - uninterrupted_volumes.append(count) - # Reset counter - count = 0 - # In the event, a participant only occupies one CAP and to ensure final counts are added - if count > 0: - uninterrupted_volumes.append(count) - # If uninterrupted_volumes not zero, multiply elements in the list by repetition time, sum and divide - if len(uninterrupted_volumes) > 0: - persistence_value = np.array(uninterrupted_volumes).sum() / len( - uninterrupted_volumes - ) - if tr: - persistence_dict.update({target: persistence_value * tr}) - else: - persistence_dict.update({target: persistence_value}) - else: - # Zero indicates that a participant has zero instances of the CAP - persistence_dict.update({target: 0}) - # Reset variables - count = 0 - uninterrupted_volumes = [] - - # Replace zeros with nan for groups with less caps than the group with the max caps - if len(cap_numbers) > group_cap_counts[group]: - persistence_dict = { - cap_number: ( - persistence_dict[cap_number] - if cap_number <= group_cap_counts[group] - else float("nan") - ) - for cap_number in cap_numbers - } - ``` - - Refactored Code: - ```python - # Initialize variable - persistence_dict = {} - # Iterate through caps - for target in cap_numbers: - # Binary representation of array - if [1,2,1,1,1,3] and target is 1, then it is [1,0,1,1,1,0] - binary_arr = np.where(predicted_subject_timeseries[subj_id][curr_run] == target, 1, 0) - # Get indices of values that equal 1; [0,2,3,4] - target_indices = np.where(binary_arr == 1)[0] - # Count the transitions, indices where diff > 1 is a transition; diff of indices = [2,1,1]; - # binary for diff > 1 = [1,0,0]; thus, segments = transitions + first_sequence(1) = 2 - segments = np.where(np.diff(target_indices, n=1) > 1, 1, 0).sum() + 1 - # Sum of ones in the binary array divided by segments, then multiplied by 1 or the tr; segment is - # always 1 at minimum due to + 1; np.where(np.diff(target_indices, n=1) > 1, 1,0).sum() is 0 when empty or the condition isn't met - persistence_dict.update({target: (binary_arr.sum() / segments) * (tr if tr else 1)}) - - # Replace zeros with nan for groups with less caps than the group with the max caps - if max(cap_numbers) > group_cap_counts[group]: - for i in range(group_cap_counts[group] + 1, max(cap_numbers) + 1): - persistence_dict.update({i: float("nan")}) - ``` - - **"transition_frequency"** - - Previous Code: - ```python - count = 0 - # Iterate through predicted values - for index in range(0, len(predicted_subject_timeseries[subj_id][curr_run])): - if index != 0: - # If the subsequent element does not equal the previous element, this is considered a transition - if ( - predicted_subject_timeseries[subj_id][curr_run][index - 1] - != predicted_subject_timeseries[subj_id][curr_run][index] - ): - count += 1 - # Populate DataFrame - new_row = [subj_id, group_name, curr_run, count] - df_dict["transition_frequency"].loc[len(df_dict["transition_frequency"])] = new_row - ``` - - - Refactored Code: - ```python - # Sum the differences that are not zero - [1,2,1,1,1,3] becomes [1,-1,0,0,2], binary representation - # for values not zero is [1,1,0,0,1] = 3 transitions - transition_frequency = np.where( - np.diff(predicted_subject_timeseries[subj_id][curr_run]) != 0, 1, 0 - ).sum() - ``` - *Note, the `n` parameter in `np.diff` defaults to 1, and differences are calculated as `out[i] = a[i+1] - a[i]`* -### 🐛 Fixes -- When a pickle file was used as input in `standardize` or `change_dtype` an error was produced, this has been fixed -and these functions accept a list of dictionaries or a list of pickle files now. - -### 💻 Metadata -- In the documentation for `CAP.caps2corr` it is now explicitly stated that the type of correlation being used is -Pearson correlation. - -## [0.14.7] - 2024-07-17 -### ♻ Changed -- Improved Warning Messages and Print Statements: - - In TimeseriesExtractor.get_bold, the subject-specific information output has been reformatted for better readability: - - - Previous Format: - ``` - Subject: 1; run:1 - Message - ``` - - - New Format: - ``` - [SUBJECT: 1 | SESSION: 1 | TASK: rest | RUN: 1] - ----------------------------------------------- - Message - ``` - - - In `CAP` class numerous warnings and statements have been changed to improve clarity: - - - Previous Format: - ``` - Optimal cluster size using silhouette method for A is 2. - ``` - - - New Format: - ``` - [GROUP: A | METHOD: silhouette] - Optimal cluster size is 2. - ``` - - - These changes should improve clarity when viewing in a terminal or when redirected to an output file by SLURM. - - Language in many statements and warnings have also been improved. - -## [0.14.6] - 2024-07-16 -### 🐛 Fixes -- For `CAP.get_caps`, when `cluster_selection_method` was used to find the optimal cluster size, the model would be -re-estimated and stored in the `self.kmeans` property for later use. Previously, the internal function that generated the -model using scikit's `KMeans` only returned the performance metrics. These metrics for each cluster size were assessed, -and the best cluster size was used to generate the optimal KMeans model with the same parameters. This is fine when -setting `random_state` with the same k since the model would produce the same initial cluster centroids and produces similar -clustering solution regardless of the number of times the model is re-generated. However, if a random state was not used, -the newly re-generated optimal model would technically differ despite having the same k, due to the random nature of KMeans -when initializing the cluster centroids. Now, the internal function returns both the performance metrics and the models, -ensuring the exact same model that was assessed is stored in the `self.kmeans`. Shouldn't be an incredibly major issue -if your models are generally stable and produce similar cluster solutions. Though when not using a random state, even -minor differences in the kmeans model even when using the same k can produce some statistical differences. Ultimately, -it is always best to ensure that the same model that the same model used for assessment and for later analyses are the -same to ensure robust results. - -## [0.14.5] - 2024-07-16 -### ♻ Changed -- In `TimeseriesExtractor`, `dummy_scans` can now be a dictionary that uses the "auto" sub-key if "auto" is set to -True, the number of dummy scans removed depend on the number of "non_steady_state_outlier_XX" columns in the -participants fMRIPrep confounds tsv file. For instance, if there are two "non_steady_state_outlier_XX" columns -detected, then `dummy_scans` is set to two since there is one "non_steady_state_outlier_XX" per outlier volume for -fMRIPrep. This is assessed for each run of all participants so ``dummy_scans`` depends on the number number of -"non_steady_state_outlier_XX" in the confound file associated with the specific participant, task, and run number. -### 🐛 Fixes -- For defensive programming purposes, instead of assuming the timing information in the event file perfectly -coincides with the timeseries. When a condition is specified and onset and duration must be used to extract the -indices corresponding to the condition of interest, the max scan index is checked to see if it exceeds the length of -the timeseries. If this condition is met, a warning is issued in the event of timing misalignment (i.e errors in event -file, incorrect repetition time, etc) and invalid indices are ignored to only extract the valid indices from the timeseries. -This is done in the event this was that are greater than the timeseries shape are ignored. - -## [0.14.4] - 2024-07-15 -### ♻ Changed -- Minor update that prints the optimal cluster size for each group when using `cluster_selection_method` in -`CAP.get_caps`. Just for information purposes. -- When error raised due to kneed not being able to detect the elbow, the group it failed for is now stated. -- Previously version 0.14.3.post1 - -## [0.14.3.post1] - YANKED -### ♻ Changed -- Minor update that prints the optimal cluster size for each group when using `cluster_selection_method` in -`CAP.get_caps`. Just for information purposes. -- When error raised due to kneed not being able to detect the elbow, the group it failed for is now stated. -- Yanked due to not being a metadata update, this should be a patch update to denote a behavioral change, -this is now version 0.14.4 to adhere a bit better to versioning practices. - -## [0.14.3] - 2024-07-14 -- Thought of some minor changes. - -### ♻ Changed -- Added new warning if `fd_threshold` is specified but `use_confounds` is False since `fd_threshold` needs the confound -file from fMRIPrep. In previous version, censoring just didn't occur and never issued a warning. -- Changed the error exception types for cosine similarity in `CAP.caps2radar` from ValueError to ZeroDivisionError -- Added ValueError in `TimeseriesExtractor.visualize_bold` if both `region` and `roi_indx` is None. -- In `TimeseriesExtractor.visualize_bold` if `roi_indx` is a string, int, or list with a single element, a title is -added to the plot. - -## [0.14.2.post2] - 2024-07-14 -### 💻 Metadata -- Simply wanted the latest metadata update to be on Zenodo and to have the same DOI as I forgot to upload -version 0.14.2.post1 there. - -## [0.14.2.post1] - 2024-07-14 -### 💻 Metadata -- Updated a warning during timeseries extraction that only included a partial reason for why the indices for condition -have been filtered out. Added information about `fd_threshold` being the reason why. - -## [0.14.2] - 2024-07-14 -### ♻ Changed -- Implemented a minor code refactoring that allows runs flagged due to "outlier_percentage", runs were all volumes will -be scrubbed due to all volumes exceeding the threshold for framewise displacement, and runs were the specified condition -returns zero indices will not undergo timeseries extraction. -- Also clarified the language in a warning that occurs when all NifTI files have been excluded or missing for a subject. -### 🐛 Fixes -- If a condition does not exist in the event file, a warning will be issued if this occurs. This should prevent empty -timeseries or errors. In the warning the condition will be named in the event of a spelling error. -- Added specific error type to except blocks for the cosine similarities that cause a division by zero error. - -## [0.14.1.post1] - 2024-07-12 -### 💻 Metadata -- Updates typehint `fd_threshold` since it was only updated in the doc string. - -## [0.14.1] - 2024-07-12 -### ♻ Changed -- In `TimeseriesExtractor`, `fd_threshold` can now be a dictionary, which includes a sub-key called "outlier_percentage", -a float value between 0 and 1 representing a percentage. Runs where the proportion of volumes exceeding the "threshold" -is higher than this percentage are removed. If `condition` is specified in `self.get_bold`, only the runs where the -proportion of volumes exceeds this value for the specific condition of interest are removed. A warning is issued -whenever a run is flagged. -- As of now, flagging and removal of runs, due to "outlier_percentage", is conducted after timeseries extraction. -This was done to minimize disrupting the original code and for easier testing for feature reliability as significant -code refactoring could cause unintended behaviors and requires longer testing for reliability. In a future patch, runs -will be assessed to see if they meet the exclusion criteria due to "outlier_percentage" prior to extraction and will be -skipped if flagged. -### 💻 Metadata -- Warning issue if cosine similarity is 0. -- Minor improvements to warning clarity. -- Changelog versioning updated for transparency since patches may include changes to parameters to improve behavior or -added paramaters to fix behavior. But these changes will be backwards compatible. - -## [0.14.0] - 2024-07-07 -### 🚀 New/Added -- More flexibility when calculating cosine similarity in the `CAP.caps2radar` function. Now a `method` and `alpha` parameter -is added to choose between calculating "traditional" cosine similarity, a more "selective" cosine similarity, or -a "combined" approach where `alpha` is used to determine the relative contributions of the `traditional` and `selective` -approach. -### 🐛 Fixes -- Added try except blocks in `CAP.caps2radar`, to handle division by zero cases. -- In `CAP.caps2surf`, `as_outline` kwarg is now its own separate layer, which should allow the outline to be build -on top of the stat map when requested. - -## [0.13.5] - 2024-07-06 -### 🐛 Fixes -- For `knn_dict`, replaces method for majority vote to another method that is more appropriate for floats -when k is greater than 1. Current method is more appropriate for atlases, which have integer values. - -## [0.13.4.post1] - 2024-07-05 -### 💻 Metadata -- Spelling fix in error message to refer to the correct variable name. - -## [0.13.4] - 2024-07-05 -### 🐛 Fixes -- For `CAP.caps2surf` and `CAP.caps2niftis`, fwhm comes after the knn method, if requested. - -## [0.13.3] - 2024-07-05 -### 🐛 Fixes -- Adds a "remove_subcortical" key to `knn_dict`. -- Uses "nearest" interpolation for Schaefer resampling so the labels are retained. -- Fixes "resolution_mm" default in `knn_dict`, which was set to "1mm" instead of 1 if not specified. - -## [0.13.2] - 2024-07-05 -### 🐛 Fixes -- Certain custom atlases may not project well from volume to surface space. A new parameter, `knn_dict` has been added to -`CAP.caps2surf()` and `CAP.caps2niftis` to apply k-nearest neighbors (knn) interpolation while leveraging the -Schaefer atlas, which projects well from volumetric to surface space. -- No longer need to add `parcel_approach` when using `CAP.caps2surf` with `fslr_giftis_dict`. - -## [0.13.1] - 2024-06-30 -### ♻ Changed -- For `CAP.caps2radar`, the `scattersize` kwarg can be used to control the size of the scatter/markers regardless -if `use_scatterpolar` is used. - -## [0.13.0.post1] - 2024-06-28 -### 💻 Metadata -- Clarifies that the p-values obtained in `CAP.caps2corr` are uncorrected. - -## [0.13.0] - 2024-06-28 -### 🚀 New/Added -- Minor update that adds some features to `CAP.caps2corr()`, specifically adds three parameters - `return_df`, `save_df`, -and `save_plots`. Now, in addition to visualizing a correlation matrix, this function can also return a pandas dataframe -containing a correlation matrix, where each element in the correlation matrix is accompanied by its p-value in -parenthesis, which is followed by an asterisk (single asterisk for < 0.05, double asterisk for 0.01, and triple asterisk -for < 0.001). These dataframes can also be saves as csv files. -- All plotting functions that use matplotlib includes `bbox_inches` as a kwarg and defaults to "tight". -- Added `annot_kws` kwargs to `CAPs.caps2plot` and `CAP.caps2corr`. - -## [0.12.2] - 2024-06-28 -### ♻ Changed -- When specified, allows `runs` parameter to be string, int, list of strings, or list of integers instead of just lists. -Always ensures it is converted to list if integer or string. -- Clarifies warning if tr not specified in `TimeseriesExtractor` by stating the `tr` is set to `None` and that extraction -will continue. -- For `CAP.get_caps`, if runs is `None`, the `self.runs` property is just None instead of being set to "all". Only affects what -is returned by `self.runs` when nothing is specified. - -## [0.12.1.post2] - 2024-06-27 -### 💻 Metadata -- Includes the updated type hints in 0.12.1.post1 and removes the unsupported operand for compatibility with -Python 3.9. - -## [0.12.1.post1] - 2024-06-27 [YANKED] -### 💻 Metadata -- Additional type hint updates. -- **Reason for Yanking**: Yanked due to potentially unsupported operand for type hinting (the vertical bar `|`) -in earlier Python versions (3.9). - -## [0.12.1] - 2024-06-27 - -### ♻ Changed -- For `merge_dicts` sorts the run keys lexicographically so that subjects that don't have the earliest run-id in the -first dictionary due to not having that run or the run being excluded still have ordered run keys in the merged -dictionary. - -### 💻 Metadata -- Updates `runs` parameters type hints so that it is known that strings can be used to0. - -## [0.12.0] - 2024-06-26 -- Entails some code cleaning and verification to ensure that the code cleaned for clarity purposes produces the same -results. - -### 🚀 New/Added -- Davies Bouldin and Variance Ratio (Calinski Harabasz) added - -### ♻ Changed -- For `CAPs.calculate_metrics()` if performing an analysis on groups where each group has a different number of CAPs, then for "temporal_fraction", -"persistence", and "counts", "nan" values will be seen for CAP numbers that exceed the group's number of CAPs. - - For instance, if group "A" has 2 CAPs but group "B" has 4 CAPs, the DataFrame will contain columns for CAP-1, - CAP-2, CAP-3, and CAP-4. However, for all members in group "A", CAP-3 and CAP-4 will contain "nan" values to - indicate that these CAPs are not applicable to the group. This differentiation helps distinguish between CAPs - that are not applicable to the group and CAPs that are applicable but had zero instances for a specific member. - -### 🐛 Fixes -- Adds error earlier when tr is not specified or able to be retrieved form the bold metadata when the condition is specified -instead of allowing the pipeline to produce this error later. -- Fixed issue with `show_figs` in `CAP.caps2surf()` showing figure when set to False. - -## [0.11.3] - 2024-06-24 -### ♻ Changed -- With parallel processing, joblib outputs are now returned as a generator as opposed to the default, which is a list, -to reduce memory usage. - -## [0.11.2] - 2024-06-23 -### ♻ Changed -- Changed how ids are organized in respective group when initializing the `CAP` class. In version 0.11.1, the ids are -sorted lexicographically: -```python3 -self._groups[group] = sorted(list(set(self._groups[group]))) -``` -This doesn't affect functionality but it may be better to respect the original user ordering.This is no longer the case. - -## [0.11.1] - 2024-06-23 -### 🐛 Fixes -- Fix for python 3.12 when using `CAP.caps2surf`. - - Changes in pathlib.py in Python 3.12 results in an error message format change. The error message now includes - quotes (e.g., "not 'Nifti1Image'") instead of the previous format without quotes ("not Nifti1Image"). This issue - arises when using ``neuromaps.transforms.mni_to_fslr`` within CAP.caps2surf() as neuromaps captures the error as a - string and checks if "not Nifti1Image" is in the string to determine if the input is a NifTI image. As a patch, - if the error occurs, a temporary .nii.gz file is created, the statistical image is saved to this file, and it is - used as input for neuromaps.transforms.mni_to_fslr. The temporary file is deleted after use. Below is the code - implementing this fix. - -```python3 -# Fix for python 3.12, saving stat map so that it is path instead of a NifTi -try: - gii_lh, gii_rh = mni152_to_fslr(stat_map, method=method, fslr_density=fslr_density) -except TypeError: - # Create temp - temp_nifti = tempfile.NamedTemporaryFile(delete=False, suffix=".nii.gz") - warnings.warn( - textwrap.dedent( - f""" - Potential error due to changes in pathlib.py in Python 3.12 causing the error - message to output as "not 'Nifti1Image'" instead of "not Nifti1Image", which - neuromaps uses to determine if the input is a Nifti1Image object. - Converting stat_map into a temporary nii.gz file (which will be automatically - deleted afterwards) at {temp_nifti.name} - """ - ) - ) - # Ensure file is closed - temp_nifti.close() - # Save temporary nifti to temp file - nib.save(stat_map, temp_nifti.name) - gii_lh, gii_rh = mni152_to_fslr( - temp_nifti.name, method=method, fslr_density=fslr_density - ) - # Delete - os.unlink(temp_nifti.name) -``` -- Final patch is for strings in triple quotes. The standard textwrap module is used to remove the indentations at each -new line. - -## [0.11.0.post2] - 2024-06-22 -### 💻 Metadata -- Very minor explanation added to `CAP.calculate_metrics` regarding using individual dictionaries from merged -dictionaries as inputs. - -## [0.11.0.post1] - 2024-06-22 -### 💻 Metadata -- Two docstring changes for `merge_dicts`, which includes nesting the return type hint and capitalizing all letters of -the docstring header for aesthetics. - -## [0.11.0] - 2024-06-22 -### 🚀 New/Added -- Added new function `change_dtype` to make it easier to change the dtypes of each subject's numpy array to assist with -memory usage, especially if doing the CAPs analysis on a local machine. -- Added new parameters - `output_dir`, `file_name`, and `return_dict` ` to `standardize` to save dictionary, the -`return_dict` defaults to True. -- Adds a new version attribute so you can check the current version using `neurocaps.__version__` - -### ♻ Changed -- Adds back python 3.12 classifier. The `CAP.caps2surf` function may still not work well but if its detected that -neurocaps is being installed using python 3.12, setuptools is installed to prevent the pkgresources error. - -### 🐛 Fixes -- Minor fix for `file_name` parameter in `merge_dicts`. If user does not supply a `file_name` when saving the dictionary, -it will provide a default file_name now instead of producing a Nonetype error. - -### 💻 Metadata -- Minor docstrings revisions, mostly to the typehint for ``subject_timeseries``. - -## [0.10.0.post2] - 2024-06-20 -### 💻 Metadata -- Minor metadata update to docstrings to remove curly braces from inside the list object of certain parameters to -not make it seem as if it is supposed to be a strings inside a dictionary which is inside a list as opposed to strings -in a list. - -## [0.10.0.post1] - 2024-06-19 -### 💻 Metadata -- Minor metadata update to denote that `run` and `runs` parameter can be a string too. - -## [0.10.0] - 2024-06-17 -### 🚀 New/Added -- `CAP` class as a `cosine_similarity` property and in `CAP.caps2radar`, there is now a `as_html` parameter to save -plotly's radar plots as an html file instead of a static png. The html files can be opened in a browser and saved as a -png from the browser. Most importantly, they are interactive. - **new to [0.10.0]** -- Made another internal attribute in CAP `CAP.subject_table` a property and setter. This property acts as a lookup -table. As a setter, it can be used to modify the table to use another subject dictionary with different subjects -not used to generate the k-means model. -- Can now plot silhouette score and have some control over the `x-axis` of elbow and silhouette plot with the "step" `**kwarg`. - -### ♻ Changed -- Default for `CAP.caps2plots` from "outer product" to "outer_product". -- Default for `CAP.calculate_metrics` from "temporal fraction" to "temporal_fraction" and "transition frequency" -to "transition_frequency". -- `n_clusters` and `cluster_selection_method` parameters moved to `CAP.get_caps` instead of being parameters in -`CAP`. - -### 🐛 Fixes -- Restriction that numpy must be less than version 2 since this breaks brainspace vtk, which is needed for plotting to -surface space. - **new to [0.10.0]** -- Adds nbformat as dependency for plotly. - **new to [0.10.0]** -- In `TimeseriesExtractor.get_bold`, several checks are done to ensure that subjects have the necessary files for -extraction. Subjects that have zero nifti, confound files (if confounds requested), event files (if requested), etc -are automatically eliminated from being added to the list for timeseries extraction. A final check assesses, the run -ID of the files to see if the subject has at least one run with all necessary files to avoid having subjects with all -the necessary files needed but all are from different runs. This is most likely a rare occurrence but it is better to be -safer to ensure that even a rare occurrence doesn't result in a crash. The continue statement that skips the subject -only worked if no condition was specified. -- Removes in-place operations when standardizing to avoid numpy casting issues due to incompatible dtypes. -- Additional deep copy such as deep copying any setter properties to ensure external changes does not result internal -changes. -- Some important fixes were left out of the original version. - - These fixes includes: - - Removal of the `epsilon` parameter in `self.get_caps` and replacement with `std[std < np.finfo(np.float64).eps] = 1.0` - to prevent divide by 0 issues and numerical instability issues. - - Deep copy `subject_timeseries` in `standardize` and `parcel_approach`. In their functions, in-place operations - are performed which could unintentionally change the external versions of these parameters -- Added try-except block in `TimeseriesExtractor.get_bold` when attempting to obtain the `tr`, to issue a warning -when `tr` isn't specified and can't be extracted from BOLD metadata. Extraction will be continued. -- Fixed error when using `silhouette` method without multiprocessing where the function called the elbow method instead -of the silhouette method. This error affects versions 0.9.6 to 0.9.9. -- Fix some file names of output by adding underscores for spaces in group names. - -### 💻 Metadata -- Drops the python 3.12 classifier. All functions except for `CAP.caps2surf` works on python 3.12. Additionally, for -python 3.12, you may need to use `pip install setuptools` if you receive an error stating that -"ModuleNotFoundError: No module named 'pkg_resources'". - new to [0.10.0] -- Ensure user knows that all image files are outputted as pngs. -- Clarifications of some doc strings, stating that Bessel's correction is used for standardizing and that for -`CAP.calculate_metrics` can accept subject timeseries not used for generating the k-means model. -- Corrects docstring for `standardize` from parameter being `subject_timeseries_list` to `subject_timeseries`. - -## [0.9.9.post3] - 2024-06-13 -### 🐛 Fixes -- Noted an issue with file naming in `CAP.calculate_metrics` that causes the suffix of the file name to append -to subsequent file names when requesting multiple metrics. While it doesn't effect the content inside the file it is an -irritating issue. For instance "-temporal_fraction.csv" became "-counts-temporal_fraction.csv" if user requested "counts" -before "temporal fraction". - -### 💻 Metadata -- But Zenodo on PyPi. - -## [0.9.9.post2] - 2024-06-13 -### 💻 Metadata -- All docstrings now at a satisfactory point of being well formatted and explanatory. -- Fixes issues with docstring not being formatted correctly when reading in an IDE like Jupyter notebook. - -## [0.9.9.post1] - 2024-06-12 -### 🐛 Fixes -- Reference before assignment issue when `use_confounds` is False do to `censor` only being when `use_confounds` -is True. - -## [0.9.9] - 2024-06-12 - -**Pylint used to check for potential errors and also used to clean code** - -### ♻ Changed -- `parcel_approach` no longer required when initializing `CAP`. It is still required for some plotting methods and the -user will be warned if it is None. This allows the use of certain methods without having to keep adding this parameter. -- For `CAP.calculate_metrics`, `file_name` parameter changed to `prefix_file_name` to better reflect that it will be -added as a prefix to the csv files. - -### 🐛 Fixes -- Fixed issue with no context manager or closing json file in `TimeseriesExtractor` where if `tr` is not specified, -the bold metadata is used to extract the tr. However, this was done without a context manager to ensure the file closes -properly afterwards. -- All imports, except for `pybids` are no longer imported in each function and are now at top level. - -## [0.9.8.post3] - 2024-06-10 -### 🐛 Fixes -- Adds a "mode" kwargs to `CAP.caps2radar` to override default plotly drawing behaviors and sets `use_scatterpolar` -argument to False. - -## [0.9.8.post2] - 2024-06-09 -### 💻 Metadata -- Significant improvements to docstrings and added homepage. - -## [0.9.8.post1] - 2024-06-08 -### 🐛 Fixes -- Uses plotly.offline to open plots generated by `CAP.caps2radar` in default browser when Python is non-interactive -to prevent hanging issue. - -## [0.9.8] - 2024-06-07 -### ♻ Changed -- Changed `vmax` and `vmin` kwargs in `CAP.caps2surf` to `color_range` -- In `CAP.caps2surf` the function no longer rounds max and min values and restricts range to -1 and 1 if the rounded -value is 0. -It just uses the max and min values from the data. - -## [0.9.8.rc1] - 2024-06-07 -🚀 New/Added -- New method in `CAP` class to plot radar plot of cosine similarity (`CAP.caps2radar`). -- New method in `CAP` class to save CAPs as niftis without plotting (`CAP.caps2niftis`). -- Added new parameter to `CAP.caps2surf`, `fslr_giftis_dict`, to allow CAPs statistical maps that were -converted to giftis externally, using tools such as Connectome Workbench, to be plotted. This parameter only requires -the `CAP` class to be initialized. - -## [0.9.7.post2] - 2024-06-03 -### ♻ Changed -- Minor change in merge_dicts() to make it explicitly clear that the dictionaries are returned in the order they are -provided in the list. Originally, the dictionaries were returned as a nested dictionary with sub-keys starting at -"dict_1" to represent the first dictionary given in the list. They now start at "dict_0" to represent the first -dictionary in the list. This doesn't affect the underlying functionality of the code; the sub-keys are simply numbered -to represent their original index in the provided list. - -## [0.9.7.post1] - 2024-06-03 -### 🐛 Fixes -- Allows user to change the maximum and minimum value displayed for `CAP.caps2plot` and `CAP.caps2surf` - -## [0.9.7] - 2024-06-02 -🚀 New/Added -- More plotting kwargs and ability to just show the left and right hemisphere when plotting nodes with `CAP.caps2plot` -for "Schaefer" and "Custom" parcellations. -- Added `suffix_title` parameter to `CAP.caps2corr` and `CAP.caps2surf`. - -### ♻ Changed -- Changed `task_title` parameter in `CAP.caps2plot` to `suffix_title`. - -## [0.9.6] - 2024-05-31 -Recommend this version if intending to use parallel processing since it uses `joblib`, which seems to be more memory -efficient than multiprocessing. - -🚀 New/Added -- Added `n_cores` parameter to `CAP.get_caps` for multiprocessing when using the silhouette or elbow method. -- More restrictions to the minimum versions allowed for dependencies. - -### ♻ Changed -- Use joblib for pickling (replaces pickle) and multiprocessing (replaces multiprocessing). - -## [0.9.5.post1] - 2024-05-30 -🚀 New/Added -- Added the `linecolor` **kwargs for `CAP.caps2corr` and `CAP.caps2plot` that should have been deployed in 0.9.5. - -## [0.9.5] - 2024-05-30 - -### 🚀 New/Added -- Added ability to create custom colormaps with `CAP.caps2surf` by simply using the cmap parameter with matplotlibs -`LinearSegmentedColormap` with the `cmap` kwarg. An example of its use can be seen in demo.ipynb and the in the README. -- Added `surface` **kwargs to `CAP.caps2surf` to use "inflated" or "veryinflated" for the surface plots. - -## [0.9.4.post1] - 2024-05-28 - -### 💻 Metadata -- Update some metadata on PyPi - -## [0.9.4] - 2024-05-27 - -### ♻ Changed - -- Improvements to docstrings in all methods in neurocaps. -- Restricts scikit-learn to version 1.4.0 and above. -- Reduced the number of default `confound_names` in the `TimeseriesExtractor` class that will be used if `use_confounds` -is True but no `confound_names` are specified. The new defaults are listed below. The previous default included -nonlinear motion parameters. -- Use default of "run-0" instead of "run-1" for the subkey in the `TimeseriesExtractor.subject_timeseries` for files -processed with `TimeseriesExtractor.get_bold` that do not have a run ID due to only being a single run in the dataset. - -```python -if high_pass: - confound_names = [ - "trans_x", - "trans_x_derivative1", - "trans_y", - "trans_y_derivative1", - "trans_z", - "trans_z_derivative1", - "rot_x", - "rot_x_derivative1", - "rot_y", - "rot_y_derivative1", - "rot_z", - "rot_z_derivative1", - ] -else: - confound_names = [ - "cosine*", - "trans_x", - "trans_x_derivative1", - "trans_y", - "trans_y_derivative1", - "trans_z", - "trans_z_derivative1", - "rot_x", - "rot_x_derivative1", - "rot_y", - "rot_y_derivative1", - "rot_z", - "rot_z_derivative1", - "a_comp_cor_00", - "a_comp_cor_01", - "a_comp_cor_02", - "a_comp_cor_03", - "a_comp_cor_04", - "a_comp_cor_05", - ] -``` - -## [0.9.3] - 2024-05-26 - -### 🚀 New/Added -- Supports nilearns versions 0.10.1, 0.10.2, 0.10.4, and above (does not include 0.10.3). - -### ♻ Changed -- Renamed `CAP.visualize_caps` to `CAP.caps2plot` for naming consistency with other methods for visualization in -the `CAP` class. - -## [0.9.2] - 2024-05-24 - -### 🚀 New/Added -- Added ability to create correlation matrices of CAPs with `CAP.caps2corr`. -- Added more **kwargs to `CAP.caps2surf`. Refer to the docstring to see optional **kwargs. - -### 🐛 Fixes -- Use the `KMeans.labels_` attribute for scikit's KMeans instead of using the `KMeans.predict` on the same dataframe -used to generate the model. It is unecessary since `KMeans.predict` will produce the same labels already stored in -`KMeans.labels_`. These labels are used for silhouette method. - -### ♻ Changed -- Minor aesthetic changes to some plots in the `CAP` class such as changing "CAPS" in the title of `CAP.caps2corr` -to "CAPs". - -## [0.9.1] - 2024-05-22 - -### 🚀 New/Added -- Ability to specify resolution for Schaefer parcellation. -- Ability to use spatial smoothing during timeseries extraction. -- Ability to save elbow plots. -- Add additional parameters - `fslr_density` and `method` to the `CAP.caps2surf` method to modify interpolation -methods from MNI152 to surface space. -- Increased number of parameters to use with scikit's `KMeans`, which is used in `CAP.get_caps`. - -### ♻ Changed -- In, `CAP.calculate_metrics` nans where used to signify the abscense of a CAP, this has been replaced with 0. Now -for persistence, counts, and temporal fraction, 0 signifies the absence of a CAP. For transition frequency, 0 means no -transition between CAPs. - -### 🐛 Fixes -- Fix for AAL surface plotting for `CAP.caps2surf`. Changed how CAPs are projected onto surface plotting by -extracting the actual sorted labels from the atlas instead of assuming the parcellation labels goes from 1 to n. -The function still assumes that 0 is the background label; however, this fixes the issue for parcellations that don't -go from 0 to 1 and go from 0 with the first parcellation label after zero starting at 2000 for instance. - -## [0.9.0] - 2024-05-13 - -### 🚀 New/Added -- Ability to project CAPs onto surface plots. - -## [0.8.9] - 2024-05-09 - -### 🚀 New/Added -- Added "Custom" as a valid keyword for `parcel_approach` in the `TimeseriesExtractor` and `CAP` classes to support -custom parcellation with bilateral nodes (nodes that have a left and right hemisphere version). Timeseries extraction, -CAPs extraction, and all visualization methods are available for custom parcellations. -- Added `exclude_niftis` parameter to `TimeseriesExtractor.get_bold` to skip over specific files during timeseries -extraction. -- Added `fd_threshold` parameter to `TimeseriesExtractor` to scrub frames that exceed a specific threshold after -nuisance regression is done. -- Added options to flush print statements during timeseries extraction. -- Added additional **kwargs for `CAP.visualize_caps`. - -### ♻ Changed -- Changed `network` parameter in `TimeseriesExtractor.visualize_bold` to `region`. -- Changed "networks" option in `visual_scope` parameter in `CAP.visualize_caps` to "regions". - -### 🐛 Fixes -- Fixed reference before assignment when specifying the repetition time (TR) when using the `tr` parameter in -`TimeseriesExtractor.get_bold`. Prior only extracting the TR from the metadata files, which is done if the `tr` -parameter was not specified worked. -- Allow bids datasets that do not specify run id or session id in their file names to be ran instead of producing an -error. Prior, only bids datasets that included "ses-#" and "run-#" in the file names worked. Files that do not have -"run-#" in it's name will include a default run-id in their sub-key to maintain the structure of the -`TimeseriesExtractor.subject_timeseries` dictionary". This default id is "run-1". -- Fixed error in `CAP.visualize_caps` when plotting "outer products" plot without subplots. - -## [0.8.8] - 2024-03-23 - -### 🚀 New/Added -- Support Windows by only allowing install of pybids if system is not Windows. On Windows machines -`TimeseriesExtractor` cannot be used but `CAP` and all other functions can be used. - -## [0.8.7] - 2024-03-15 - -### 🚀 New/Added -- Added `merge_dicts` to be able to combine different subject_timeseries and only return shared subjects. -- Print names of confounds used for each subject and run when extracting timeseries for transparency. -- Ability to extract timeseries using the AAL or Schaefer parcellation. -- Ability to use multiprocessing to speed up timeseries extraction. -- Can be used to extract task (entire task timeseries or a single specific condition) or resting-state data. -- Ability to denoise data during extraction using band pass filtering, confounds, detrending, and removing dummy scans. -- Can visualize the extracted timeseries at the node or network level. -- Ability to perform Co-activation Patterns (CAPs) analysis on separate groups or all subjects. -- Can use silhouette method or elbow method to determine optimal cluster size and the optimal kmeans model will be -saved. -- Can visualize kneed plots for elbow method. -- Can visualize CAPs using heatmaps or outer product plots at the network or node level of the Schaefer or AAL atlas. -- Can calculate temporal frequency, persistence, counts, and transition frequency. As well as save each as a csv file. +# Changelog + +All notable future changes to neurocaps will be documented in this file. + +*Note*: All versions in this file are deployed on PyPi. + +## [Versioning] + +**Beyond version 0.10.0, versioning for the 0.x.x series for this package will work as:** + +`0.minor.patch.postN` + +- *.minor* : Introduces new features and may include potential breaking changes. Any breaking changes will be explicitly +noted in the changelog (i.e new functions or parameters, changes in parameter defaults or function names, etc). +- *.patch* : Will contain fixes for any identified bugs, may include modifications or an added parameter for +improvements/enhancements. Fixes and modifications will be backwards compatible. +- *.postN* : Consists of only metadata-related changes, such as updates to type hints or doc strings/documentation. + +## [0.18.11] - 2024-11-27 +### 🐛 Fixes +- An error in a setter method that did not use `raise`. + +## [0.18.10] - 2024-11-27 +### 🚀 New/Added +- Added deleter method for `subject_timeseries` and `concatenated_timeseries` properties + +## [0.18.9] - 2024-11-25 +### 🚀 New/Added +- Custom error to warn about querying issues +- Add optional dependency for demo +### 🐛 Fixes +- Documentation rendering issues +- Restrict from downloading the latest vtk 9.4.0 for `caps2surf` to work. + +## [0.18.8.post0] - 2024-11-23 +### 📖 Documentation +- Update to documentation to show example directory structure. + +## [0.18.8] - 2024-11-22 +### 🚀 New/Added +- Added "use_sample_mask" key for `fd_threshold` parameter, which if set to True, generates a sample mask to pass +to nilearn's `NiftiLabelsMasker` for censoring prior to nuisance regression. +Internally, the ``clean__extrapolate`` parameter is used to set extrapolate to False. If condition filtering is +requested, when "use_sample_mask" key for `fd_threshold` parameter is True, then the truncated timeseries is +temporarily padded to ensure the correct indices corresponding to the condition are obtained. +- Added new property. + +## [0.18.7] - 2024-11-18 +### 🐛 Fixes +- Fixes projection of CAPs onto NiFTI atlas by preventing in-place modification. Previously, if a new CAP value matched +a subsequent atlas label ID, it could cause incorrect coordinate assignments. + +## [0.18.6] - 2024-11-18 +- Minor code cleaning +### 📖 Documentation +- Readme example fix + +## [0.18.5] - 2024-11-16 +### ♻ Changed +- Updated Dependencies: + - NumPy: version 2.0 and above can be installed. + - BrainSpace: requires version 0.1.16 and above. + +## [0.18.4] - 2024-11-15 +### 🐛 Fixes +- Corrected region names for version "3v2" of the AAL atlas. +### ♻ Changed +- Added a specific logged warning when no confound names are found. If some confound names are missing, +they will still be listed accordingly. +- Added a specific logged warnings for methods in `CAP` that use the `runs` parameter. Warnings are issued if a subject +is missing any requested run, with an additional warning if all runs are missing. + +## [0.18.3.post0] - 2024-11-14 +### 📖 Documentation +- Added reference for `merge_dicts`. + +## [0.18.3.post0] - 2024-11-10 +### 📖 Documentation +- Significant documentation revisions for clarity and precision. +- Also, updates outdated documentation for `CAP.get_caps` to clarify that figure generation and saving parameters can +be used with any `cluster_selection_method`, not just specific ones as previously implied. + +## [0.18.3] - 2024-11-10 +### 🐛 Fixes +- More conservative maxsizes for `@lru_cache`, change `@cache` in `TimeseriesExtractor` to `@lru_cache`. +- Clean unused import. + +## [0.18.2] - 2024-11-09 +- A simple pre commit hook added to remove a few trailing whitespace, add new lines, etc. + +### ♻ Changed +- Add's `_get_target_indices` and `_build_tree` to the init file for a shorter import path if cache needs to be cleared or +assessed. essentially allows: + +New import: +```python +from neurocaps._utils import _build_tree, _get_target_indices +``` + +Previous import: +```python +from neurocaps._utils.analysis.cap2statmap import ( + _build_tree, + _get_target_indices, +) +``` + +### 📖 Documentation +- Very minor doc fix. + +## [0.18.1] - 2024-11-08 +### 🚀 New/Added +- Added "cbarlabels_size" kwarg to several plots to allow the font size of the colorbar labels to increase or decrease. +### ♻ Changed +- For `CAP.caps2radar`, when the `legend` kwarg is set to None, the legend will be removed entirely. + +## [0.18.0] - 2024-11-07 +### ♻ Changed +- In `TimeseriesExtractor.get_bold`, location of `parallel_log_config` parameter in function signature moved +from being the last parameter to underneath `n_cores`. Additionally, `exclude_niftis` moved from being the second to last +parameter to being underneath `exclude_subjects`. + +*Changes related to `knn_dict`, which is only relevant for certain atlases that project poorly to surface space or +has a sparsity issue* + +- Added a "reference_atlas" key to allow Schaefer or AAL to be used as the reference atlas. +- The "remove_subcortical" key changed to "remove_labels". +- Default "k" from 1 to 3. + +### 🐛 Fixes +*Fixes only related to `knn_dict`* + +- "remove_labels" now only removes the labels from being interpolated as opposed to removing the label from being +interpolated in addition to removing the corresponding indices from the atlas entirely. +- Certain internal helper functions - `_get_target_indices` and `_build_tree` - from +`neurocaps._utils.analyis.cap2statmap` now use functool's `lru_cache` decorator so that the indices that the +non-background coordinated that need interpolation as well as the indices that don't need interpolation, based on +"remove_labels" are only computed once per session for every unique parameter combination. +- Logged information related to `knn_dict` appears once per call of `CAP.caps2niftis` or `CAP.caps2surf` instead of +for every iteration performed within these functions. + +### 📖 Documentation +- Some documentation revisions. + +## [0.17.11.post1] - 2024-11-04 +### 📖 Documentation +- Documentation clarification + +## [0.17.11.post0] - 2024-11-03 +### 📖 Documentation +- Minor fix to a versionchanged directive + +## [0.17.11] - 2024-11-03 +### ♻ Changed +- In `CAP.calculate_metrics`, if `continuous_runs` used, then in the "Run" column, the labels for these runs will +now follow the "run-" format and has changed from "continuous_runs" to run-continuous" +- Also in `CAP.calculate_metrics`, the group labels in the dataframe will no longer replace whitespace with underline. +Do names such as "High ADHD" wil no longer change to "High_ADHD" in the dataframe. +- The averaged transition matrix dataframe from `transition_matrix` now contains to index name "From\To" to show that +the index CAPs are "From" and the column CAPs are "To". +### 🐛 Fixes +- More robust error checking to ensure that the ``subject_timeseries`` follows the correct format when the +``subject_timeseries`` setter property is used in ``TimeseriesExtractor``. +- For ``space`` setter property is used in ``TimeseriesExtractor``, checks to ensure it is a string. + +## [0.17.10] - 2024-11-02 +### ♻ Changed +- Logger names now use `__name__` instead of `__name__.split(".")[-1]` +- Module folder and file naming in `neurocaps._utils` changed, only `_utils` has the leading underscore. +- Default logger now includes logger name in logged message. +### 🐛 Fixes +- Prevents logging duplication in certain user-defined logging scenarios when logs redirected. + +## [0.17.9.post0] - 2024-11-02 +### 📖 Documentation +- Minor doc changes + +## [0.17.9] - 2024-10-31 +### 📖 Documentation +- Enhanced documentation +- Documentation renders properly with Pylance in VSCode. +### ♻ Changed +- The ``parcel_approach`` stored as an property will remove the initial configuration sub-keys parameters ("n_rois", +"resolution_mm", "yeo_networks" for Schaefer and "version" for AAL) after the information from these sub-keys are used +internally to retrieve the appropriate parcellation. Thus, the dictionary will retain the primary keys: +"maps", "nodes", and "regions". However, similar to previous behavior, additional sub-keys can still be added, such as +adding a sub-key consisting of metadata, but they will be ignored and won't affect core operation. +- When ``parcel_approach`` is included as a setter, or the required keys ("maps", "nodes", and "regions") +are detected, it will no longer reset the "maps", "nodes", and "regions" for "Schaefer" and "AAL". This allows the +"maps", "nodes", and "regions" to be modified for "Schaefer" and "AAL" if needed. + +## [0.17.8.post1] - 2024-10-30 +### 💻 Metadata +- Readme revisions on Pypi. + +## [0.17.8.post0] - 2024-10-30 +### 💻 Metadata +- Shortens Readme on Pypi. + +## [0.17.8] - 2024-10-29 +### 🚀 New/Added +- For `TimeseriesExtractor.get_bold()`, a new `parallel_log_config` parameter has been added to pass an instance of +`multiprocessing.Manager.Queue` to redirect logs if parallel processing is used. +### ♻ Changed +- For `TimeseriesExtractor.get_bold()`, logging of additional subject-specific messages are also controlled by `verbose`. + +## [0.17.7] - 2024-10-25 +### 🚀 New/Added +- Added `vmin` and `vmin` kwargs for `transition_matrix` and `CAP.caps2corr`. +### 🐛 Fixes +- Fix issue with x and y labels for `CAP.caps2plot` not changing in size when `xticklabels_size` and +`yticklabels_size` modified. + +## [0.17.6] - 2024-10-19 +### 🚀 New/Added +- Added "n_before" and "n_after" subkeys for when `fd_threshold` is a dictionary. This the frame exceeding the +fd threshold to be scrubbed including an additional "n" number of frames before or after the exceeding frame to also +be scrubbed. +### 🐛 Fixes +- Log message that specifies certain confounds not found is now set as a warning instead of info. Additionally, there +is no Nonetype error if zero confounds are found in the data frame. +- Fix that appears to have only affected Windows where the system defined a default root handler (stderr) after the +top level loggers were initialized. This caused loggers initialized inside functions, such as `_extract_timeseries` +to adopt this root handler and timeseries extraction logs had a slightly different formatting. This is now fixed with +the following internal code. + +```python +"""Internal logging function and class for flushing""" + +import logging, sys + +# Global variables to determine if a handler is user defined or defined by OS +_USER_ROOT_HANDLER = None +_USER_MODULE_HANDLERS = {} + + +class _Flush(logging.StreamHandler): + def emit(self, record): + super().emit(record) + self.flush() + + +def _logger(name, flush=False, top_level=True): + global _USER_ROOT_HANDLER, _USER_MODULE_HANDLERS + + logger = logging.getLogger(name.split(".")[-1]) + + # Windows appears to assign stderr has the root handler after the top level loggers are assigned, which causes + # any loggers not assigned at top level to adopt this handler. Global variable used to assess if the base root + # handler is user defined or assigned by the system + if top_level == True: + _USER_ROOT_HANDLER = logging.getLogger().hasHandlers() + _USER_MODULE_HANDLERS[logger.name] = logging.getLogger(logger.name).hasHandlers() + + if not logger.level: + logger.setLevel(logging.INFO) + + # Check if user defined root handler or assigned a specific handler for module + default_handlers = _USER_ROOT_HANDLER or _USER_MODULE_HANDLERS[logger.name] + + # Works to see if root has handler and propagate if it does + logger.propagate = True if _USER_ROOT_HANDLER else False + + # Add or messages will repeat several times due to multiple handlers if same name used + if not default_handlers and not (logger.name == "_extract_timeseries" and top_level): + # If no user specified default handler, any handler is assigned by OS and is cleared + if logger.name == "_extract_timeseries": + logger.handlers.clear() + + if flush: + handler = _Flush(sys.stdout) + else: + handler = logging.StreamHandler(sys.stdout) + + handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) + logger.addHandler(handler) + + return logger +``` + +## [0.17.5] - 2024-10-14 +### 🚀 New/Added +- Added `dtype` parameter to `TimeseriesExtractor` with default set to None. This parameter is passed to nilearn's +`load_img` function. +### ♻ Changed +- Some speed increase if calling `TimeseriesExtractor.get_bold` multiple times in the same script. New internal function +uses `functools` `@cache` when calling `BIDSLayout` now. + +```python +@staticmethod +@cache +def _call_layout(bids_dir, pipeline_name): + try: + import bids + except ModuleNotFoundError: + raise ModuleNotFoundError( + "This function relies on the pybids package to query subject-specific files. " + "If on Windows, pybids does not install by default to avoid long path error issues " + "during installation. Try using `pip install pybids` or `pip install neurocaps[windows]`." + ) + + bids_dir = os.path.normpath(bids_dir).rstrip(os.path.sep) + + if bids_dir.endswith("derivatives"): + bids_dir = os.path.dirname(bids_dir) + + if pipeline_name: + pipeline_name = ( + os.path.normpath(pipeline_name).lstrip(os.path.sep).rstrip(os.path.sep) + ) + if pipeline_name.startswith("derivatives"): + pipeline_name = pipeline_name[len("derivatives") :].lstrip(os.path.sep) + layout = bids.BIDSLayout( + bids_dir, + derivatives=os.path.join(bids_dir, "derivatives", pipeline_name), + ) + else: + layout = bids.BIDSLayout(bids_dir, derivatives=True) + + LG.info(f"{layout}") + + return layout +``` + +## [0.17.4] - 2024-10-12 +- `CAP.caps2radar` test for Github Actions added due to this [action](https://github.com/coactions/setup-xvfb) +resolving the VTK issue. Test only for Linux. Now all functions are tested on Github Actions. +### 🐛 Fixes +- `CAP.caps2radar` uses try except block to show figures and should be able to display and close figures in a +Jupyter notebook or python CL. +- In `CAP.caps2radar` "round" kwarg was never used in code and was removed. +- When tick values are not specified in the `radialaxis` kwargs in `CAP.caps2radar` function, default tick values +no longer produces an error and now does the following: + +```python +if "tickvals" not in plot_dict["radialaxis"] and "range" not in plot_dict["radialaxis"]: + default_ticks = [ + max_value / 4, + max_value / 2, + 3 * max_value / 4, + max_value, + ] + plot_dict["radialaxis"]["tickvals"] = [round(x, 2) for x in default_ticks] +``` +### ♻ Changed +- To prevent numerical stability issues when scaling, previous versions did the following when the standard deviation +for a column was essentially zero: + +```python +std = np.std(arr, axis=0, ddof=1) +eps = np.finfo(np.float64).eps +std[std < eps] = 1.0 +``` + +The same method is employed; however, the smallest positive float representation used now considers the dtype +instead of being set to the smallest representation for "float64": +```python +std = np.std(arr, axis=0, ddof=1) +eps = np.finfo(std.dtype).eps +std[std < eps] = 1.0 +``` + +## [0.17.3] - 2024-10-08 +### 🐛 Fixes +- Fixes specific error that occurs when using a suffix name and saving nifti in `CAP.caps2surf`. + +## [0.17.2.post0] - 2024-10-06 +### 💻 Metadata +- Minor clarification in `CAP.caps2radar` function + +## [0.17.2] - 2024-10-06 +### ♻ Changed +- Internal refactoring and minor change to saved filenames for some functions for consistency. + +## [0.17.1] - 2024-10-01 +### ♻ Changed +- The `CAP.caps2radar` function now calculates the cosine similarity to the positive and negative activations of +a CAP cluster centroid separately. Each region has two traces (one for cosine similarity with positive activation +and one for cosine similarity with negative calculations). The plots should be easier to interpret and aligns better +with visualizations in CAP research. + +## [0.17.0] - 2024-09-21 +### 🚀 New/Added +- In `CAP.caps2radar`, "round" (rounds to three decimal points by default) and "linewidth" kwargs added. +### ♻ Changed +- In `TimeseriesExtractor`, `parcel_approach` and `fwhm` have changed positions. `parcel_approach` is second in the +list and `fwhm` is seventh in the list. +- `flush_print` changed to `flush`. +- In `CAP.caps2radar`, both `method` and `alpha` removed. Only the traditional cosine similarity calculation is +computed. +- In `CAP.calculate_metrics`, calculation for counts changed to abide by the formula, +temporal fraction = (persistence * counts)/total volumes which can be found in the supplemental of +Yang et al., 2021](https://doi.org/10.1016/j.neuroimage.2021.118193). Counts is now the frequency of initiations +of a specific CAP. +### 💻 Metadata +- Version directives less than 0.16.0 removed. + +## [0.16.5] - 2024-09-16 +- This update exclusively relates to improving documentation as well as improving the language in the error and +information messages for clarity. For instance, when a subject is skipped during timeseries extraction, instead of +`"[SUBJECT: 01 | SESSION: 002 | TASK: rest] Processing skipped: {message}"` it is now +`"[SUBJECT: 01 | SESSION: 002 | TASK: rest] Timeseries Extraction Skipped: {message}"`. Language in primarily in some +function descriptions have also been included. + +## [0.16.4] - 2024-09-16 +### ♻ Changed +- All uses of `print` and `warnings.warn` in package replaced with `logging.info` and `logging.warning`. The internal +function that creates the logger: +```python +import logging, sys + + +class _Flush(logging.StreamHandler): + def emit(self, record): + super().emit(record) + self.flush() + + +def _logger(name, level=logging.INFO, flush=False): + logger = logging.getLogger(name.split(".")[-1]) + logger.setLevel(level) + # Works to see if root has handler and propagate if it does + logger.propagate = logging.getLogger().hasHandlers() + # Add or messages will repeat several times due to multiple handlers if same name used + if not logger.hasHandlers(): + if flush: + handler = _Flush(sys.stdout) + else: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")) + logger.addHandler(handler) + + return logger +``` +- **Note**: The logger is initialized within the [internal time series extraction function]((https://github.com/donishadsmith/neurocaps/blob/900bf7a89d3ff16a8dd91310c8d177c5b5d6de8a/neurocaps/_utils/_timeseriesextractor_internals/_extract_timeseries.py#L12)) to ensure that each child +process has its own independent logger. This guarantees that subject-level information and warnings will be properly +logged, regardless of whether parallel processing is used or not. + +For non-parallel processing, the logger can be configured by a user with a command like the following: + +```python +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.StreamHandler(sys.stdout), + logging.FileHandler("info.out"), + ], +) +``` + +- Subject-specific messages are now more compact. + +**OLD:** +``` +List of confound regressors that will be used during timeseries extraction if available in confound dataframe: Cosine*, Rot*. + +BIDS Layout: ...0.4_ses001-022/ds000031_R1.0.4 | Subjects: 1 | Sessions: 1 | Runs: 1 + +[SUBJECT: 01 | SESSION: 002 | TASK: rest | RUN: 001] +---------------------------------------------------- +Preparing for timeseries extraction using - [FILE: '/Users/runner/work/neurocaps/neurocaps/tests/ds000031_R1.0.4_ses001-022/ds000031_R1.0.4/derivatives/fmriprep_1.0.0/fmriprep/sub-01/ses-002/func/sub-01_ses-002_task-rest_run-001_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz'] + +[SUBJECT: 01 | SESSION: 002 | TASK: rest | RUN: 001] +---------------------------------------------------- +The following confounds will be for nuisance regression: Cosine00, Cosine01, Cosine02, Cosine03, Cosine04, Cosine05, Cosine06, RotX, RotY, RotZ, aCompCor02, aCompCor03, aCompCor04, aCompCor05. +``` + +**NEW:** +``` +2024-09-16 00:17:11,689 [INFO] List of confound regressors that will be used during timeseries extraction if available in confound dataframe: Cosine*, aComp*, Rot*. +2024-09-16 00:17:12,113 [INFO] BIDS Layout: ...0.4_ses001-022/ds000031_R1.0.4 | Subjects: 1 | Sessions: 1 | Runs: 1 +2024-09-16 00:17:13,914 [INFO] [SUBJECT: 01 | SESSION: 002 | TASK: rest | RUN: 001] Preparing for timeseries extraction using [FILE: sub-01_ses-002_task-rest_run-001_space-MNI152NLin2009cAsym_desc-preproc_bold.nii.gz]. +2024-09-16 00:17:13,917 [INFO] [SUBJECT: 01 | SESSION: 002 | TASK: rest | RUN: 001] The following confounds will be for nuisance regression: Cosine00, Cosine01, Cosine02, Cosine03, Cosine04, Cosine05, Cosine06, aCompCor00, aCompCor01, aCompCor02, aCompCor03, aCompCor04, aCompCor05, RotX, RotY, RotZ. +``` +*Note that only the absolute path is no longer outputted, only the file's basename* +*Jupyter Notebook may show an additional space between the "[" and "INFO" for subject level info* + +## [0.16.3.post0] - 2024-09-14 +### 💻 Metadata +- Uploading fixed readme to Pypi + +## [0.16.3] - 2024-09-14 +- Internal refactoring was completed, primarily in `CAPs.caps2plot`, `TimeseriesExtractor.get_bold`, and an +internal function `_extract_timeseries`. +- All existing pytest tests passed following the refactoring. +### 🐛 Fixes +- Minor improvements were made to error messages for better clarity. +- Annotations can now be specified for `CAP.caps2plot` regional heatmap. + +## [0.16.2.post1] - 2024-08-23 +### 💻 Metadata +- Fix truncated table in README, which did not show all values correctly due to missing an additional row header. + +## [0.16.2] - 2024-08-22 +### 🚀 New/Added +- Transition probabilities has been added to `CAP.calculate_metrics`. Below is a snippet from the codebase +of how the calculation is done. +```python +if "transition_probability" in metrics: + temp_dict[group].loc[len(temp_dict[group])] = [ + subj_id, + group, + curr_run, + ] + [ + 0.0 + ] * (temp_dict[group].shape[-1] - 3) + # Get number of transitions + trans_dict = { + target: np.sum( + np.where( + predicted_subject_timeseries[subj_id][curr_run][:-1] == target, + 1, + 0, + ) + ) + for target in group_caps[group] + } + indx = temp_dict[group].index[-1] + # Iterate through products and calculate all symmetric pairs/off-diagonals + for prod in products_unique[group]: + target1, target2 = prod[0], prod[1] + trans_array = predicted_subject_timeseries[subj_id][curr_run].copy() + # Set all values not equal to target1 or target2 to zero + trans_array[(trans_array != target1) & (trans_array != target2)] = 0 + trans_array[np.where(trans_array == target1)] = 1 + trans_array[np.where(trans_array == target2)] = 3 + # 2 indicates forward transition target1 -> target2; -2 means reverse/backward transition target2 -> target1 + diff_array = np.diff(trans_array, n=1) + # Avoid division by zero errors and calculate both the forward and reverse transition + if trans_dict[target1] != 0: + temp_dict[group].loc[indx, f"{target1}.{target2}"] = float( + np.sum(np.where(diff_array == 2, 1, 0)) / trans_dict[target1] + ) + if trans_dict[target2] != 0: + temp_dict[group].loc[indx, f"{target2}.{target1}"] = float( + np.sum(np.where(diff_array == -2, 1, 0)) / trans_dict[target2] + ) + + # Calculate the probability for the self transitions/diagonals + for target in group_caps[group]: + if trans_dict[target] == 0: + continue + # Will include the {target}.{target} column, but the value is initially set to zero + columns = temp_dict[group].filter(regex=rf"^{target}\.").columns.tolist() + cumulative = temp_dict[group].loc[indx, columns].values.sum() + temp_dict[group].loc[indx, f"{target}.{target}"] = 1.0 - cumulative +``` +Below is a simplified version of the above snippet. +```python +import itertools, math, pandas as pd, numpy as np + +groups = [["101", "A", "1"], ["102", "B", "1"]] +timeseries_dict = { + "101": np.array([1, 1, 1, 1, 2, 2, 1, 4, 3, 5, 3, 3, 5, 5, 6, 7]), + "102": np.array([1, 2, 1, 1, 3, 3, 1, 4, 3, 5, 3, 3, 4, 5, 6, 8, 7]), +} +caps = list(range(1, 9)) +# Get all combinations of transitions +products = list(itertools.product(caps, caps)) +df = pd.DataFrame( + columns=["Subject_ID", "Group", "Run"] + [f"{x}.{y}" for x, y in products] +) +# Filter out all reversed products and products with the self transitions +products_unique = [] +for prod in products: + if prod[0] == prod[1]: + continue + # Include only the first instance of symmetric pairs + if (prod[1], prod[0]) not in products_unique: + products_unique.append(prod) + +for info in groups: + df.loc[len(df)] = info + [0.0] * (df.shape[-1] - 3) + timeseries = timeseries_dict[info[0]] + # Get number of transitions + trans_dict = { + target: np.sum(np.where(timeseries[:-1] == target, 1, 0)) for target in caps + } + indx = df.index[-1] + # Iterate through products and calculate all symmetric pairs/off-diagonals + for prod in products_unique: + target1, target2 = prod[0], prod[1] + trans_array = timeseries.copy() + # Set all values not equal to target1 or target2 to zero + trans_array[(trans_array != target1) & (trans_array != target2)] = 0 + trans_array[np.where(trans_array == target1)] = 1 + trans_array[np.where(trans_array == target2)] = 3 + # 2 indicates forward transition target1 -> target2; -2 means reverse/backward transition target2 -> target1 + diff_array = np.diff(trans_array, n=1) + # Avoid division by zero errors and calculate both the forward and reverse transition + if trans_dict[target1] != 0: + df.loc[indx, f"{target1}.{target2}"] = float( + np.sum(np.where(diff_array == 2, 1, 0)) / trans_dict[target1] + ) + if trans_dict[target2] != 0: + df.loc[indx, f"{target2}.{target1}"] = float( + np.sum(np.where(diff_array == -2, 1, 0)) / trans_dict[target2] + ) + + # Calculate the probability for the self transitions/diagonals + for target in caps: + if trans_dict[target] == 0: + continue + # Will include the {target}.{target} column, but the value is initially set to zero + columns = df.filter(regex=rf"^{target}\.").columns.tolist() + cumulative = df.loc[indx, columns].values.sum() + df.loc[indx, f"{target}.{target}"] = 1.0 - cumulative +``` +- Added new external function - ``transition_matrix``, which generates and visualizes the average transition probabilities +for all groups, using the transition probability dataframe outputted by `CAP.calculate_metrics` + +## [0.16.1.post3] - 2024-08-07 +### 💻 Metadata +- Minor change to clarify the language in the docstring referring to the Custom parcellation approach and update readme +on PyPi for the installation instructions. + +## [0.16.1.post2] - 2024-08-06 +### 💻 Metadata +- Correct output for example in readme. + +## [0.16.1.post1] - 2024-08-06 +### 💻 Metadata +- Update outdated example in readme. + +## [0.16.1] - 2024-08-06 +### ♻ Changed +- For `knn_dict`, cKdtree is replaced with Kdtree and scipy is restricted to 1.6.0 or later since that is the version +were Kdtree used the C implementation. +- `TimeseriesExtractor.get_bold` can now be used on Windows, pybids still does not install by default to prevent +long path error but `pip install neurocaps[windows]` can be used for installation. +- All instances of textwrap replaced with normal strings, printed warnings or messages will be longer in length now +and occupies less vertical screen space. + +## [0.16.0] - 2024-07-31 +### ♻ Changed +- In `CAP.caps2surf`, the `save_stat_map` parameter has been changed to `save_stat_maps`. +- Slight improvements in a few errors/exceptions to improve their informativeness. +- Now, when a subject's run is excluded due to exceeding the fd threshold, the percentage of their volumes +exceeding the threshold is given as opposed to simply stating that they have been excluded. +### 🐛 Fixes +- Fix a specific instance when `tr` is not specified for `TimseriesExtractor.get_bold`. When the `tr` is not specified, +the code attempts to check the the bold metadata/json file in the derivatives directory to extract the +repetition time. Now, it will check for this file in both the derivatives and root bids dir. The code will also +raise an error earlier if the tr isn't specified, cannot be extracted from the bold metadata file, and bandpass filtering +is requested. +- A warning check that is done to assess if indices for a certain condition is outside a possible range due to +duration mismatch, incorrect tr, etc is now also done before calculating the percentage of volumes exceeding the threshold +to not dilute calculations. Before this check was only done before extracting the condition from the timeseries array. +### 💻 Metadata +- Very minor documentation updates for `TimseriesExtractor.get_bold`. + +## [0.15.2] - 2024-07-23 +### ♻ Changed +- Created a specific message when dummy_scans = {"auto": True} and zero "non_steady_state_outlier_XX" are found +when `verbose=True`. +- Regardless if `parcel_approach`, whether used as a setter or input, accepts pickles. +### 🐛 Fixes +Fixed a reference before assignment issue in `merge_dicts`. This occurred when only the merged dictionary was requested +to be saved without saving the reduced dictionaries, and no user-provided file_names were given. In this scenario, +the default name for the merged dictionary is now correctly used. + +## [0.15.1] - 2024-07-23 +### 🚀 New/Added +- In `TimeseriesExtractor`, "min" and "max" sub-keys can now be used when `dummy_scans` is a dictionary and the +"auto" sub-key is True. The "min" sub-key is used to set the minimum dummy scans to remove if the number of +"non_steady_state_outlier_XX" columns detected is less than this value and the "max" sub-key is used to set the +maximum number of dummy scans to remove if the number of "non_steady_state_outlier_XX" columns detected exceeds this +value. + +## [0.15.0] - 2024-07-21 +### 🚀 New/Added +- `save_reduced_dicts` parameter to `merge_dicts` so that the reduced dictionaries can also be saved instead of only +being returned. + +### ♻ Changed +- Some parameter names, inputs, and outputs for non-class functions - `merge_dicts`, `change_dtype`, and `standardize` +have changed to improve consistency across these functions. + - `merge_dicts` + - `return_combined_dict` has been changed to `return_merged_dict`. + - `file_name` has been changed to `file_names` since the reduced dicts can also be saved now. + - Key in output dictionary containing the merged dictionary changed from "combined" to "merged". + - `standardize` & `change_dtypes` + - `subject_timeseries` has been changed to `subject_timeseries_list`, the same as in `merge_dicts`. + - `file_name` has been changed to `file_names`. + - `return_dict` has been changed to `return_dicts`. +- The returned dictionary for `merge_dicts`, `change_dtype`, and `standardize` is only +`dict[str, dict[str, dict[str, np.ndarray]]]` now. + +- In `CAP.calculate_metrics`, the metrics calculations, except for "temporal_fraction" have been refactored to remove an +import or use numpy operations to reduce needed to create the same calculation. + - **"counts"** + - Previous Code: + ```python + # Get frequency + frequency_dict = dict( + collections.Counter(predicted_subject_timeseries[subj_id][curr_run]) + ) + # Sort the keys + sorted_frequency_dict = {key: frequency_dict[key] for key in sorted(list(frequency_dict))} + # Add zero to missing CAPs for participants that exhibit zero instances of a certain CAP + if len(sorted_frequency_dict) != len(cap_numbers): + sorted_frequency_dict = { + cap_number: ( + sorted_frequency_dict[cap_number] + if cap_number in list(sorted_frequency_dict) + else 0 + ) + for cap_number in cap_numbers + } + # Replace zeros with nan for groups with less caps than the group with the max caps + if len(cap_numbers) > group_cap_counts[group]: + sorted_frequency_dict = { + cap_number: ( + sorted_frequency_dict[cap_number] + if cap_number <= group_cap_counts[group] + else float("nan") + ) + for cap_number in cap_numbers + } + ``` + - Refactored Code: + ```python + # Get frequency; + frequency_dict = { + key: np.where(predicted_subject_timeseries[subj_id][curr_run] == key, 1, 0).sum() + for key in range(1, group_cap_counts[group] + 1) + } + # Replace zeros with nan for groups with less caps than the group with the max caps + if max(cap_numbers) > group_cap_counts[group]: + for i in range(group_cap_counts[group] + 1, max(cap_numbers) + 1): + frequency_dict.update({i: float("nan")}) + ``` + - **"temporal_fraction"** + - Previous Code: + ```python + proportion_dict = { + key: item / (len(predicted_subject_timeseries[subj_id][curr_run])) + for key, item in sorted_frequency_dict.items() + } + ``` + - "Refactored Code": Nothing other than some parameter names have changed. + ```python + proportion_dict = { + key: value / (len(predicted_subject_timeseries[subj_id][curr_run])) + for key, value in frequency_dict.items() + } + ``` + - **"persistence"** + - Previous Code: + ```python + # Initialize variable + persistence_dict = {} + uninterrupted_volumes = [] + count = 0 + # Iterate through caps + for target in cap_numbers: + # Iterate through each element and count uninterrupted volumes that equal target + for index in range(0, len(predicted_subject_timeseries[subj_id][curr_run])): + if predicted_subject_timeseries[subj_id][curr_run][index] == target: + count += 1 + # Store count in list if interrupted and not zero + else: + if count != 0: + uninterrupted_volumes.append(count) + # Reset counter + count = 0 + # In the event, a participant only occupies one CAP and to ensure final counts are added + if count > 0: + uninterrupted_volumes.append(count) + # If uninterrupted_volumes not zero, multiply elements in the list by repetition time, sum and divide + if len(uninterrupted_volumes) > 0: + persistence_value = np.array(uninterrupted_volumes).sum() / len( + uninterrupted_volumes + ) + if tr: + persistence_dict.update({target: persistence_value * tr}) + else: + persistence_dict.update({target: persistence_value}) + else: + # Zero indicates that a participant has zero instances of the CAP + persistence_dict.update({target: 0}) + # Reset variables + count = 0 + uninterrupted_volumes = [] + + # Replace zeros with nan for groups with less caps than the group with the max caps + if len(cap_numbers) > group_cap_counts[group]: + persistence_dict = { + cap_number: ( + persistence_dict[cap_number] + if cap_number <= group_cap_counts[group] + else float("nan") + ) + for cap_number in cap_numbers + } + ``` + - Refactored Code: + ```python + # Initialize variable + persistence_dict = {} + # Iterate through caps + for target in cap_numbers: + # Binary representation of array - if [1,2,1,1,1,3] and target is 1, then it is [1,0,1,1,1,0] + binary_arr = np.where(predicted_subject_timeseries[subj_id][curr_run] == target, 1, 0) + # Get indices of values that equal 1; [0,2,3,4] + target_indices = np.where(binary_arr == 1)[0] + # Count the transitions, indices where diff > 1 is a transition; diff of indices = [2,1,1]; + # binary for diff > 1 = [1,0,0]; thus, segments = transitions + first_sequence(1) = 2 + segments = np.where(np.diff(target_indices, n=1) > 1, 1, 0).sum() + 1 + # Sum of ones in the binary array divided by segments, then multiplied by 1 or the tr; segment is + # always 1 at minimum due to + 1; np.where(np.diff(target_indices, n=1) > 1, 1,0).sum() is 0 when empty or the condition isn't met + persistence_dict.update({target: (binary_arr.sum() / segments) * (tr if tr else 1)}) + + # Replace zeros with nan for groups with less caps than the group with the max caps + if max(cap_numbers) > group_cap_counts[group]: + for i in range(group_cap_counts[group] + 1, max(cap_numbers) + 1): + persistence_dict.update({i: float("nan")}) + ``` + - **"transition_frequency"** + - Previous Code: + ```python + count = 0 + # Iterate through predicted values + for index in range(0, len(predicted_subject_timeseries[subj_id][curr_run])): + if index != 0: + # If the subsequent element does not equal the previous element, this is considered a transition + if ( + predicted_subject_timeseries[subj_id][curr_run][index - 1] + != predicted_subject_timeseries[subj_id][curr_run][index] + ): + count += 1 + # Populate DataFrame + new_row = [subj_id, group_name, curr_run, count] + df_dict["transition_frequency"].loc[len(df_dict["transition_frequency"])] = new_row + ``` + + - Refactored Code: + ```python + # Sum the differences that are not zero - [1,2,1,1,1,3] becomes [1,-1,0,0,2], binary representation + # for values not zero is [1,1,0,0,1] = 3 transitions + transition_frequency = np.where( + np.diff(predicted_subject_timeseries[subj_id][curr_run]) != 0, 1, 0 + ).sum() + ``` + *Note, the `n` parameter in `np.diff` defaults to 1, and differences are calculated as `out[i] = a[i+1] - a[i]`* +### 🐛 Fixes +- When a pickle file was used as input in `standardize` or `change_dtype` an error was produced, this has been fixed +and these functions accept a list of dictionaries or a list of pickle files now. + +### 💻 Metadata +- In the documentation for `CAP.caps2corr` it is now explicitly stated that the type of correlation being used is +Pearson correlation. + +## [0.14.7] - 2024-07-17 +### ♻ Changed +- Improved Warning Messages and Print Statements: + - In TimeseriesExtractor.get_bold, the subject-specific information output has been reformatted for better readability: + + - Previous Format: + ``` + Subject: 1; run:1 - Message + ``` + + - New Format: + ``` + [SUBJECT: 1 | SESSION: 1 | TASK: rest | RUN: 1] + ----------------------------------------------- + Message + ``` + + - In `CAP` class numerous warnings and statements have been changed to improve clarity: + + - Previous Format: + ``` + Optimal cluster size using silhouette method for A is 2. + ``` + + - New Format: + ``` + [GROUP: A | METHOD: silhouette] - Optimal cluster size is 2. + ``` + + - These changes should improve clarity when viewing in a terminal or when redirected to an output file by SLURM. + - Language in many statements and warnings have also been improved. + +## [0.14.6] - 2024-07-16 +### 🐛 Fixes +- For `CAP.get_caps`, when `cluster_selection_method` was used to find the optimal cluster size, the model would be +re-estimated and stored in the `self.kmeans` property for later use. Previously, the internal function that generated the +model using scikit's `KMeans` only returned the performance metrics. These metrics for each cluster size were assessed, +and the best cluster size was used to generate the optimal KMeans model with the same parameters. This is fine when +setting `random_state` with the same k since the model would produce the same initial cluster centroids and produces similar +clustering solution regardless of the number of times the model is re-generated. However, if a random state was not used, +the newly re-generated optimal model would technically differ despite having the same k, due to the random nature of KMeans +when initializing the cluster centroids. Now, the internal function returns both the performance metrics and the models, +ensuring the exact same model that was assessed is stored in the `self.kmeans`. Shouldn't be an incredibly major issue +if your models are generally stable and produce similar cluster solutions. Though when not using a random state, even +minor differences in the kmeans model even when using the same k can produce some statistical differences. Ultimately, +it is always best to ensure that the same model that the same model used for assessment and for later analyses are the +same to ensure robust results. + +## [0.14.5] - 2024-07-16 +### ♻ Changed +- In `TimeseriesExtractor`, `dummy_scans` can now be a dictionary that uses the "auto" sub-key if "auto" is set to +True, the number of dummy scans removed depend on the number of "non_steady_state_outlier_XX" columns in the +participants fMRIPrep confounds tsv file. For instance, if there are two "non_steady_state_outlier_XX" columns +detected, then `dummy_scans` is set to two since there is one "non_steady_state_outlier_XX" per outlier volume for +fMRIPrep. This is assessed for each run of all participants so ``dummy_scans`` depends on the number number of +"non_steady_state_outlier_XX" in the confound file associated with the specific participant, task, and run number. +### 🐛 Fixes +- For defensive programming purposes, instead of assuming the timing information in the event file perfectly +coincides with the timeseries. When a condition is specified and onset and duration must be used to extract the +indices corresponding to the condition of interest, the max scan index is checked to see if it exceeds the length of +the timeseries. If this condition is met, a warning is issued in the event of timing misalignment (i.e errors in event +file, incorrect repetition time, etc) and invalid indices are ignored to only extract the valid indices from the timeseries. +This is done in the event this was that are greater than the timeseries shape are ignored. + +## [0.14.4] - 2024-07-15 +### ♻ Changed +- Minor update that prints the optimal cluster size for each group when using `cluster_selection_method` in +`CAP.get_caps`. Just for information purposes. +- When error raised due to kneed not being able to detect the elbow, the group it failed for is now stated. +- Previously version 0.14.3.post1 + +## [0.14.3.post1] - YANKED +### ♻ Changed +- Minor update that prints the optimal cluster size for each group when using `cluster_selection_method` in +`CAP.get_caps`. Just for information purposes. +- When error raised due to kneed not being able to detect the elbow, the group it failed for is now stated. +- Yanked due to not being a metadata update, this should be a patch update to denote a behavioral change, +this is now version 0.14.4 to adhere a bit better to versioning practices. + +## [0.14.3] - 2024-07-14 +- Thought of some minor changes. + +### ♻ Changed +- Added new warning if `fd_threshold` is specified but `use_confounds` is False since `fd_threshold` needs the confound +file from fMRIPrep. In previous version, censoring just didn't occur and never issued a warning. +- Changed the error exception types for cosine similarity in `CAP.caps2radar` from ValueError to ZeroDivisionError +- Added ValueError in `TimeseriesExtractor.visualize_bold` if both `region` and `roi_indx` is None. +- In `TimeseriesExtractor.visualize_bold` if `roi_indx` is a string, int, or list with a single element, a title is +added to the plot. + +## [0.14.2.post2] - 2024-07-14 +### 💻 Metadata +- Simply wanted the latest metadata update to be on Zenodo and to have the same DOI as I forgot to upload +version 0.14.2.post1 there. + +## [0.14.2.post1] - 2024-07-14 +### 💻 Metadata +- Updated a warning during timeseries extraction that only included a partial reason for why the indices for condition +have been filtered out. Added information about `fd_threshold` being the reason why. + +## [0.14.2] - 2024-07-14 +### ♻ Changed +- Implemented a minor code refactoring that allows runs flagged due to "outlier_percentage", runs were all volumes will +be scrubbed due to all volumes exceeding the threshold for framewise displacement, and runs were the specified condition +returns zero indices will not undergo timeseries extraction. +- Also clarified the language in a warning that occurs when all NifTI files have been excluded or missing for a subject. +### 🐛 Fixes +- If a condition does not exist in the event file, a warning will be issued if this occurs. This should prevent empty +timeseries or errors. In the warning the condition will be named in the event of a spelling error. +- Added specific error type to except blocks for the cosine similarities that cause a division by zero error. + +## [0.14.1.post1] - 2024-07-12 +### 💻 Metadata +- Updates typehint `fd_threshold` since it was only updated in the doc string. + +## [0.14.1] - 2024-07-12 +### ♻ Changed +- In `TimeseriesExtractor`, `fd_threshold` can now be a dictionary, which includes a sub-key called "outlier_percentage", +a float value between 0 and 1 representing a percentage. Runs where the proportion of volumes exceeding the "threshold" +is higher than this percentage are removed. If `condition` is specified in `self.get_bold`, only the runs where the +proportion of volumes exceeds this value for the specific condition of interest are removed. A warning is issued +whenever a run is flagged. +- As of now, flagging and removal of runs, due to "outlier_percentage", is conducted after timeseries extraction. +This was done to minimize disrupting the original code and for easier testing for feature reliability as significant +code refactoring could cause unintended behaviors and requires longer testing for reliability. In a future patch, runs +will be assessed to see if they meet the exclusion criteria due to "outlier_percentage" prior to extraction and will be +skipped if flagged. +### 💻 Metadata +- Warning issue if cosine similarity is 0. +- Minor improvements to warning clarity. +- Changelog versioning updated for transparency since patches may include changes to parameters to improve behavior or +added paramaters to fix behavior. But these changes will be backwards compatible. + +## [0.14.0] - 2024-07-07 +### 🚀 New/Added +- More flexibility when calculating cosine similarity in the `CAP.caps2radar` function. Now a `method` and `alpha` parameter +is added to choose between calculating "traditional" cosine similarity, a more "selective" cosine similarity, or +a "combined" approach where `alpha` is used to determine the relative contributions of the `traditional` and `selective` +approach. +### 🐛 Fixes +- Added try except blocks in `CAP.caps2radar`, to handle division by zero cases. +- In `CAP.caps2surf`, `as_outline` kwarg is now its own separate layer, which should allow the outline to be build +on top of the stat map when requested. + +## [0.13.5] - 2024-07-06 +### 🐛 Fixes +- For `knn_dict`, replaces method for majority vote to another method that is more appropriate for floats +when k is greater than 1. Current method is more appropriate for atlases, which have integer values. + +## [0.13.4.post1] - 2024-07-05 +### 💻 Metadata +- Spelling fix in error message to refer to the correct variable name. + +## [0.13.4] - 2024-07-05 +### 🐛 Fixes +- For `CAP.caps2surf` and `CAP.caps2niftis`, fwhm comes after the knn method, if requested. + +## [0.13.3] - 2024-07-05 +### 🐛 Fixes +- Adds a "remove_subcortical" key to `knn_dict`. +- Uses "nearest" interpolation for Schaefer resampling so the labels are retained. +- Fixes "resolution_mm" default in `knn_dict`, which was set to "1mm" instead of 1 if not specified. + +## [0.13.2] - 2024-07-05 +### 🐛 Fixes +- Certain custom atlases may not project well from volume to surface space. A new parameter, `knn_dict` has been added to +`CAP.caps2surf()` and `CAP.caps2niftis` to apply k-nearest neighbors (knn) interpolation while leveraging the +Schaefer atlas, which projects well from volumetric to surface space. +- No longer need to add `parcel_approach` when using `CAP.caps2surf` with `fslr_giftis_dict`. + +## [0.13.1] - 2024-06-30 +### ♻ Changed +- For `CAP.caps2radar`, the `scattersize` kwarg can be used to control the size of the scatter/markers regardless +if `use_scatterpolar` is used. + +## [0.13.0.post1] - 2024-06-28 +### 💻 Metadata +- Clarifies that the p-values obtained in `CAP.caps2corr` are uncorrected. + +## [0.13.0] - 2024-06-28 +### 🚀 New/Added +- Minor update that adds some features to `CAP.caps2corr()`, specifically adds three parameters - `return_df`, `save_df`, +and `save_plots`. Now, in addition to visualizing a correlation matrix, this function can also return a pandas dataframe +containing a correlation matrix, where each element in the correlation matrix is accompanied by its p-value in +parenthesis, which is followed by an asterisk (single asterisk for < 0.05, double asterisk for 0.01, and triple asterisk +for < 0.001). These dataframes can also be saves as csv files. +- All plotting functions that use matplotlib includes `bbox_inches` as a kwarg and defaults to "tight". +- Added `annot_kws` kwargs to `CAPs.caps2plot` and `CAP.caps2corr`. + +## [0.12.2] - 2024-06-28 +### ♻ Changed +- When specified, allows `runs` parameter to be string, int, list of strings, or list of integers instead of just lists. +Always ensures it is converted to list if integer or string. +- Clarifies warning if tr not specified in `TimeseriesExtractor` by stating the `tr` is set to `None` and that extraction +will continue. +- For `CAP.get_caps`, if runs is `None`, the `self.runs` property is just None instead of being set to "all". Only affects what +is returned by `self.runs` when nothing is specified. + +## [0.12.1.post2] - 2024-06-27 +### 💻 Metadata +- Includes the updated type hints in 0.12.1.post1 and removes the unsupported operand for compatibility with +Python 3.9. + +## [0.12.1.post1] - 2024-06-27 [YANKED] +### 💻 Metadata +- Additional type hint updates. +- **Reason for Yanking**: Yanked due to potentially unsupported operand for type hinting (the vertical bar `|`) +in earlier Python versions (3.9). + +## [0.12.1] - 2024-06-27 + +### ♻ Changed +- For `merge_dicts` sorts the run keys lexicographically so that subjects that don't have the earliest run-id in the +first dictionary due to not having that run or the run being excluded still have ordered run keys in the merged +dictionary. + +### 💻 Metadata +- Updates `runs` parameters type hints so that it is known that strings can be used to0. + +## [0.12.0] - 2024-06-26 +- Entails some code cleaning and verification to ensure that the code cleaned for clarity purposes produces the same +results. + +### 🚀 New/Added +- Davies Bouldin and Variance Ratio (Calinski Harabasz) added + +### ♻ Changed +- For `CAPs.calculate_metrics()` if performing an analysis on groups where each group has a different number of CAPs, then for "temporal_fraction", +"persistence", and "counts", "nan" values will be seen for CAP numbers that exceed the group's number of CAPs. + - For instance, if group "A" has 2 CAPs but group "B" has 4 CAPs, the DataFrame will contain columns for CAP-1, + CAP-2, CAP-3, and CAP-4. However, for all members in group "A", CAP-3 and CAP-4 will contain "nan" values to + indicate that these CAPs are not applicable to the group. This differentiation helps distinguish between CAPs + that are not applicable to the group and CAPs that are applicable but had zero instances for a specific member. + +### 🐛 Fixes +- Adds error earlier when tr is not specified or able to be retrieved form the bold metadata when the condition is specified +instead of allowing the pipeline to produce this error later. +- Fixed issue with `show_figs` in `CAP.caps2surf()` showing figure when set to False. + +## [0.11.3] - 2024-06-24 +### ♻ Changed +- With parallel processing, joblib outputs are now returned as a generator as opposed to the default, which is a list, +to reduce memory usage. + +## [0.11.2] - 2024-06-23 +### ♻ Changed +- Changed how ids are organized in respective group when initializing the `CAP` class. In version 0.11.1, the ids are +sorted lexicographically: +```python3 +self._groups[group] = sorted(list(set(self._groups[group]))) +``` +This doesn't affect functionality but it may be better to respect the original user ordering.This is no longer the case. + +## [0.11.1] - 2024-06-23 +### 🐛 Fixes +- Fix for python 3.12 when using `CAP.caps2surf`. + - Changes in pathlib.py in Python 3.12 results in an error message format change. The error message now includes + quotes (e.g., "not 'Nifti1Image'") instead of the previous format without quotes ("not Nifti1Image"). This issue + arises when using ``neuromaps.transforms.mni_to_fslr`` within CAP.caps2surf() as neuromaps captures the error as a + string and checks if "not Nifti1Image" is in the string to determine if the input is a NifTI image. As a patch, + if the error occurs, a temporary .nii.gz file is created, the statistical image is saved to this file, and it is + used as input for neuromaps.transforms.mni_to_fslr. The temporary file is deleted after use. Below is the code + implementing this fix. + +```python3 +# Fix for python 3.12, saving stat map so that it is path instead of a NifTi +try: + gii_lh, gii_rh = mni152_to_fslr(stat_map, method=method, fslr_density=fslr_density) +except TypeError: + # Create temp + temp_nifti = tempfile.NamedTemporaryFile(delete=False, suffix=".nii.gz") + warnings.warn(textwrap.dedent(f""" + Potential error due to changes in pathlib.py in Python 3.12 causing the error + message to output as "not 'Nifti1Image'" instead of "not Nifti1Image", which + neuromaps uses to determine if the input is a Nifti1Image object. + Converting stat_map into a temporary nii.gz file (which will be automatically + deleted afterwards) at {temp_nifti.name} + """)) + # Ensure file is closed + temp_nifti.close() + # Save temporary nifti to temp file + nib.save(stat_map, temp_nifti.name) + gii_lh, gii_rh = mni152_to_fslr( + temp_nifti.name, method=method, fslr_density=fslr_density + ) + # Delete + os.unlink(temp_nifti.name) +``` +- Final patch is for strings in triple quotes. The standard textwrap module is used to remove the indentations at each +new line. + +## [0.11.0.post2] - 2024-06-22 +### 💻 Metadata +- Very minor explanation added to `CAP.calculate_metrics` regarding using individual dictionaries from merged +dictionaries as inputs. + +## [0.11.0.post1] - 2024-06-22 +### 💻 Metadata +- Two docstring changes for `merge_dicts`, which includes nesting the return type hint and capitalizing all letters of +the docstring header for aesthetics. + +## [0.11.0] - 2024-06-22 +### 🚀 New/Added +- Added new function `change_dtype` to make it easier to change the dtypes of each subject's numpy array to assist with +memory usage, especially if doing the CAPs analysis on a local machine. +- Added new parameters - `output_dir`, `file_name`, and `return_dict` ` to `standardize` to save dictionary, the +`return_dict` defaults to True. +- Adds a new version attribute so you can check the current version using `neurocaps.__version__` + +### ♻ Changed +- Adds back python 3.12 classifier. The `CAP.caps2surf` function may still not work well but if its detected that +neurocaps is being installed using python 3.12, setuptools is installed to prevent the pkgresources error. + +### 🐛 Fixes +- Minor fix for `file_name` parameter in `merge_dicts`. If user does not supply a `file_name` when saving the dictionary, +it will provide a default file_name now instead of producing a Nonetype error. + +### 💻 Metadata +- Minor docstrings revisions, mostly to the typehint for ``subject_timeseries``. + +## [0.10.0.post2] - 2024-06-20 +### 💻 Metadata +- Minor metadata update to docstrings to remove curly braces from inside the list object of certain parameters to +not make it seem as if it is supposed to be a strings inside a dictionary which is inside a list as opposed to strings +in a list. + +## [0.10.0.post1] - 2024-06-19 +### 💻 Metadata +- Minor metadata update to denote that `run` and `runs` parameter can be a string too. + +## [0.10.0] - 2024-06-17 +### 🚀 New/Added +- `CAP` class as a `cosine_similarity` property and in `CAP.caps2radar`, there is now a `as_html` parameter to save +plotly's radar plots as an html file instead of a static png. The html files can be opened in a browser and saved as a +png from the browser. Most importantly, they are interactive. - **new to [0.10.0]** +- Made another internal attribute in CAP `CAP.subject_table` a property and setter. This property acts as a lookup +table. As a setter, it can be used to modify the table to use another subject dictionary with different subjects +not used to generate the k-means model. +- Can now plot silhouette score and have some control over the `x-axis` of elbow and silhouette plot with the "step" `**kwarg`. + +### ♻ Changed +- Default for `CAP.caps2plots` from "outer product" to "outer_product". +- Default for `CAP.calculate_metrics` from "temporal fraction" to "temporal_fraction" and "transition frequency" +to "transition_frequency". +- `n_clusters` and `cluster_selection_method` parameters moved to `CAP.get_caps` instead of being parameters in +`CAP`. + +### 🐛 Fixes +- Restriction that numpy must be less than version 2 since this breaks brainspace vtk, which is needed for plotting to +surface space. - **new to [0.10.0]** +- Adds nbformat as dependency for plotly. - **new to [0.10.0]** +- In `TimeseriesExtractor.get_bold`, several checks are done to ensure that subjects have the necessary files for +extraction. Subjects that have zero nifti, confound files (if confounds requested), event files (if requested), etc +are automatically eliminated from being added to the list for timeseries extraction. A final check assesses, the run +ID of the files to see if the subject has at least one run with all necessary files to avoid having subjects with all +the necessary files needed but all are from different runs. This is most likely a rare occurrence but it is better to be +safer to ensure that even a rare occurrence doesn't result in a crash. The continue statement that skips the subject +only worked if no condition was specified. +- Removes in-place operations when standardizing to avoid numpy casting issues due to incompatible dtypes. +- Additional deep copy such as deep copying any setter properties to ensure external changes does not result internal +changes. +- Some important fixes were left out of the original version. + - These fixes includes: + - Removal of the `epsilon` parameter in `self.get_caps` and replacement with `std[std < np.finfo(np.float64).eps] = 1.0` + to prevent divide by 0 issues and numerical instability issues. + - Deep copy `subject_timeseries` in `standardize` and `parcel_approach`. In their functions, in-place operations + are performed which could unintentionally change the external versions of these parameters +- Added try-except block in `TimeseriesExtractor.get_bold` when attempting to obtain the `tr`, to issue a warning +when `tr` isn't specified and can't be extracted from BOLD metadata. Extraction will be continued. +- Fixed error when using `silhouette` method without multiprocessing where the function called the elbow method instead +of the silhouette method. This error affects versions 0.9.6 to 0.9.9. +- Fix some file names of output by adding underscores for spaces in group names. + +### 💻 Metadata +- Drops the python 3.12 classifier. All functions except for `CAP.caps2surf` works on python 3.12. Additionally, for +python 3.12, you may need to use `pip install setuptools` if you receive an error stating that +"ModuleNotFoundError: No module named 'pkg_resources'". - new to [0.10.0] +- Ensure user knows that all image files are outputted as pngs. +- Clarifications of some doc strings, stating that Bessel's correction is used for standardizing and that for +`CAP.calculate_metrics` can accept subject timeseries not used for generating the k-means model. +- Corrects docstring for `standardize` from parameter being `subject_timeseries_list` to `subject_timeseries`. + +## [0.9.9.post3] - 2024-06-13 +### 🐛 Fixes +- Noted an issue with file naming in `CAP.calculate_metrics` that causes the suffix of the file name to append +to subsequent file names when requesting multiple metrics. While it doesn't effect the content inside the file it is an +irritating issue. For instance "-temporal_fraction.csv" became "-counts-temporal_fraction.csv" if user requested "counts" +before "temporal fraction". + +### 💻 Metadata +- But Zenodo on PyPi. + +## [0.9.9.post2] - 2024-06-13 +### 💻 Metadata +- All docstrings now at a satisfactory point of being well formatted and explanatory. +- Fixes issues with docstring not being formatted correctly when reading in an IDE like Jupyter notebook. + +## [0.9.9.post1] - 2024-06-12 +### 🐛 Fixes +- Reference before assignment issue when `use_confounds` is False do to `censor` only being when `use_confounds` +is True. + +## [0.9.9] - 2024-06-12 + +**Pylint used to check for potential errors and also used to clean code** + +### ♻ Changed +- `parcel_approach` no longer required when initializing `CAP`. It is still required for some plotting methods and the +user will be warned if it is None. This allows the use of certain methods without having to keep adding this parameter. +- For `CAP.calculate_metrics`, `file_name` parameter changed to `prefix_file_name` to better reflect that it will be +added as a prefix to the csv files. + +### 🐛 Fixes +- Fixed issue with no context manager or closing json file in `TimeseriesExtractor` where if `tr` is not specified, +the bold metadata is used to extract the tr. However, this was done without a context manager to ensure the file closes +properly afterwards. +- All imports, except for `pybids` are no longer imported in each function and are now at top level. + +## [0.9.8.post3] - 2024-06-10 +### 🐛 Fixes +- Adds a "mode" kwargs to `CAP.caps2radar` to override default plotly drawing behaviors and sets `use_scatterpolar` +argument to False. + +## [0.9.8.post2] - 2024-06-09 +### 💻 Metadata +- Significant improvements to docstrings and added homepage. + +## [0.9.8.post1] - 2024-06-08 +### 🐛 Fixes +- Uses plotly.offline to open plots generated by `CAP.caps2radar` in default browser when Python is non-interactive +to prevent hanging issue. + +## [0.9.8] - 2024-06-07 +### ♻ Changed +- Changed `vmax` and `vmin` kwargs in `CAP.caps2surf` to `color_range` +- In `CAP.caps2surf` the function no longer rounds max and min values and restricts range to -1 and 1 if the rounded +value is 0. +It just uses the max and min values from the data. + +## [0.9.8.rc1] - 2024-06-07 +🚀 New/Added +- New method in `CAP` class to plot radar plot of cosine similarity (`CAP.caps2radar`). +- New method in `CAP` class to save CAPs as niftis without plotting (`CAP.caps2niftis`). +- Added new parameter to `CAP.caps2surf`, `fslr_giftis_dict`, to allow CAPs statistical maps that were +converted to giftis externally, using tools such as Connectome Workbench, to be plotted. This parameter only requires +the `CAP` class to be initialized. + +## [0.9.7.post2] - 2024-06-03 +### ♻ Changed +- Minor change in merge_dicts() to make it explicitly clear that the dictionaries are returned in the order they are +provided in the list. Originally, the dictionaries were returned as a nested dictionary with sub-keys starting at +"dict_1" to represent the first dictionary given in the list. They now start at "dict_0" to represent the first +dictionary in the list. This doesn't affect the underlying functionality of the code; the sub-keys are simply numbered +to represent their original index in the provided list. + +## [0.9.7.post1] - 2024-06-03 +### 🐛 Fixes +- Allows user to change the maximum and minimum value displayed for `CAP.caps2plot` and `CAP.caps2surf` + +## [0.9.7] - 2024-06-02 +🚀 New/Added +- More plotting kwargs and ability to just show the left and right hemisphere when plotting nodes with `CAP.caps2plot` +for "Schaefer" and "Custom" parcellations. +- Added `suffix_title` parameter to `CAP.caps2corr` and `CAP.caps2surf`. + +### ♻ Changed +- Changed `task_title` parameter in `CAP.caps2plot` to `suffix_title`. + +## [0.9.6] - 2024-05-31 +Recommend this version if intending to use parallel processing since it uses `joblib`, which seems to be more memory +efficient than multiprocessing. + +🚀 New/Added +- Added `n_cores` parameter to `CAP.get_caps` for multiprocessing when using the silhouette or elbow method. +- More restrictions to the minimum versions allowed for dependencies. + +### ♻ Changed +- Use joblib for pickling (replaces pickle) and multiprocessing (replaces multiprocessing). + +## [0.9.5.post1] - 2024-05-30 +🚀 New/Added +- Added the `linecolor` **kwargs for `CAP.caps2corr` and `CAP.caps2plot` that should have been deployed in 0.9.5. + +## [0.9.5] - 2024-05-30 + +### 🚀 New/Added +- Added ability to create custom colormaps with `CAP.caps2surf` by simply using the cmap parameter with matplotlibs +`LinearSegmentedColormap` with the `cmap` kwarg. An example of its use can be seen in demo.ipynb and the in the README. +- Added `surface` **kwargs to `CAP.caps2surf` to use "inflated" or "veryinflated" for the surface plots. + +## [0.9.4.post1] - 2024-05-28 + +### 💻 Metadata +- Update some metadata on PyPi + +## [0.9.4] - 2024-05-27 + +### ♻ Changed + +- Improvements to docstrings in all methods in neurocaps. +- Restricts scikit-learn to version 1.4.0 and above. +- Reduced the number of default `confound_names` in the `TimeseriesExtractor` class that will be used if `use_confounds` +is True but no `confound_names` are specified. The new defaults are listed below. The previous default included +nonlinear motion parameters. +- Use default of "run-0" instead of "run-1" for the subkey in the `TimeseriesExtractor.subject_timeseries` for files +processed with `TimeseriesExtractor.get_bold` that do not have a run ID due to only being a single run in the dataset. + +```python +if high_pass: + confound_names = [ + "trans_x", + "trans_x_derivative1", + "trans_y", + "trans_y_derivative1", + "trans_z", + "trans_z_derivative1", + "rot_x", + "rot_x_derivative1", + "rot_y", + "rot_y_derivative1", + "rot_z", + "rot_z_derivative1", + ] +else: + confound_names = [ + "cosine*", + "trans_x", + "trans_x_derivative1", + "trans_y", + "trans_y_derivative1", + "trans_z", + "trans_z_derivative1", + "rot_x", + "rot_x_derivative1", + "rot_y", + "rot_y_derivative1", + "rot_z", + "rot_z_derivative1", + "a_comp_cor_00", + "a_comp_cor_01", + "a_comp_cor_02", + "a_comp_cor_03", + "a_comp_cor_04", + "a_comp_cor_05", + ] +``` + +## [0.9.3] - 2024-05-26 + +### 🚀 New/Added +- Supports nilearns versions 0.10.1, 0.10.2, 0.10.4, and above (does not include 0.10.3). + +### ♻ Changed +- Renamed `CAP.visualize_caps` to `CAP.caps2plot` for naming consistency with other methods for visualization in +the `CAP` class. + +## [0.9.2] - 2024-05-24 + +### 🚀 New/Added +- Added ability to create correlation matrices of CAPs with `CAP.caps2corr`. +- Added more **kwargs to `CAP.caps2surf`. Refer to the docstring to see optional **kwargs. + +### 🐛 Fixes +- Use the `KMeans.labels_` attribute for scikit's KMeans instead of using the `KMeans.predict` on the same dataframe +used to generate the model. It is unecessary since `KMeans.predict` will produce the same labels already stored in +`KMeans.labels_`. These labels are used for silhouette method. + +### ♻ Changed +- Minor aesthetic changes to some plots in the `CAP` class such as changing "CAPS" in the title of `CAP.caps2corr` +to "CAPs". + +## [0.9.1] - 2024-05-22 + +### 🚀 New/Added +- Ability to specify resolution for Schaefer parcellation. +- Ability to use spatial smoothing during timeseries extraction. +- Ability to save elbow plots. +- Add additional parameters - `fslr_density` and `method` to the `CAP.caps2surf` method to modify interpolation +methods from MNI152 to surface space. +- Increased number of parameters to use with scikit's `KMeans`, which is used in `CAP.get_caps`. + +### ♻ Changed +- In, `CAP.calculate_metrics` nans where used to signify the abscense of a CAP, this has been replaced with 0. Now +for persistence, counts, and temporal fraction, 0 signifies the absence of a CAP. For transition frequency, 0 means no +transition between CAPs. + +### 🐛 Fixes +- Fix for AAL surface plotting for `CAP.caps2surf`. Changed how CAPs are projected onto surface plotting by +extracting the actual sorted labels from the atlas instead of assuming the parcellation labels goes from 1 to n. +The function still assumes that 0 is the background label; however, this fixes the issue for parcellations that don't +go from 0 to 1 and go from 0 with the first parcellation label after zero starting at 2000 for instance. + +## [0.9.0] - 2024-05-13 + +### 🚀 New/Added +- Ability to project CAPs onto surface plots. + +## [0.8.9] - 2024-05-09 + +### 🚀 New/Added +- Added "Custom" as a valid keyword for `parcel_approach` in the `TimeseriesExtractor` and `CAP` classes to support +custom parcellation with bilateral nodes (nodes that have a left and right hemisphere version). Timeseries extraction, +CAPs extraction, and all visualization methods are available for custom parcellations. +- Added `exclude_niftis` parameter to `TimeseriesExtractor.get_bold` to skip over specific files during timeseries +extraction. +- Added `fd_threshold` parameter to `TimeseriesExtractor` to scrub frames that exceed a specific threshold after +nuisance regression is done. +- Added options to flush print statements during timeseries extraction. +- Added additional **kwargs for `CAP.visualize_caps`. + +### ♻ Changed +- Changed `network` parameter in `TimeseriesExtractor.visualize_bold` to `region`. +- Changed "networks" option in `visual_scope` parameter in `CAP.visualize_caps` to "regions". + +### 🐛 Fixes +- Fixed reference before assignment when specifying the repetition time (TR) when using the `tr` parameter in +`TimeseriesExtractor.get_bold`. Prior only extracting the TR from the metadata files, which is done if the `tr` +parameter was not specified worked. +- Allow bids datasets that do not specify run id or session id in their file names to be ran instead of producing an +error. Prior, only bids datasets that included "ses-#" and "run-#" in the file names worked. Files that do not have +"run-#" in it's name will include a default run-id in their sub-key to maintain the structure of the +`TimeseriesExtractor.subject_timeseries` dictionary". This default id is "run-1". +- Fixed error in `CAP.visualize_caps` when plotting "outer products" plot without subplots. + +## [0.8.8] - 2024-03-23 + +### 🚀 New/Added +- Support Windows by only allowing install of pybids if system is not Windows. On Windows machines +`TimeseriesExtractor` cannot be used but `CAP` and all other functions can be used. + +## [0.8.7] - 2024-03-15 + +### 🚀 New/Added +- Added `merge_dicts` to be able to combine different subject_timeseries and only return shared subjects. +- Print names of confounds used for each subject and run when extracting timeseries for transparency. +- Ability to extract timeseries using the AAL or Schaefer parcellation. +- Ability to use multiprocessing to speed up timeseries extraction. +- Can be used to extract task (entire task timeseries or a single specific condition) or resting-state data. +- Ability to denoise data during extraction using band pass filtering, confounds, detrending, and removing dummy scans. +- Can visualize the extracted timeseries at the node or network level. +- Ability to perform Co-activation Patterns (CAPs) analysis on separate groups or all subjects. +- Can use silhouette method or elbow method to determine optimal cluster size and the optimal kmeans model will be +saved. +- Can visualize kneed plots for elbow method. +- Can visualize CAPs using heatmaps or outer product plots at the network or node level of the Schaefer or AAL atlas. +- Can calculate temporal frequency, persistence, counts, and transition frequency. As well as save each as a csv file. diff --git a/demos/openneuro_demo.ipynb b/demos/openneuro_demo.ipynb index c8258aac..6b464dda 100644 --- a/demos/openneuro_demo.ipynb +++ b/demos/openneuro_demo.ipynb @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -58,7 +58,11 @@ " os.environ[\"DISPLAY\"] = \":0.0\"\n", " !apt-get install -y xvfb\n", " !Xvfb :0 -screen 0 1024x768x24 &> /dev/null &\n", - " !Xvfb :0 -screen 0 1024x768x24 &> /dev/null &" + " !Xvfb :0 -screen 0 1024x768x24 &> /dev/null &\n", + " !pip install jupyter_bokeh\n", + " import IPython\n", + " IPython.Application.instance().kernel.do_shutdown(True)\n", + " !yes | plotly_get_chrome" ] }, { @@ -3761,7 +3765,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.13.9" } }, "nbformat": 4, diff --git a/demos/simulated_demo.ipynb b/demos/simulated_demo.ipynb index f23b797c..52b7e6fa 100644 --- a/demos/simulated_demo.ipynb +++ b/demos/simulated_demo.ipynb @@ -41,7 +41,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -69,7 +69,11 @@ " os.environ[\"DISPLAY\"] = \":0.0\"\n", " !apt-get install -y xvfb\n", " !Xvfb :0 -screen 0 1024x768x24 &> /dev/null &\n", - " !Xvfb :0 -screen 0 1024x768x24 &> /dev/null &" + " !Xvfb :0 -screen 0 1024x768x24 &> /dev/null &\n", + " !pip install jupyter_bokeh\n", + " import IPython\n", + " IPython.Application.instance().kernel.do_shutdown(True)\n", + " !yes | plotly_get_chrome" ] }, { diff --git a/docker/Dockerfile b/docker/Dockerfile index b37534b1..2cbd2db3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -35,29 +35,26 @@ RUN apt-get update && apt-get install -y \ ln -sf /usr/bin/python3.12 /usr/bin/python3 && \ curl https://bootstrap.pypa.io/get-pip.py | python3.12 -# Create user and home directory; -# recursively change ownership of home directory to user; -# add user to sudo group -# Append no password requirement for sudo users to sudoers -RUN useradd -md /home/user user && \ - chown -R user /home/user && \ - adduser user sudo && \ - echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +ENV NILEARN_SHARED_DATA=/opt/neurocaps/data/nilearn_data \ + NEUROMAPS_DATA=/opt/neurocaps/data/neuromaps-data \ + NEUROCAPS_DATA=/opt/neurocaps/data/neurocaps_data -WORKDIR /home/user -ENV HOME="/home/user" +RUN mkdir -p /opt/neurocaps/src \ + /opt/neurocaps/demos \ + $NILEARN_SHARED_DATA \ + $NEUROMAPS_DATA \ + $NEUROCAPS_DATA + +WORKDIR /opt/neurocaps COPY neurocaps/ src/neurocaps/ -COPY demos/* demos/ -COPY docker/scripts/entrypoint.sh /usr/local/bin COPY pyproject.toml src/ -COPY tests/data/nilearn_data /home/user/nilearn_data -COPY tests/data/neuromaps-data /home/user/neuromaps-data +COPY demos/* demos/ +COPY tests/data/nilearn_data $NILEARN_SHARED_DATA +COPY tests/data/neuromaps-data $NEUROMAPS_DATA +COPY docker/scripts/entrypoint.sh /usr/local/bin/ -RUN chown -R user:user /home/user/ && \ - chmod -R 775 /home/user/ && \ - chown user /usr/local/bin/entrypoint.sh && \ - sed -i "s/\r$//" /usr/local/bin/entrypoint.sh && \ +RUN sed -i "s/\r$//" /usr/local/bin/entrypoint.sh && \ chmod +x /usr/local/bin/entrypoint.sh RUN pip install --upgrade pip setuptools && \ @@ -66,12 +63,12 @@ RUN pip install --upgrade pip setuptools && \ RUN yes | plotly_get_chrome -EXPOSE 9999 -LABEL maintainer="Donisha Smith " +RUN chmod -R a+rwX /opt/neurocaps RUN mkdir -p /tmp/.X11-unix && chmod 1777 /tmp/.X11-unix -USER user +EXPOSE 9999 +LABEL maintainer="Donisha Smith " ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/docker/scripts/entrypoint.sh b/docker/scripts/entrypoint.sh index b21a861b..1c387013 100644 --- a/docker/scripts/entrypoint.sh +++ b/docker/scripts/entrypoint.sh @@ -2,11 +2,7 @@ Xvfb :0 -screen 0 1600x1200x24 & export DISPLAY=:0 -while !xset q &> /dev/null; do - sleep 0.1 -done - -if [ "$1" = "notebook" ]; then +if [[ "$1" == "notebook" || "$1" == "jupyter" || "$1" == "jupyter-notebook" ]]; then exec jupyter notebook --allow-root --no-browser --ip=0.0.0.0 --port=9999 else exec "$@" diff --git a/docs/introduction.rst b/docs/introduction.rst index 5dea451b..2a09424a 100644 --- a/docs/introduction.rst +++ b/docs/introduction.rst @@ -9,7 +9,7 @@ :alt: Python Versions .. image:: https://img.shields.io/badge/DOI-10.5281%2Fzenodo.11642615-teal - :target: https://doi.org/10.5281/zenodo.18529846 + :target: https://doi.org/10.5281/zenodo.19602764 :alt: DOI .. image:: https://img.shields.io/badge/Source%20Code-neurocaps-purple diff --git a/docs/tutorials/tutorial-2.ipynb b/docs/tutorials/tutorial-2.ipynb index e7485205..ab036eb1 100644 --- a/docs/tutorials/tutorial-2.ipynb +++ b/docs/tutorials/tutorial-2.ipynb @@ -15,7 +15,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -37,7 +37,11 @@ " os.environ[\"DISPLAY\"] = \":0.0\"\n", " !apt-get install -y xvfb\n", " !Xvfb :0 -screen 0 1024x768x24 &> /dev/null &\n", - " !Xvfb :0 -screen 0 1024x768x24 &> /dev/null &" + " !Xvfb :0 -screen 0 1024x768x24 &> /dev/null &\n", + " !pip install jupyter_bokeh\n", + " import IPython\n", + " IPython.Application.instance().kernel.do_shutdown(True)\n", + " !yes | plotly_get_chrome" ] }, { diff --git a/docs/tutorials/tutorial-8.ipynb b/docs/tutorials/tutorial-8.ipynb index 0af2f81e..4f334e94 100644 --- a/docs/tutorials/tutorial-8.ipynb +++ b/docs/tutorials/tutorial-8.ipynb @@ -14,7 +14,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "62138ae7", "metadata": {}, "outputs": [], @@ -32,7 +32,11 @@ " os.environ[\"DISPLAY\"] = \":0.0\"\n", " !apt-get install -y xvfb\n", " !Xvfb :0 -screen 0 1024x768x24 &> /dev/null &\n", - " !Xvfb :0 -screen 0 1024x768x24 &> /dev/null &" + " !Xvfb :0 -screen 0 1024x768x24 &> /dev/null &\n", + " !pip install jupyter_bokeh\n", + " import IPython\n", + " IPython.Application.instance().kernel.do_shutdown(True)\n", + " !yes | plotly_get_chrome" ] }, { @@ -968,7 +972,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.0" + "version": "3.13.9" } }, "nbformat": 4, diff --git a/docs/user_guide/installation.rst b/docs/user_guide/installation.rst index f745dec6..219d5467 100644 --- a/docs/user_guide/installation.rst +++ b/docs/user_guide/installation.rst @@ -1,6 +1,6 @@ Installation ============ -**Requires Python 3.9-3.14.** +**Requires Python 3.10-3.14.** Standard Installation --------------------- diff --git a/hooks/grep.py b/hooks/grep.py index dc5199a8..1549ef41 100644 --- a/hooks/grep.py +++ b/hooks/grep.py @@ -1,7 +1,5 @@ import os, subprocess, sys -from typing import Union - IS_WINDOWS: bool = sys.platform == "win32" BASENAME: str = os.path.join(os.path.dirname(__file__).removesuffix("hooks").rstrip(os.sep), "neurocaps") FILES : list[str] = [ @@ -9,7 +7,7 @@ os.path.join(BASENAME, "extraction", "timeseries_extractor.py"), ] -def get_cmd(filename: str) -> Union[str, list[str]]: +def get_cmd(filename: str) -> str | list[str]: if not IS_WINDOWS: # Pattern used two negative look behinds to ignore "self" preceded by a backtick or del + whitespace # then a negative look ahead for "self" that is not followed by underscore, followed by word. @@ -23,7 +21,7 @@ def get_cmd(filename: str) -> Union[str, list[str]]: return cmd -def get_stdout(cmd: Union[str, list[str]]) -> str: +def get_stdout(cmd: str | list[str]) -> str: output = subprocess.run(cmd, shell=not IS_WINDOWS, capture_output=True, text=True) return output.stdout diff --git a/neurocaps/__init__.py b/neurocaps/__init__.py index 00bee7fd..0eb91783 100644 --- a/neurocaps/__init__.py +++ b/neurocaps/__init__.py @@ -22,4 +22,4 @@ __all__ = ["analysis", "extraction", "exceptions", "utils"] # Version in single place -__version__ = "0.37.2" +__version__ = "0.37.4" diff --git a/neurocaps/analysis/_internals/serialize.py b/neurocaps/analysis/_internals/serialize.py index 7d44f36a..da5616fe 100644 --- a/neurocaps/analysis/_internals/serialize.py +++ b/neurocaps/analysis/_internals/serialize.py @@ -1,5 +1,4 @@ import os -from typing import Union import joblib @@ -10,8 +9,8 @@ def dicts_to_pickles( output_dir: str, dict_list: list[dict], caller: str, - filenames: Union[str, None] = None, - message: Union[str, None] = None, + filenames: str | None = None, + message: str | None = None, save_reduced_dicts: bool = False, ) -> None: """ diff --git a/neurocaps/analysis/cap/_internals/cluster.py b/neurocaps/analysis/cap/_internals/cluster.py index 4863b38f..84effb8e 100644 --- a/neurocaps/analysis/cap/_internals/cluster.py +++ b/neurocaps/analysis/cap/_internals/cluster.py @@ -1,6 +1,6 @@ """Internal module containing helper functions for ``CAP.get_caps``.""" -from typing import Any, Union +from typing import Any import numpy as np import matplotlib.pyplot as plt @@ -25,7 +25,7 @@ def setup_groups( - subject_timeseries: SubjectTimeseries, groups_dict: Union[dict[str, str], None] + subject_timeseries: SubjectTimeseries, groups_dict: dict[str, str] | None ) -> tuple[dict[str, str], dict[str, str]]: """Used to resolve ``self._groups`` and ``self._subject_table``.""" if groups_dict is None: @@ -93,7 +93,7 @@ def create_group_map(subject_table: dict[str, str], group_dict: dict[str, str]) def concatenate_timeseries( subject_timeseries: SubjectTimeseries, group_dict: dict[str, list[str]], - runs: Union[list[int], list[str], None], + runs: list[int | str] | None, progress_bar: bool, ) -> dict[str, NDArray]: """ @@ -152,8 +152,8 @@ def scale(concatenated_timeseries: dict[str, NDArray]) -> dict[str, NDArray]: def get_runs( - requested_runs: Union[list[int], list[str], None], curr_runs: list[str] -) -> tuple[list[str], Union[list[str], None]]: + requested_runs: list[int | str] | None, curr_runs: list[str] +) -> tuple[list[str], list[str] | None]: """ Filters the current runs available for a subject if specific runs are requested. Also returns a list of missing runs that were requested @@ -170,7 +170,7 @@ def get_runs( def perform_kmeans( n_cluster: int, configs: dict[str, Any], concatenated_timeseries: NDArray, method: str -) -> Union[KMeans, tuple[dict[int, float], dict[int, KMeans]]]: +) -> KMeans | tuple[dict[int, float], dict[int, KMeans]]: """ Uses scikit-learn to perform k-means clustering on concatenated timeseries data in both sequential and parallel contexts. Also uses scikit-learn to provide cluster performance metrics. @@ -203,10 +203,10 @@ def select_optimal_clusters( concatenated_timeseries_dict, method: str, n_clusters: list[int], - n_cores: Union[int, None], + n_cores: int | None, configs: dict[str, Any], show_figs: bool, - output_dir: Union[str, None], + output_dir: str | None, plot_output_format: str, progress_bar: bool, **kwargs, @@ -311,7 +311,7 @@ def plot_cluster_performance( method: str, group_name: str, performance_dict: dict[str, float], - optimal_n_clusters: Union[int, None], + optimal_n_clusters: int | None, show_figs: bool, plot_dict: dict[str, Any], ) -> None: @@ -357,7 +357,7 @@ def plot_cluster_performance( def save_cluster_performance_figure( fig: Figure, - output_dir: Union[str, None], + output_dir: str | None, plot_output_format: bool, group_name: str, method_name: str, diff --git a/neurocaps/analysis/cap/_internals/getter.py b/neurocaps/analysis/cap/_internals/getter.py index d3841592..fbe4fda7 100644 --- a/neurocaps/analysis/cap/_internals/getter.py +++ b/neurocaps/analysis/cap/_internals/getter.py @@ -1,7 +1,6 @@ """A class which is responsible for accessing attributes in ``CAP``.""" import copy, sys -from typing import Union import numpy as np from numpy.typing import NDArray @@ -17,7 +16,7 @@ def __init__(self): ### Attributes exist when CAP initialized @property - def parcel_approach(self) -> Union[ParcelApproach, None]: + def parcel_approach(self) -> ParcelApproach | None: """ Parcellation information with "maps" (path to parcellation file), "nodes" (labels), and "regions" (anatomical regions or networks). This property is also settable (accepts a @@ -26,17 +25,17 @@ def parcel_approach(self) -> Union[ParcelApproach, None]: return self._parcel_approach @parcel_approach.setter - def parcel_approach(self, parcel_dict: Union[ParcelConfig, ParcelApproach, str]) -> None: + def parcel_approach(self, parcel_dict: ParcelConfig | ParcelApproach | str) -> None: self._parcel_approach = check_parcel_approach(parcel_approach=parcel_dict, caller="setter") @property - def groups(self) -> Union[dict[str, list[str]], None]: + def groups(self) -> dict[str, list[str]] | None: """Mapping of groups names to lists of subject IDs. Returns a deep copy.""" return copy.deepcopy(self._groups) ### Attributes exist when CAP.get_caps() used @property - def subject_table(self) -> Union[dict[str, str], None]: + def subject_table(self) -> dict[str, str] | None: """ Lookup table mapping subject IDs to their groups. Derived from ``self.groups`` each time ``self.get_caps()`` is ran. While this property can be modified using its setter, any @@ -56,7 +55,7 @@ def subject_table(self, subject_dict: dict[str, str]) -> None: ) @property - def n_clusters(self) -> Union[int, list[int], None]: + def n_clusters(self) -> int | list[int] | None: """ An integer or list of integers representing the number of clusters used for k-means. Defined after running ``self.get_caps()``. @@ -64,7 +63,7 @@ def n_clusters(self) -> Union[int, list[int], None]: return getattr(self, "_n_clusters", None) @property - def n_cores(self) -> Union[int, None]: + def n_cores(self) -> int | None: """ Number of cores specified used for multiprocessing with Joblib. Defined after running ``self.get_caps()``. @@ -72,14 +71,14 @@ def n_cores(self) -> Union[int, None]: return getattr(self, "_n_cores", None) @property - def runs(self) -> Union[list[Union[int, str]], None]: + def runs(self) -> list[int | str] | None: """ Run IDs specified in the analysis. Defined after running ``self.get_caps()``. """ return getattr(self, "_runs", None) @property - def standardize(self) -> Union[bool, None]: + def standardize(self) -> bool | None: """ Whether region-of-interests (ROIs)/columns were standardized during analysis. Defined after running ``self.get_caps()``. @@ -87,7 +86,7 @@ def standardize(self) -> Union[bool, None]: return getattr(self, "_standardize", None) @property - def concatenated_timeseries(self) -> Union[dict[str, NDArray[np.floating]], None]: + def concatenated_timeseries(self) -> dict[str, NDArray[np.floating]] | None: """ Group-specific concatenated timeseries data. Can be deleted using ``del self.concatenated_timeseries``. Defined after running ``self.get_caps()``. Returns @@ -108,7 +107,7 @@ def concatenated_timeseries(self) -> None: del self._concatenated_timeseries @property - def means(self) -> Union[dict[str, NDArray[np.floating]], None]: + def means(self) -> dict[str, NDArray[np.floating]] | None: """ Group-specific feature means if standardization was applied. Defined after running ``self.get_caps()``. Returns a deep copy. @@ -120,7 +119,7 @@ def means(self) -> Union[dict[str, NDArray[np.floating]], None]: return copy.deepcopy(getattr(self, "_mean_vec", None)) @property - def stdev(self) -> Union[dict[str, NDArray[np.floating]], None]: + def stdev(self) -> dict[str, NDArray[np.floating]] | None: """ Group-specific feature standard deviations if standardization was applied. Defined after running ``self.get_caps()``. Returns a deep copy. @@ -135,7 +134,7 @@ def stdev(self) -> Union[dict[str, NDArray[np.floating]], None]: return copy.deepcopy(getattr(self, "_stdev_vec", None)) @property - def kmeans(self) -> Union[dict[str, KMeans], None]: + def kmeans(self) -> dict[str, KMeans] | None: """ Group-specific k-means models. Defined after running ``self.get_caps()``. Returns a deep copy. @@ -147,7 +146,7 @@ def kmeans(self) -> Union[dict[str, KMeans], None]: return copy.deepcopy(getattr(self, "_kmeans", None)) @property - def caps(self) -> Union[dict[str, dict[str, NDArray[np.floating]]], None]: + def caps(self) -> dict[str, dict[str, NDArray[np.floating]]] | None: """ Cluster centroids for each group and CAP. Defined after running ``self.get_caps()``. Returns a deep copy. @@ -155,7 +154,7 @@ def caps(self) -> Union[dict[str, dict[str, NDArray[np.floating]]], None]: return copy.deepcopy(getattr(self, "_caps", None)) @property - def cluster_scores(self) -> Union[dict[str, Union[str, dict[str, float]]], None]: + def cluster_scores(self) -> dict[str, str | dict[str, float]] | None: """ Scores for different cluster sizes by group. Defined after running ``self.get_caps()``. @@ -166,7 +165,7 @@ def cluster_scores(self) -> Union[dict[str, Union[str, dict[str, float]]], None] return getattr(self, "_cluster_scores", None) @property - def cluster_selection_method(self) -> Union[str, None]: + def cluster_selection_method(self) -> str | None: """ Method used to identify the optimal number of clusters. Defined after running ``self.get_caps()``. @@ -179,7 +178,7 @@ def cluster_selection_method(self) -> Union[str, None]: return attr @property - def optimal_n_clusters(self) -> Union[dict[str, int], None]: + def optimal_n_clusters(self) -> dict[str, int] | None: """ Optimal number of clusters by group if cluster selection was used. Defined after running ``self.get_caps()``. @@ -191,7 +190,7 @@ def optimal_n_clusters(self) -> Union[dict[str, int], None]: return getattr(self, "_optimal_n_clusters", None) @property - def variance_explained(self) -> Union[dict[str, float], None]: + def variance_explained(self) -> dict[str, float] | None: """ Total variance explained by each group's model. Defined after running ``self.get_caps()``. @@ -205,7 +204,7 @@ def variance_explained(self) -> Union[dict[str, float], None]: @property def region_means( self, - ) -> Union[dict[str, dict[str, Union[list[str], NDArray[np.floating]]]], None]: + ) -> dict[str, dict[str, list[str] | NDArray[np.floating]]] | None: """ Region-averaged values used for visualization. Defined after running ``self.caps2plot()``. @@ -216,7 +215,7 @@ def region_means( return getattr(self, "_region_means", None) @property - def outer_products(self) -> Union[dict[str, dict[str, NDArray[np.floating]]], None]: + def outer_products(self) -> dict[str, dict[str, NDArray[np.floating]]] | None: """ Outer product matrices for visualization. Defined after running ``self.caps2plot()``. @@ -230,7 +229,7 @@ def outer_products(self) -> Union[dict[str, dict[str, NDArray[np.floating]]], No @property def cosine_similarity( self, - ) -> Union[dict[str, dict[str, Union[list[str], NDArray[np.floating]]]], None]: + ) -> dict[str, dict[str, list[str] | NDArray[np.floating]]] | None: """ Cosine similarities between CAPs and the regions specified in ``parcel_approach``. Defined after running ``self.caps2radar()``. diff --git a/neurocaps/analysis/cap/_internals/matrix.py b/neurocaps/analysis/cap/_internals/matrix.py index 83c71174..a0dafb85 100644 --- a/neurocaps/analysis/cap/_internals/matrix.py +++ b/neurocaps/analysis/cap/_internals/matrix.py @@ -4,7 +4,7 @@ """ import collections, re -from typing import Any, Union +from typing import Any import numpy as np import matplotlib.pyplot as plt @@ -89,7 +89,7 @@ def extract_scope_information( parcel_approach: ParcelApproach, add_custom_node_labels: bool, cap_dict: dict[str, dict[str, NDArray]], - region_means_dict=Union[dict[str, dict[str, NDArray]], None], + region_means_dict=dict[str, dict[str, NDArray]] | None, ) -> tuple[dict[str, dict[str, NDArray]], list[str]]: """ Extracting region means of each CAP from ``self._region_means`` if scope is "region" else @@ -153,7 +153,7 @@ def sort_custom_node_names(parcel_approach: ParcelApproach) -> list[str]: def collapse_node_labels( parcel_approach: ParcelApproach, - custom_nodes: Union[list[str], None] = None, + custom_nodes: list[str] | None = None, ) -> tuple[list[str], list[str]]: """ Collapses node labels names (based on unique node names and hemisphere) for plotting @@ -246,10 +246,10 @@ def generate_outer_product_plots( cap_dict: dict[str, dict[str, NDArray]], full_labels: list[str], subplots: bool, - output_dir: Union[str, None], + output_dir: str | None, plot_output_format: str, - suffix_title: Union[str, None], - suffix_filename: Union[str, None], + suffix_title: str | None, + suffix_filename: str | None, show_figs: bool, scope: str, parcel_approach: ParcelApproach, @@ -376,7 +376,7 @@ def initialize_outer_product_subplot( cap_dict: dict[str, dict[str, NDArray]], group_name: str, plot_dict: dict[str, Any], - suffix_title: Union[str, None], + suffix_title: str | None, ) -> tuple[plt.Figure, plt.Axes, tuple[int, int], tuple[int, int]]: """ Initializes the subplot for "outer_product". @@ -420,10 +420,10 @@ def generate_heatmap_plots( plot_dict: dict[str, Any], cap_dict: dict[str, dict[str, NDArray]], full_labels: list[str], - output_dir: Union[str, None], + output_dir: str | None, plot_output_format: str, - suffix_title: Union[str, None], - suffix_filename: Union[str, None], + suffix_title: str | None, + suffix_filename: str | None, show_figs: bool, scope: str, parcel_approach: ParcelApproach, diff --git a/neurocaps/analysis/cap/_internals/metrics.py b/neurocaps/analysis/cap/_internals/metrics.py index 19491eb3..ffcd1666 100644 --- a/neurocaps/analysis/cap/_internals/metrics.py +++ b/neurocaps/analysis/cap/_internals/metrics.py @@ -1,7 +1,6 @@ """Internal module for computing temporal dynamic metrics.""" import itertools, os -from typing import Union import numpy as np import pandas as pd @@ -14,7 +13,7 @@ LG = setup_logger(__name__) -def filter_metrics(metrics: Union[list[str], tuple[str], None]) -> list[str]: +def filter_metrics(metrics: list[str] | tuple[str]) -> list[str]: """ Filters metrics to ensure only the supported metrics ("temporal_fraction", "persistence", "counts", "transition_frequency") are in the list. Maintains the order of the original @@ -99,7 +98,7 @@ def create_columns_names( group_names: list[str], cap_names: list[str], pairs: dict[str, list[tuple[int, int]]], -) -> dict[str, Union[list[str], dict[str, list[str]]]]: +) -> dict[str, list[str] | dict[str, list[str]]]: """ Creates the column names for each requested metric. Used downstream has the column names for each metrics dataframe. @@ -123,7 +122,7 @@ def create_columns_names( def initialize_all_metrics_dict( metrics: list[str], group_names: list[str] -) -> dict[str, Union[list, dict[str, list]]]: +) -> dict[str, list | dict[str, list]]: """ Initializes a dictionary intended to store all computations for a metric across all subjects. The dictionary will be converted to a dataframe downstream. @@ -181,7 +180,7 @@ def compute_counts(arr: NDArray, n_caps: int) -> dict[str, int]: return count_dict -def compute_persistence(arr: NDArray, n_caps: int, tr: Union[float, int, None]) -> dict[str, float]: +def compute_persistence(arr: NDArray, n_caps: int, tr: float | int | None) -> dict[str, float]: """ Computes persistence for the subject and run specified in ``sub_info`` and inserts new row in the dataframe. Assumes one-based values in ``arr``. @@ -227,8 +226,8 @@ def segments( def add_nans_to_dict( - max_cap: int, n_group_caps: int, curr_dict: dict[str, Union[float, int]] -) -> dict[str, Union[float, int]]: + max_cap: int, n_group_caps: int, curr_dict: dict[str, float | int] +) -> dict[str, float | int]: """Adds NaN for groups with less CAPs than the group with the greatest number of CAPs.""" if max_cap > n_group_caps: for i in range(n_group_caps + 1, max_cap + 1): @@ -238,9 +237,9 @@ def add_nans_to_dict( def convert_dict_to_df( - columns_names_dict: dict[str, Union[list[str], dict[str, list[str]]]], - all_metrics_dict: dict[str, Union[list, dict[str, list]]], -) -> dict[str, Union[pd.DataFrame, dict[str, pd.DataFrame]]]: + columns_names_dict: dict[str, list[str] | dict[str, list[str]]], + all_metrics_dict: dict[str, list | dict[str, list]], +) -> dict[str, pd.DataFrame | dict[str, pd.DataFrame]]: """ Appends the data ``all_metrics_dict`` to its respective dataframe in ``df_dict``. """ @@ -306,8 +305,8 @@ def compute_transition_probability( def save_metrics( output_dir: str, group_names: list[str], - df_dict: dict[str, Union[pd.DataFrame, dict[str, pd.DataFrame]]], - prefix_filename: Union[str, None], + df_dict: dict[str, pd.DataFrame | dict[str, pd.DataFrame]], + prefix_filename: str | None, ) -> None: """Saves the metric dataframes as csv files.""" if not output_dir: diff --git a/neurocaps/analysis/cap/_internals/radar.py b/neurocaps/analysis/cap/_internals/radar.py index d81ae793..ed6d7478 100644 --- a/neurocaps/analysis/cap/_internals/radar.py +++ b/neurocaps/analysis/cap/_internals/radar.py @@ -1,7 +1,7 @@ """Internal module containing functions for creating radar plots.""" import os, sys -from typing import Any, Union +from typing import Any import numpy as np import pandas as pd @@ -20,9 +20,9 @@ def update_radar_dict( cap_dict: dict[str, NDArray], - radar_dict: Union[dict[str, str]], + radar_dict: dict[str, str], parcel_approach: ParcelApproach, -) -> dict[str, Union[str, float]]: +) -> dict[str, str | float]: """ Updates a dictionary containing information about the cosine similarity between each region specified in a parcellation and the positive and negative activations in a CAP vector for @@ -92,10 +92,10 @@ def compute_cosine_similarity( def generate_radar_plot( use_scatterpolar: bool, - radar_dict: dict[str, Union[str, float]], + radar_dict: dict[str, str | float], cap_name: str, group_name: str, - suffix_title: Union[str, None], + suffix_title: str | None, plot_dict: dict[str, Any], ) -> go.Figure: """Generates radar plots.""" @@ -204,7 +204,7 @@ def show_radar_plot(fig: go.Figure, show_figure: bool) -> None: def save_radar_plot( fig: go.Figure, scale: int, - output_dir: Union[str, None], + output_dir: str | None, plot_output_format: str, suffix_filename: str, group_name: str, diff --git a/neurocaps/analysis/cap/_internals/spatial.py b/neurocaps/analysis/cap/_internals/spatial.py index 34229b3c..afaa9db7 100644 --- a/neurocaps/analysis/cap/_internals/spatial.py +++ b/neurocaps/analysis/cap/_internals/spatial.py @@ -2,7 +2,6 @@ import inspect from functools import lru_cache -from typing import Union import nibabel as nib, numpy as np from nilearn import datasets, image @@ -13,7 +12,7 @@ def cap_to_img( atlas_file: str, cap_vector: NDArray[np.floating], - knn_dict: dict[str, Union[int, list[int], str]], + knn_dict: dict[str, int | list[int] | str], ) -> nib.nifti1.Nifti1Image: """ Projects cluster centroids (CAPs) on to the parcellation map. Also if specified, performs @@ -57,7 +56,7 @@ def build_tree(atlas: nib.nifti1.Nifti1Image) -> tuple[KDTree, NDArray[np.intp]] def get_remove_indices( - atlas: nib.nifti1.Nifti1Image, remove_labels: Union[list[int], NDArray[np.integer]] + atlas: nib.nifti1.Nifti1Image, remove_labels: list[int] | NDArray[np.integer] ) -> tuple[NDArray[np.intp], ...]: """If requested, gets the coordinates containing a specified label to be removed.""" remove_indxs = np.where(np.isin(atlas.get_fdata(), remove_labels)) @@ -67,7 +66,7 @@ def get_remove_indices( def perform_knn( atlas: nib.nifti1.Nifti1Image, - knn_dict: dict[str, Union[int, list[int], str]], + knn_dict: dict[str, int | list[int] | str], stat_map: nib.nifti1.Nifti1Image, ) -> nib.nifti1.Nifti1Image: """ diff --git a/neurocaps/analysis/cap/_internals/surface.py b/neurocaps/analysis/cap/_internals/surface.py index da6b9914..35dcd5f1 100644 --- a/neurocaps/analysis/cap/_internals/surface.py +++ b/neurocaps/analysis/cap/_internals/surface.py @@ -1,7 +1,7 @@ """Internal module for generating surface plots.""" import os, tempfile -from typing import Any, Union +from typing import Any import nibabel as nib import matplotlib.pyplot as plt @@ -78,7 +78,7 @@ def generate_surface_plot( plot_dict: dict[str, Any], group_name: str, cap_name: str, - suffix_title: Union[str, None], + suffix_title: str | None, ) -> Figure: """Creates the surface plot.""" # Code adapted from example on https://surfplot.readthedocs.io/ @@ -143,11 +143,11 @@ def generate_surface_plot( def save_surface_plot( - fig: Union[Figure, Axes], + fig: Figure | Axes, plot_dict: dict[str, Any], output_dir: str, plot_output_format: str, - suffix_filename: Union[str, None], + suffix_filename: str | None, stat_map: nib.Nifti1Image, save_stat_maps: bool, group_name: str, @@ -174,7 +174,7 @@ def save_nifti_img(img: nib.Nifti1Image, output_dir: str, filename: str) -> None nib.save(img, os.path.join(output_dir, filename)) -def show_surface_plot(fig: Union[Figure, Axes], show_fig: bool) -> None: +def show_surface_plot(fig: Figure | Axes, show_fig: bool) -> None: """Visualizes a single surface plot.""" try: plt.show(fig) if show_fig else plt.close(fig) diff --git a/neurocaps/analysis/cap/cap.py b/neurocaps/analysis/cap/cap.py index 5712956f..6fb76329 100644 --- a/neurocaps/analysis/cap/cap.py +++ b/neurocaps/analysis/cap/cap.py @@ -2,7 +2,7 @@ import itertools from copy import deepcopy -from typing import Any, Callable, Literal, Optional, Union +from typing import Any, Callable, Literal from typing_extensions import Self import numpy as np @@ -231,8 +231,8 @@ class CAP(CAPGetter): def __init__( self, - parcel_approach: Optional[Union[ParcelConfig, ParcelApproach, str]] = None, - groups: Optional[dict[str, list[str]]] = None, + parcel_approach: ParcelConfig | ParcelApproach | str | None = None, + groups: dict[str, list[str]] | None = None, ) -> None: if parcel_approach is not None: parcel_approach = check_parcel_approach(parcel_approach=parcel_approach, caller="CAP") @@ -262,21 +262,21 @@ def __init__( def get_caps( self, - subject_timeseries: Union[SubjectTimeseries, str], - runs: Optional[Union[int, str, list[int], list[str]]] = None, - n_clusters: Union[int, list[int], range] = 5, - cluster_selection_method: Optional[ - Literal["elbow", "davies_bouldin", "silhouette", "variance_ratio"] - ] = None, - random_state: Optional[int] = None, - init: Union[Literal["k-means++", "random"], Callable, ArrayLike] = "k-means++", - n_init: Union[Literal["auto"], int] = "auto", + subject_timeseries: SubjectTimeseries | str, + runs: int | str | list[int | str] | None = None, + n_clusters: int | list[int] | range = 5, + cluster_selection_method: ( + Literal["elbow", "davies_bouldin", "silhouette", "variance_ratio"] | None + ) = None, + random_state: int | None = None, + init: Literal["k-means++", "random"] | Callable | ArrayLike = "k-means++", + n_init: Literal["auto"] | int = "auto", max_iter: int = 300, tol: float = 0.0001, algorithm: Literal["lloyd", "elkan"] = "lloyd", standardize: bool = True, - n_cores: Optional[int] = None, - output_dir: Optional[str] = None, + n_cores: int | None = None, + output_dir: str | None = None, plot_output_format: str = "png", show_figs: bool = False, progress_bar: bool = False, @@ -532,27 +532,27 @@ def clear_groups(self) -> None: @check_required_attributes(required_attrs=["_kmeans"]) def calculate_metrics( self, - subject_timeseries: Union[SubjectTimeseries, str], - runs: Optional[Union[int, str, list[int], list[str]]] = None, + subject_timeseries: SubjectTimeseries | str, + runs: int | str | list[int | str] | None = None, continuous_runs: bool = False, - metrics: Union[ + metrics: ( Literal[ "temporal_fraction", "persistence", "counts", "transition_frequency", "transition_probability", - ], - list[str], - tuple[str, ...], - None, - ] = ("temporal_fraction", "persistence", "counts", "transition_frequency"), - tr: Optional[float] = None, - output_dir: Optional[str] = None, - prefix_filename: Optional[str] = None, + ] + | list[str] + | tuple[str, ...] + | None + ) = ("temporal_fraction", "persistence", "counts", "transition_frequency"), + tr: float | None = None, + output_dir: str | None = None, + prefix_filename: str | None = None, return_df: bool = True, progress_bar: bool = False, - ) -> Union[dict[str, pd.DataFrame], dict[str, dict[str, pd.DataFrame]], None]: + ) -> dict[str, pd.DataFrame] | dict[str, dict[str, pd.DataFrame]]: """ Calculate Participant-wise CAP Metrics. @@ -896,8 +896,8 @@ def calculate_metrics( @check_required_attributes(required_attrs=["_kmeans"]) def return_cap_labels( self, - subject_timeseries: Union[SubjectTimeseries, str], - runs: Optional[Union[int, str, list[int], list[str]]] = None, + subject_timeseries: SubjectTimeseries | str, + runs: int | str | list[int | str] | None = None, continuous_runs: bool = False, shift_labels: bool = False, ) -> dict[str, dict[str, NDArray]]: @@ -1042,17 +1042,15 @@ def return_cap_labels( @check_required_attributes(["_parcel_approach", "_caps"]) def caps2plot( self, - plot_options: Union[ - Literal["heatmap", "outer_product"], list[Literal["heatmap", "outer_product"]] - ] = "heatmap", - visual_scope: Union[ - Literal["regions", "nodes"], list[Literal["regions", "nodes"]] - ] = "regions", + plot_options: ( + Literal["heatmap", "outer_product"] | list[Literal["heatmap", "outer_product"]] + ) = "heatmap", + visual_scope: Literal["regions", "nodes"] | list[Literal["regions", "nodes"]] = "regions", subplots: bool = False, - output_dir: Optional[str] = None, + output_dir: str | None = None, plot_output_format: str = "png", - suffix_filename: Optional[str] = None, - suffix_title: Optional[str] = None, + suffix_filename: str | None = None, + suffix_title: str | None = None, show_figs: bool = True, **kwargs, ) -> Self: @@ -1222,16 +1220,16 @@ def caps2plot( def caps2corr( self, method: str = "pearson", - output_dir: Optional[str] = None, + output_dir: str | None = None, plot_output_format: str = "png", - suffix_filename: Optional[str] = None, - suffix_title: Optional[str] = None, + suffix_filename: str | None = None, + suffix_title: str | None = None, save_plots: bool = True, save_df: bool = False, show_figs: bool = True, return_df: bool = False, **kwargs, - ) -> Union[dict[str, pd.DataFrame], None]: + ) -> dict[str, pd.DataFrame] | None: """ Generate a Correlation Matrix for CAPs. @@ -1350,8 +1348,8 @@ def caps2corr( def caps2niftis( self, output_dir: str, - suffix_filename: Optional[str] = None, - knn_dict: Optional[dict[str, Union[int, list[int], str]]] = None, + suffix_filename: str | None = None, + knn_dict: dict[str, int | list[int] | str] | None = None, progress_bar: bool = False, ) -> Self: """ @@ -1435,8 +1433,8 @@ def caps2niftis( @staticmethod def _validate_knn_dict( - knn_dict: Union[dict[str, Any], None], - ) -> Union[dict[str, Union[int, list[int], str]], None]: + knn_dict: dict[str, Any] | None, + ) -> dict[str, int | list[int] | str] | None: """Validates the ``knn_dict``.""" if not knn_dict: return None @@ -1492,12 +1490,12 @@ def caps2surf( self, fslr_density: Literal["32k", "164k"] = "32k", method: Literal["linear", "nearest"] = "linear", - output_dir: Optional[str] = None, + output_dir: str | None = None, plot_output_format: str = "png", - suffix_filename: Optional[str] = None, - suffix_title: Optional[str] = None, + suffix_filename: str | None = None, + suffix_title: str | None = None, save_stat_maps: bool = False, - knn_dict: Optional[dict[str, Union[int, list[int], str]]] = None, + knn_dict: dict[str, int | list[int] | str] | None = None, show_figs: bool = True, progress_bar: bool = False, **kwargs, @@ -1641,10 +1639,10 @@ def caps2surf( def caps2radar( self, use_scatterpolar: bool = False, - output_dir: Optional[str] = None, + output_dir: str | None = None, plot_output_format: str = "png", - suffix_filename: Optional[str] = None, - suffix_title: Optional[str] = None, + suffix_filename: str | None = None, + suffix_title: str | None = None, show_figs: bool = True, **kwargs, ) -> Self: diff --git a/neurocaps/analysis/change_dtype.py b/neurocaps/analysis/change_dtype.py index 48132465..ada61876 100644 --- a/neurocaps/analysis/change_dtype.py +++ b/neurocaps/analysis/change_dtype.py @@ -1,7 +1,5 @@ """Function for changing the dtype of timeseries data.""" -from typing import Optional, Union - import numpy as np from ._internals import serialize @@ -10,12 +8,12 @@ def change_dtype( - subject_timeseries_list: Union[list[SubjectTimeseries], list[str]], - dtype: Union[str, np.floating], - output_dir: Optional[str] = None, - filenames: Optional[list[str]] = None, + subject_timeseries_list: list[SubjectTimeseries] | list[str], + dtype: str | np.floating, + output_dir: str | None = None, + filenames: list[str] | None = None, return_dicts: bool = True, -) -> Union[dict[str, SubjectTimeseries], None]: +) -> dict[str, SubjectTimeseries] | None: """ Perform Participant-wise Dtype Conversion. diff --git a/neurocaps/analysis/merge.py b/neurocaps/analysis/merge.py index 82898673..a78ca25c 100644 --- a/neurocaps/analysis/merge.py +++ b/neurocaps/analysis/merge.py @@ -1,7 +1,6 @@ """Function for merging timeseries data across dictionaries.""" from copy import deepcopy -from typing import Optional, Union import numpy as np @@ -11,13 +10,13 @@ def merge_dicts( - subject_timeseries_list: Union[list[SubjectTimeseries], list[str]], - output_dir: Optional[Union[str, str]] = None, - filenames: Optional[list[str]] = None, + subject_timeseries_list: list[SubjectTimeseries] | list[str], + output_dir: str | None = None, + filenames: list[str] | None = None, save_reduced_dicts: bool = False, return_merged_dict: bool = True, return_reduced_dicts: bool = False, -) -> Union[dict[str, SubjectTimeseries], None]: +) -> dict[str, SubjectTimeseries] | None: """ Merge Participant Timeseries Across Multiple Sessions or Tasks. diff --git a/neurocaps/analysis/standardize.py b/neurocaps/analysis/standardize.py index 09b08a16..6afbf4c3 100644 --- a/neurocaps/analysis/standardize.py +++ b/neurocaps/analysis/standardize.py @@ -1,7 +1,5 @@ """Function to standardize timeseries within subject runs.""" -from typing import Optional, Union - from ._internals import serialize from neurocaps.utils import _io as io_utils from neurocaps.typing import SubjectTimeseries @@ -9,11 +7,11 @@ def standardize( - subject_timeseries_list: Union[list[SubjectTimeseries], list[str]], - output_dir: Optional[str] = None, - filenames: Optional[list[str]] = None, + subject_timeseries_list: list[SubjectTimeseries] | list[str], + output_dir: str | None = None, + filenames: list[str] | None = None, return_dicts: bool = True, -) -> Union[dict[str, SubjectTimeseries], None]: +) -> dict[str, SubjectTimeseries] | None: """ Perform Participant-wise Timeseries Standardization Within Runs. diff --git a/neurocaps/analysis/transition_matrix.py b/neurocaps/analysis/transition_matrix.py index 4f62e201..eced7e43 100644 --- a/neurocaps/analysis/transition_matrix.py +++ b/neurocaps/analysis/transition_matrix.py @@ -1,7 +1,5 @@ """Function for averaging subject-level transition probabilities and producing visualizations.""" -from typing import Optional, Union - import pandas as pd from neurocaps.utils import PlotDefaults @@ -12,16 +10,16 @@ def transition_matrix( trans_dict: dict[str, pd.DataFrame], - output_dir: Optional[str] = None, + output_dir: str | None = None, plot_output_format: str = "png", - suffix_filename: Optional[str] = None, - suffix_title: Optional[str] = None, + suffix_filename: str | None = None, + suffix_title: str | None = None, save_plots: bool = True, save_df: bool = True, show_figs: bool = True, return_df: bool = True, **kwargs, -) -> Union[pd.DataFrame, None]: +) -> pd.DataFrame | None: """ Generate and Visualize the Averaged Transition Probabilities. diff --git a/neurocaps/extraction/_internals/bids_query.py b/neurocaps/extraction/_internals/bids_query.py index d4ac2488..96b25e68 100644 --- a/neurocaps/extraction/_internals/bids_query.py +++ b/neurocaps/extraction/_internals/bids_query.py @@ -2,7 +2,7 @@ import json, os, re -from typing import Any, Optional, Union +from typing import Any from neurocaps.utils._logging import setup_logger @@ -13,7 +13,7 @@ def setup_extraction( layout: Any, subj_ids: list[str], space: str, - exclude_niftis: Union[list[str], None], + exclude_niftis: list[str] | None, signal_clean_info: dict[str, Any], task_info: dict[str, Any], verbose: bool, @@ -110,10 +110,10 @@ def query_files( extension: str, subj_id: str, scope: str = "derivatives", - suffix: Union[str, None] = None, - desc: Union[str, None] = None, + suffix: str | None = None, + desc: str | None = None, event: bool = False, - space: Optional[str] = None, + space: str | None = None, ): """ Queries specific files (sorted lexicographically) using ``BidsLayout``. @@ -147,11 +147,11 @@ def query_files( def build_dict( - base: dict[str, Union[Any, None]], + base: dict[str, Any | None], space: str, signal_clean_info: dict[str, Any], task_info: dict[str, Any], -) -> dict[str, Union[str, None]]: +) -> dict[str, str | None]: """Builds dictionary containing subject-specific files queried using ``BIDSLayout``.""" files = {} files["niftis"] = query_files( @@ -183,7 +183,7 @@ def build_dict( return files -def exclude_nifti_files(niftis: list[str], exclude_niftis: Union[str, list[str]]) -> list[str]: +def exclude_nifti_files(niftis: list[str], exclude_niftis: str | list[str]) -> list[str]: """Excludes certain NIfTI files based on ``exclude_niftis``.""" exclude_niftis = exclude_niftis if isinstance(exclude_niftis, list) else [exclude_niftis] @@ -203,10 +203,10 @@ def create_header(subj_id: str, task_info: dict[str, Any]) -> str: def check_files( - files: dict[str, Union[list[str], None]], + files: dict[str, list[str] | None], signal_clean_info: dict[str, Any], task_info: dict[str, Any], -) -> tuple[Union[bool, None], Union[str, None]]: +) -> tuple[bool, str | None]: """ Simple initial check to ensure the required files are needed based on certain parameters ``__init__``. @@ -325,7 +325,7 @@ def get_tr( signal_clean_info: dict[str, Any], task_info: dict[str, Any], verbose: bool, -) -> Union[float, int, None]: +) -> float | int | None: """Gets repetition time.""" try: if task_info["tr"]: diff --git a/neurocaps/extraction/_internals/confounds.py b/neurocaps/extraction/_internals/confounds.py index fa90d1a0..fbc750bb 100644 --- a/neurocaps/extraction/_internals/confounds.py +++ b/neurocaps/extraction/_internals/confounds.py @@ -10,7 +10,8 @@ def check_confound_names(high_pass, user_confounds, n_acompcor_separate): """ Pipeline for checking confound names. The default confound names ("basic") depend on whether - high-pass filtering is specified. + high-pass filtering is specified. In addition, ``high_pass`` set to ``None`` if cosine parameters + are in the confound names. """ if user_confounds == "basic": if high_pass: @@ -58,6 +59,8 @@ def check_confound_names(high_pass, user_confounds, n_acompcor_separate): ), "`confound_names` must be a non-empty list." confound_names = user_confounds + high_pass = _check_cosine_high_pass_conflict(high_pass, confound_names) + confound_names = check_wildcard_confounds(confound_names) if n_acompcor_separate: @@ -67,7 +70,25 @@ def check_confound_names(high_pass, user_confounds, n_acompcor_separate): LG.info(f"Confound regressors to be used if available: {', '.join(confound_names)}.") - return deepcopy(confound_names) + return deepcopy(confound_names), high_pass + + +def _check_cosine_high_pass_conflict(high_pass, confound_names): + """ + Checks if both ``high_pass`` and cosine regressors are specified. If so, ``high_pass`` is set + to None to avoid multicollinearity between nilearn's DCT basis functions and fMRIPrep's cosine + regressors. + """ + if high_pass is not None and any(confound.startswith("cosine") for confound in confound_names): + LG.warning( + "Both `high_pass` and cosine regressors were specified in `confound_names`. Using both " + "introduces near-identical DCT regressors, resulting in multicollinearity issues. " + "`high_pass` has been set to None; cosine regressors from fMRIPrep will be used for " + "high-pass filtering instead." + ) + high_pass = None + + return high_pass def remove_a_comp_cor(confound_names, user_confounds, n): diff --git a/neurocaps/extraction/_internals/getter.py b/neurocaps/extraction/_internals/getter.py index 745cf7ac..be3ad59b 100644 --- a/neurocaps/extraction/_internals/getter.py +++ b/neurocaps/extraction/_internals/getter.py @@ -1,7 +1,6 @@ """A class which is responsible for accessing attributes in ``TimeSeriesExtractor``.""" import copy, sys -from typing import Union import numpy as np @@ -38,11 +37,11 @@ def parcel_approach(self) -> ParcelApproach: return copy.deepcopy(self._parcel_approach) @parcel_approach.setter - def parcel_approach(self, parcel_dict: Union[ParcelConfig, ParcelApproach, str]) -> None: + def parcel_approach(self, parcel_dict: ParcelConfig | ParcelApproach | str) -> None: self._parcel_approach = check_parcel_approach(parcel_approach=parcel_dict, caller="setter") @property - def signal_clean_info(self) -> Union[dict[str, Union[bool, int, float, str]], None]: + def signal_clean_info(self) -> dict[str, bool | int | float | str] | None: """ Dictionary containing signal cleaning parameters. Returns a deep copy. """ @@ -52,7 +51,7 @@ def signal_clean_info(self) -> Union[dict[str, Union[bool, int, float, str]], No # Exist when TimeSeriesExtractor.get_bold() used @property - def task_info(self) -> Union[dict[str, Union[str, int]], None]: + def task_info(self) -> dict[str, str | int] | None: """ Dictionary containing all task-related information such. Defined after running ``self.get_bold()``. @@ -61,7 +60,7 @@ def task_info(self) -> Union[dict[str, Union[str, int]], None]: # Gets initialized and populated in TimeSeriesExtractor.get_bold(), @property - def subject_ids(self) -> Union[list[str], None]: + def subject_ids(self) -> list[str] | None: """ A list containing all subject IDs retrieved from ``BIDSLayout`` for timeseries extraction. Defined after running ``self.get_bold()``. @@ -69,7 +68,7 @@ def subject_ids(self) -> Union[list[str], None]: return getattr(self, "_subject_ids", None) @property - def n_cores(self) -> Union[int, None]: + def n_cores(self) -> int | None: """ Number of cores used for multiprocessing with Joblib. Defined after running ``self.get_bold()``. @@ -77,7 +76,7 @@ def n_cores(self) -> Union[int, None]: return getattr(self, "_n_cores", None) @property - def subject_timeseries(self) -> Union[SubjectTimeseries, None]: + def subject_timeseries(self) -> SubjectTimeseries | None: """ A dictionary mapping subject IDs to their run IDs and their associated timeseries (TRs x ROIs) as a NumPy array. Can be deleted using ``del self.subject_timeseries``. @@ -87,7 +86,7 @@ def subject_timeseries(self) -> Union[SubjectTimeseries, None]: return getattr(self, "_subject_timeseries", None) @subject_timeseries.setter - def subject_timeseries(self, subject_dict: Union[SubjectTimeseries, str]) -> None: + def subject_timeseries(self, subject_dict: SubjectTimeseries | str) -> None: subject_dict = io_utils.get_obj(subject_dict) self._validate_timeseries(subject_dict) @@ -99,7 +98,7 @@ def subject_timeseries(self) -> None: del self._subject_timeseries @property - def qc(self) -> Union[dict[str, dict[str, dict[str, Union[float, int]]]], None]: + def qc(self) -> dict[str, dict[str, dict[str, float | int]]] | None: """ A dictionary reporting quality control, which maps subject IDs to their run IDs and information related to framewise displacement and dummy scans. Returns a reference. diff --git a/neurocaps/extraction/_internals/postprocess.py b/neurocaps/extraction/_internals/postprocess.py index 0524043d..fc7b2795 100644 --- a/neurocaps/extraction/_internals/postprocess.py +++ b/neurocaps/extraction/_internals/postprocess.py @@ -4,7 +4,6 @@ from dataclasses import dataclass, field from functools import cached_property from packaging.version import parse -from typing import Optional, Union import nilearn, numpy as np, pandas as pd from nilearn.maskers import NiftiLabelsMasker @@ -30,31 +29,31 @@ class RunData: parcel_approach: ParcelApproach = field(default_factory=dict) signal_clean_info: dict = field(default_factory=dict) task_info: dict = field(default_factory=dict) - tr: Union[int, None] = None + tr: int | None = None verbose: bool = False # Run-specific attributes - files: dict[str, Optional[str]] = field(default_factory=dict) - head: Union[str, None] = None - dummy_vols: Union[int, None] = None + files: dict[str, str | None] = field(default_factory=dict) + head: str | None = None + dummy_vols: int | None = None censored_frames: list[int] = field(default_factory=list) - sample_mask: Union[np.typing.NDArray[np.bool_], None] = None + sample_mask: np.typing.NDArray[np.bool_] | None = None censored_ends: list[int] = field(default_factory=list) - fd_array_len: Union[int, None] = None + fd_array_len: int | None = None # Event condition scan indices and qc information for condition scans: list[int] = field(default_factory=list) - n_total_scans: Union[int, None] = None - n_censored_scans: Union[int, None] = None - n_interpolated_scans: Union[int, None] = None + n_total_scans: int | None = None + n_censored_scans: int | None = None + n_interpolated_scans: int | None = None # Avoid timeseries extraction skip_run: bool = False # Stats for qc - mean_fd: Union[float, None] = None - std_fd: Union[float, None] = None - high_motion_len_mean: Union[float, None] = None - high_motion_len_std: Union[float, None] = None + mean_fd: float | None = None + std_fd: float | None = None + high_motion_len_mean: float | None = None + high_motion_len_std: float | None = None @property - def session(self) -> Union[int, str, None]: + def session(self) -> int | str | None: return self.task_info["session"] @property @@ -62,7 +61,7 @@ def task(self) -> str: return self.task_info["task"] @property - def condition(self) -> Union[str, None]: + def condition(self) -> str | None: return self.task_info["condition"] @property @@ -78,22 +77,22 @@ def use_confounds(self) -> bool: return self.signal_clean_info["use_confounds"] @property - def confound_names(self) -> Union[list[str], None]: + def confound_names(self) -> list[str] | None: return self.signal_clean_info["confound_names"] @property - def n_acompcor_separate(self) -> Union[int, None]: + def n_acompcor_separate(self) -> int | None: return self.signal_clean_info["n_acompcor_separate"] @property - def fd_thresh(self) -> Union[float, None]: + def fd_thresh(self) -> float | None: if isinstance(self.signal_clean_info["fd_threshold"], dict): return self.signal_clean_info["fd_threshold"]["threshold"] else: return self.signal_clean_info["fd_threshold"] @property - def out_percent(self) -> Union[float, None]: + def out_percent(self) -> float | None: if isinstance(self.signal_clean_info["fd_threshold"], dict): return self.signal_clean_info["fd_threshold"].get("outlier_percentage") @@ -106,17 +105,17 @@ def censored_vals(self) -> tuple[int, int]: ) @property - def n_before(self) -> Union[int, None]: + def n_before(self) -> int | None: if isinstance(self.signal_clean_info["fd_threshold"], dict): return self.signal_clean_info["fd_threshold"].get("n_before") @property - def n_after(self) -> Union[int, None]: + def n_after(self) -> int | None: if isinstance(self.signal_clean_info["fd_threshold"], dict): return self.signal_clean_info["fd_threshold"].get("n_after") @property - def pass_mask_to_nilearn(self) -> Union[bool, None]: + def pass_mask_to_nilearn(self) -> bool | None: if isinstance(self.signal_clean_info["fd_threshold"], dict): return ( True @@ -127,12 +126,12 @@ def pass_mask_to_nilearn(self) -> Union[bool, None]: return False @property - def interpolate(self) -> Union[bool, None]: + def interpolate(self) -> bool | None: if isinstance(self.signal_clean_info["fd_threshold"], dict): return self.signal_clean_info["fd_threshold"].get("interpolate") @cached_property - def confound_df(self) -> Union[pd.DataFrame, None]: + def confound_df(self) -> pd.DataFrame | None: if self.files["confound"]: return pd.read_csv(self.files["confound"], sep="\t") else: diff --git a/neurocaps/extraction/_internals/vizualization.py b/neurocaps/extraction/_internals/vizualization.py index c3404b7d..ddf2e0ae 100644 --- a/neurocaps/extraction/_internals/vizualization.py +++ b/neurocaps/extraction/_internals/vizualization.py @@ -1,7 +1,7 @@ """Module containing helper functions related to visualizing BOLD data.""" import os -from typing import Any, Union +from typing import Any import numpy as np import matplotlib.pyplot as plt @@ -20,7 +20,7 @@ def get_roi_indices( - parcel_approach: ParcelApproach, roi_indx: Union[int, str, list[str], list[int]] + parcel_approach: ParcelApproach, roi_indx: int | str | list[str] | list[int] ) -> NDArray: """Gets the indices for a specified node or nodes from ``parcel_approach``.""" parc_name = get_parc_name(parcel_approach) @@ -72,7 +72,7 @@ def get_region_indices(parcel_approach: ParcelApproach, region: str) -> NDArray: def get_plot_indxs( parcel_approach: ParcelApproach, - roi_indx: Union[int, str, list[str], list[int]] = None, + roi_indx: int | str | list[str] | list[int] = None, region: str = None, ): """Retrieve the indices from the subject's timeseries data to plot.""" @@ -89,7 +89,7 @@ def create_bold_figure( parcel_approach: ParcelApproach, figsize: tuple[int, int], plot_indxs: NDArray, - roi_indx: Union[int, str, list[str], list[int]] = None, + roi_indx: int | str | list[str] | list[int] = None, region: str = None, ): """Generate the BOLD figure.""" @@ -117,7 +117,7 @@ def create_bold_figure( def save_bold_figure( - fig: Union[Figure, Axes], + fig: Figure | Axes, plot_dict: dict[str, Any], subj_id: str, run_name: str, diff --git a/neurocaps/extraction/timeseries_extractor.py b/neurocaps/extraction/timeseries_extractor.py index dcaa2ed0..fa00d098 100644 --- a/neurocaps/extraction/timeseries_extractor.py +++ b/neurocaps/extraction/timeseries_extractor.py @@ -4,7 +4,7 @@ from copy import deepcopy from functools import lru_cache from multiprocessing.queues import Queue -from typing import Any, Literal, Optional, Union +from typing import Any, Literal from typing_extensions import Self from joblib import Parallel, delayed @@ -71,6 +71,11 @@ class TimeseriesExtractor(TimeseriesExtractorGetter): high_pass : :obj:`float`, :obj:`int`, or :obj:`None`, default=None Filters out signals below the specified cutoff frequency. + .. versionchanged:: 0.37.4 + When cosine regressors (e.g., "cosine*") are detected in user-specified + ``confound_names``, ``high_pass`` is automatically set to None to avoid + multicollinearity between nilearn's DCT basis functions and fMRIPrep's cosine + regressors. fwhm : :obj:`float`, :obj:`int`, or :obj:`None`, default=None Applies spatial smoothing to data (in millimeters). @@ -266,18 +271,18 @@ class TimeseriesExtractor(TimeseriesExtractorGetter): def __init__( self, space: str = "MNI152NLin2009cAsym", - parcel_approach: Union[ParcelConfig, ParcelApproach, str, None] = None, + parcel_approach: ParcelConfig | ParcelApproach | str | None = None, standardize: bool = True, detrend: bool = False, - low_pass: Optional[Union[float, int]] = None, - high_pass: Optional[Union[float, int]] = None, - fwhm: Optional[Union[float, int]] = None, + low_pass: float | int | None = None, + high_pass: float | int | None = None, + fwhm: float | int | None = None, use_confounds: bool = True, - confound_names: Optional[Union[list[str], Literal["basic"]]] = "basic", - fd_threshold: Optional[Union[float, dict[str, Union[bool, float, int]]]] = None, - n_acompcor_separate: Optional[int] = None, - dummy_scans: Optional[Union[int, dict[str, Union[bool, int]], Literal["auto"]]] = None, - dtype: Optional[Union[str, Literal["auto"]]] = None, + confound_names: list[str] | Literal["basic"] = "basic", + fd_threshold: float | dict[str, bool | float | int] | None = None, + n_acompcor_separate: int | None = None, + dummy_scans: int | dict[str, bool | int] | Literal["auto"] | None = None, + dtype: str | Literal["auto"] | None = None, ) -> None: self._space = space @@ -286,8 +291,7 @@ def __init__( if use_confounds: if confound_names: - # Replace confounds if not None - confound_names = check_confound_names( + confound_names, high_pass = check_confound_names( high_pass, confound_names, n_acompcor_separate ) @@ -414,18 +418,18 @@ def get_bold( self, bids_dir: str, task: str, - session: Optional[Union[int, str]] = None, - runs: Optional[Union[int, str, list[int], list[str]]] = None, - condition: Optional[str] = None, + session: int | str | None = None, + runs: int | str | list[int | str] = None, + condition: str | None = None, condition_tr_shift: int = 0, - tr: Optional[Union[int, float]] = None, - slice_time_ref: Union[int, float] = 0.0, - run_subjects: Optional[Union[str, list[str]]] = None, - exclude_subjects: Optional[Union[str, list[str]]] = None, - exclude_niftis: Optional[Union[str, list[str]]] = None, - pipeline_name: Optional[str] = None, - n_cores: Optional[int] = None, - parallel_log_config: Optional[dict[str, Union[Queue, int]]] = None, + tr: int | float | None = None, + slice_time_ref: int | float = 0.0, + run_subjects: str | list[str] | None = None, + exclude_subjects: str | list[str] | None = None, + exclude_niftis: str | list[str] | None = None, + pipeline_name: str | None = None, + n_cores: int | None = None, + parallel_log_config: dict[str, Queue | int] | None = None, verbose: bool = True, progress_bar: bool = False, ) -> Self: @@ -775,7 +779,7 @@ def _validate_get_bold_params(self) -> None: @staticmethod @lru_cache(maxsize=4) - def _call_layout(bids_dir: str, pipeline_name: Union[str, None]) -> Any: + def _call_layout(bids_dir: str, pipeline_name: str | None) -> Any: """ Returns ``BIDSLayout``. @@ -817,8 +821,8 @@ def _call_layout(bids_dir: str, pipeline_name: Union[str, None]) -> Any: def _expand_dicts( self, - subject_timeseries: Union[SubjectTimeseries, None], - qc: Union[dict[str, dict[str, dict[str, Union[float, int]]]], None], + subject_timeseries: SubjectTimeseries | None, + qc: dict[str, dict[str, dict[str, float | int]]] | None, ) -> None: """ Aggregates individual subject timeseries and qc dictionaries into ``self._subject_timeseries`` @@ -830,7 +834,7 @@ def _expand_dicts( self._qc.update(qc) @check_required_attributes(required_attrs=["_subject_timeseries"]) - def timeseries_to_pickle(self, output_dir: str, filename: Optional[str] = None) -> Self: + def timeseries_to_pickle(self, output_dir: str, filename: str | None = None) -> Self: """ Save the Extracted Subject Timeseries. @@ -860,10 +864,10 @@ def timeseries_to_pickle(self, output_dir: str, filename: Optional[str] = None) @check_required_attributes(required_attrs=["_qc"]) def report_qc( self, - output_dir: Optional[str] = None, - filename: Optional[str] = None, + output_dir: str | None = None, + filename: str | None = None, return_df: bool = True, - ) -> Union[DataFrame, None]: + ) -> DataFrame | None: """ Report Quality Control Information. @@ -963,8 +967,8 @@ def report_qc( return df def _create_output_filename( - self, output_dir: Union[str, None], filename: Union[str, None], caller: str - ) -> Union[str, None]: + self, output_dir: str | None, filename: str | None, caller: str + ) -> str | None: """Creates the output filename for ``report_qc`` and ``timeseries_to_pickle``.""" if not output_dir: return None @@ -985,14 +989,14 @@ def _create_output_filename( @check_required_attributes(required_attrs=["_subject_timeseries"]) def visualize_bold( self, - subj_id: Union[int, str], - run: Optional[Union[int, str]] = None, - roi_indx: Optional[Union[int, str, list[str], list[int]]] = None, - region: Optional[str] = None, + subj_id: int | str, + run: int | str | None = None, + roi_indx: int | str | list[str] | list[int] | None = None, + region: str | None = None, show_figs: bool = True, - output_dir: Optional[str] = None, + output_dir: str | None = None, plot_output_format: str = "png", - filename: Optional[str] = None, + filename: str | None = None, **kwargs, ) -> Self: """ diff --git a/neurocaps/typing.py b/neurocaps/typing.py index e4f66de6..b31960e3 100644 --- a/neurocaps/typing.py +++ b/neurocaps/typing.py @@ -1,6 +1,6 @@ """Module containing custom types.""" -from typing import Any, Literal, TypedDict, Union +from typing import Any, Literal, TypedDict from typing_extensions import Required, NotRequired from numpy import floating @@ -108,9 +108,9 @@ class AALParcelConfig(TypedDict): version: NotRequired[str] -ParcelConfig = Union[ - dict[Literal["Schaefer"], SchaeferParcelConfig], dict[Literal["AAL"], AALParcelConfig] -] +ParcelConfig = ( + dict[Literal["Schaefer"], SchaeferParcelConfig] | dict[Literal["AAL"], AALParcelConfig] +) """ Type Definition for the Parcellation Configurations. @@ -277,8 +277,8 @@ class CustomRegionHemispheres(TypedDict): `_ """ - lh: Required[Union[list[int], range]] - rh: Required[Union[list[int], range]] + lh: Required[list[int] | range] + rh: Required[list[int] | range] class CustomParcelApproach(ParcelApproachBase): @@ -367,16 +367,14 @@ class CustomParcelApproach(ParcelApproachBase): `_ """ - regions: NotRequired[ - Union[dict[str, Union[list[int], range]], dict[str, CustomRegionHemispheres]] - ] + regions: NotRequired[dict[str, list[int] | range] | dict[str, CustomRegionHemispheres]] -ParcelApproach = Union[ - dict[Literal["Schaefer"], SchaeferParcelApproach], - dict[Literal["AAL"], AALParcelApproach], - dict[Literal["Custom"], CustomParcelApproach], -] +ParcelApproach = ( + dict[Literal["Schaefer"], SchaeferParcelApproach] + | dict[Literal["AAL"], AALParcelApproach] + | dict[Literal["Custom"], CustomParcelApproach] +) """ Type Definition for the Parcellation Approaches. diff --git a/neurocaps/utils/_io.py b/neurocaps/utils/_io.py index 34bf23c1..b713bc59 100644 --- a/neurocaps/utils/_io.py +++ b/neurocaps/utils/_io.py @@ -4,7 +4,7 @@ """ import os, copy, json, pickle -from typing import Any, Optional, Union +from typing import Any import joblib @@ -32,7 +32,7 @@ def issue_file_warning(param_name: str, param: str, output_dir: str) -> None: ) -def filename(basename: str, add_name: Union[str, None], pos: str) -> str: +def filename(basename: str, add_name: str | None, pos: str) -> str: """ Adds the the prefix or suffix to the file name depending on if ``pos`` is "prefix" or "suffix". """ @@ -103,7 +103,7 @@ def open_pickle_file(filename: str) -> dict: def check_file_exist( filename: str, raise_errors: bool = True, return_flag: bool = False -) -> Union[bool, None]: +) -> bool | None: """Check if a file exists.""" # Dont end with periods if not isinstance(filename, str) and raise_errors: @@ -116,9 +116,7 @@ def check_file_exist( return file_exist if return_flag else None -def check_ext( - filename: str, supported_ext: list[str], return_ext: bool = False -) -> Union[str, None]: +def check_ext(filename: str, supported_ext: list[str], return_ext: bool = False) -> str | None: """Checks if a file has a valid extension.""" ext = f".{filename.partition('.')[-1]}" @@ -132,16 +130,14 @@ def check_ext( return ext if return_ext else None -def validate_file( - filename: str, supported_ext: list[str], return_ext: bool = False -) -> Union[str, None]: +def validate_file(filename: str, supported_ext: list[str], return_ext: bool = False) -> str | None: """Validates file and returns extension if True.""" check_file_exist(filename) return check_ext(filename, supported_ext, return_ext) -def validate_plot_output_format(output_dir: Optional[str], output_format: str, caller=str) -> str: +def validate_plot_output_format(output_dir: str | None, output_format: str, caller: str) -> str: """Validates the supported output format for files.""" if not output_dir: return output_format diff --git a/neurocaps/utils/_logging.py b/neurocaps/utils/_logging.py index 370c4fa6..127009e7 100644 --- a/neurocaps/utils/_logging.py +++ b/neurocaps/utils/_logging.py @@ -7,7 +7,6 @@ import logging, sys from logging.handlers import QueueHandler from multiprocessing.queues import Queue -from typing import Union # Global variables to determine if a handler is user defined or defined by OS _PARALLEL_MODULE = "neurocaps.extraction._internals.postprocess" @@ -27,7 +26,7 @@ def _suppress_third_party_loggers(): def setup_logger( name: str, top_level: bool = True, - parallel_log_config: Union[dict[str, Union[Queue, int]], None] = None, + parallel_log_config: dict[str, Queue | int] | None = None, ): """ Generates module specific loggers, defaults to outputting logs at the informational level to @@ -82,7 +81,7 @@ def setup_logger( return logger -def setup_queuehandler(logger: logging.Logger, parallel_log_config: dict[str, Union[Queue, int]]): +def setup_queuehandler(logger: logging.Logger, parallel_log_config: dict[str, Queue | int]): # Only QueueHandler will be in handler list logger.handlers.clear() queue = parallel_log_config.get("queue") @@ -98,7 +97,7 @@ def setup_queuehandler(logger: logging.Logger, parallel_log_config: dict[str, Un return logger -def add_default_handler(logger: logging.Logger, format: Union[str, None] = None): +def add_default_handler(logger: logging.Logger, format: str | None = None): """Add a default handler and a format.""" # Safeguard; ensure a clean state for "extract_timeseries" since it is used in parallel and # sequential contexts diff --git a/neurocaps/utils/_plot_utils.py b/neurocaps/utils/_plot_utils.py index b3a4e3d2..b490401f 100644 --- a/neurocaps/utils/_plot_utils.py +++ b/neurocaps/utils/_plot_utils.py @@ -1,7 +1,7 @@ """Classes to centralize plotting utility functions.""" import inspect, os -from typing import Any, Union +from typing import Any import matplotlib.pyplot as plt, seaborn from matplotlib.axes import Axes @@ -38,11 +38,11 @@ def base_kwargs(plot_dict: dict, line: bool = True, edge: bool = True) -> dict[s @staticmethod def border( - display: Union[Axes, Figure], + display: Axes | Figure, plot_dict: dict[str, Any], axhline: int, - axvline: Union[int, None] = None, - ) -> Union[Axes, Figure]: + axvline: int | None = None, + ) -> Axes | Figure: if not plot_dict["borderwidths"]: return display @@ -65,11 +65,11 @@ def border( @staticmethod def label_size( - display: Union[Axes, Figure], + display: Axes | Figure, plot_dict: dict[str, Any], set_x: bool = True, set_y: bool = True, - ) -> Union[Axes, Figure]: + ) -> Axes | Figure: if set_x: display.set_xticklabels( display.get_xticklabels(), @@ -91,7 +91,7 @@ def label_size( return display @staticmethod - def set_ticks(display: Union[Axes, Figure], labels: list[str]) -> Union[Axes, Figure]: + def set_ticks(display: Axes | Figure, labels: list[str]) -> Axes | Figure: ticks = [i for i, label in enumerate(labels) if label] display.set_xticks(ticks) @@ -103,12 +103,12 @@ def set_ticks(display: Union[Axes, Figure], labels: list[str]) -> Union[Axes, Fi @staticmethod def set_title( - display: Union[Axes, Figure], + display: Axes | Figure, title: str, - suffix: Union[str, None], + suffix: str | None, plot_dict: dict[str, Any], is_subplot: bool = False, - ) -> Union[Axes, Figure]: + ) -> Axes | Figure: title = f"{title} {suffix}" if suffix else title if is_subplot: @@ -124,7 +124,7 @@ def set_title( @staticmethod def save_fig( - fig: Union[Axes, Figure], + fig: Axes | Figure, plot_dict: dict[str, Any], output_dir: str, plot_output_format: str, @@ -154,7 +154,7 @@ class MatrixVisualizer: @staticmethod def create_display( df: DataFrame, plot_dict: dict[str, Any], suffix_title: str, group_name: str, caller: str - ) -> Union[Axes, Figure]: + ) -> Axes | Figure: # Refresh grid for each iteration plt.figure(figsize=plot_dict["figsize"]) @@ -182,7 +182,7 @@ def create_display( @staticmethod def save_contents( - display: Union[Axes, Figure], + display: Axes | Figure, plot_dict: dict[str, Any], output_dir: str, plot_output_format: str, diff --git a/neurocaps/utils/datasets/_fetch.py b/neurocaps/utils/datasets/_fetch.py index 331b0517..52de954b 100644 --- a/neurocaps/utils/datasets/_fetch.py +++ b/neurocaps/utils/datasets/_fetch.py @@ -1,7 +1,7 @@ """Module containing functions related to fetching the preset parcellation approaches.""" import os -from typing import Literal, Union +from typing import Literal import numpy as np @@ -38,10 +38,13 @@ def is_valid_preset(name: str) -> None: def get_data_dir() -> str: """ - Gets the full path for 'neurocaps_data' in the users home directory. If it does not - exist, then the directory and the subfolders are created. + Gets the full path for 'neurocaps_data' in the users home directory or path set by + environmental variable. If it does not exist, then the directory and the subfolders + are created. """ - data_dir = os.path.expanduser(os.path.join("~", "neurocaps_data")) + if not (data_dir := os.environ.get("NEUROCAPS_DATA")): + data_dir = os.path.expanduser(os.path.join("~", "neurocaps_data")) + # Create manually instead of using `fetch_files` to notify user that a directory was created io_utils.makedir(data_dir) @@ -53,7 +56,7 @@ def get_preset_subdir_paths(root_dir) -> tuple[str, str]: return os.path.join(root_dir, "presets", "jsons"), os.path.join(root_dir, "presets", "niftis") -def check_n_nodes(name: str, n_nodes: Union[int, None]) -> Union[int, None]: +def check_n_nodes(name: str, n_nodes: int | None) -> int | None: """Checks if the number of nodes for the parcellation is valid.""" has_node_info = name in ATLAS_N_NODES if not has_node_info: @@ -100,7 +103,7 @@ def get_osf_file_url(filename: str, download_mock: bool = True) -> str: def fetch_custom_parcel_approach( - name: str, n_nodes: Union[int, None], overwrite: bool = False + name: str, n_nodes: int | None, overwrite: bool = False ) -> dict[Literal["Custom"], CustomParcelApproach]: """Fetches the "Custom" parcellation approach. Current valid names are "HCPex" and "4S".""" is_valid_preset(name) @@ -125,10 +128,10 @@ def fetch_custom_parcel_approach( def fetch_files_from_osf( - filenames: Union[list[str], None], + filenames: list[str] | None, overwrite: bool = False, resume: bool = True, - verbose: Union[bool, int] = 1, + verbose: bool | int = 1, download_mock: bool = False, ) -> None: """ diff --git a/neurocaps/utils/parcellations.py b/neurocaps/utils/parcellations.py index ba95b1cc..5803dbd4 100644 --- a/neurocaps/utils/parcellations.py +++ b/neurocaps/utils/parcellations.py @@ -1,6 +1,6 @@ """Public utility functions for parcellations.""" -from typing import Any, Literal, Optional, Union +from typing import Any, Literal import pandas as pd @@ -14,7 +14,7 @@ def fetch_preset_parcel_approach( - name: str, n_nodes: Optional[int] = None + name: str, n_nodes: int | None = None ) -> dict[Literal["Custom"], CustomParcelApproach]: """ Fetches a Preset "Custom" Parcellation Approach. @@ -24,6 +24,10 @@ def fetch_preset_parcel_approach( Open Science Framework (OSF) if the corresponding files are not present in the directory. + .. versionchanged:: 0.37.3 + The data directory location can now be overridden via the ``NEUROCAPS_DATA`` + environment variable. If unset, the default ``~/neurocaps_data`` location is used. + Parameters ---------- name : :obj:`str` @@ -64,12 +68,12 @@ def fetch_preset_parcel_approach( def generate_custom_parcel_approach( - filepath_or_df: Union[pd.DataFrame, str], + filepath_or_df: pd.DataFrame | str, maps_path: str, column_map: dict[Literal["nodes", "regions", "hemispheres"], str], - hemisphere_map: Optional[dict[Literal["lh", "rh"], list[str]]] = None, - background_label: Optional[str] = "Background", - metadata: Optional[dict[str, Any]] = None, + hemisphere_map: dict[Literal["lh", "rh"], list[str]] | None = None, + background_label: str | None = "Background", + metadata: dict[str, Any] | None = None, ) -> dict[Literal["Custom"], CustomParcelApproach]: """ Generate a "Custom" Parcellation Approach From a Tabular Metadata File. @@ -231,7 +235,7 @@ def _validate_column_map(column_map: dict[Literal["nodes", "regions", "hemispher ) -def _open_tabular_data(filepath_or_df: Union[pd.DataFrame, str]) -> pd.DataFrame: +def _open_tabular_data(filepath_or_df: pd.DataFrame | str) -> pd.DataFrame: """Opens a tabular dataset.""" if isinstance(filepath_or_df, pd.DataFrame): return filepath_or_df @@ -260,7 +264,7 @@ def _check_if_columns_exists( def _validate_hemisphere_map( - hemisphere_map: Optional[dict[Literal["lh", "rh"], list[str]]], + hemisphere_map: dict[Literal["lh", "rh"], list[str]] | None, ) -> None: """Validates ``hemisphere_map``.""" if not hemisphere_map: @@ -281,8 +285,8 @@ def _validate_hemisphere_map( def _construct_regions_dict( df: pd.DataFrame, column_map: dict[Literal["nodes", "regions", "hemispheres"], str], - hemisphere_map: Optional[dict[Literal["lh", "rh"], list[str]]], -) -> dict[str, Union[dict[Literal["lh", "rh"], list[int]], list[int]]]: + hemisphere_map: dict[Literal["lh", "rh"], list[str]] | None, +) -> dict[str, dict[Literal["lh", "rh"], list[int]]]: """Construct the "regions" dictionary.""" regions_dict = {} diff --git a/neurocaps/utils/samples_generators.py b/neurocaps/utils/samples_generators.py index 19678315..40f1077d 100644 --- a/neurocaps/utils/samples_generators.py +++ b/neurocaps/utils/samples_generators.py @@ -1,15 +1,12 @@ """Module for creating simulated datasets.""" import json, os -from typing import Optional from joblib import Parallel, delayed from numpy.typing import NDArray from tqdm.auto import tqdm -import nibabel as nib -import numpy as np -import pandas as pd +import nibabel as nib, numpy as np, pandas as pd from neurocaps.analysis.cap._internals.surface import save_nifti_img from neurocaps.typing import SubjectTimeseries @@ -24,8 +21,8 @@ def simulate_bids_dataset( n_runs: int = 1, n_volumes: int = 100, task_name: str = "rest", - output_dir: Optional[str] = None, - n_cores: Optional[int] = None, + output_dir: str | None = None, + n_cores: int | None = None, progress_bar: bool = False, ) -> str: """ diff --git a/pyproject.toml b/pyproject.toml index 2e900bef..e0b55505 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ license = "MIT" authors = [{name = "Donisha Smith", email = "donishasmith@outlook.com"}] description = "Co-activation Patterns (CAPs) Python package" readme = "README.md" -requires-python = ">=3.9.0" +requires-python = ">=3.10.0" keywords = [ "python", @@ -29,7 +29,6 @@ classifiers = [ "Intended Audience :: Education", "Intended Audience :: Science/Research", "Topic :: Scientific/Engineering", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -67,7 +66,7 @@ dependencies = [ [project.optional-dependencies] all = ["neurocaps[benchmark, demo, development, test, windows]"] -benchmark = ["asv", "virtualenv<20.36.2"] +benchmark = ["asv", "virtualenv<21.2.4"] demo = ["ipywidgets", "openneuro-py"] development = ["pre-commit"] test = ["pytest", "pytest-cov", "pytest-rerunfailures"] diff --git a/tests/test_CAP.py b/tests/test_CAP.py index 02b7468b..0e62a816 100644 --- a/tests/test_CAP.py +++ b/tests/test_CAP.py @@ -1055,6 +1055,7 @@ def test_caps2corr(tmp_dir, method): check_outputs(tmp_dir, {"pkl": 1}, plot_type="pickle") +@pytest.mark.flaky(reruns=5) @pytest.mark.parametrize( "timeseries, parcel_approach", [ diff --git a/tests/test_TimeseriesExtractor.py b/tests/test_TimeseriesExtractor.py index 4f21f786..976ba008 100644 --- a/tests/test_TimeseriesExtractor.py +++ b/tests/test_TimeseriesExtractor.py @@ -375,6 +375,53 @@ def test_default_confounds(): assert extractor.signal_clean_info["confound_names"] is None +def test_high_pass_and_cosine_conflict(caplog): + """ + Tests to assess high pass parameter handling when cosine parameters are specified and not specified + in confound names. + """ + import logging + + confound_names = ["cosine*", "trans*", "rot*", "a_comp_cor_00", "a_comp_cor_01"] + with caplog.at_level(logging.WARNING): + extractor = TimeseriesExtractor( + high_pass=0.008, + confound_names=confound_names, + ) + + msg = ( + "Both `high_pass` and cosine regressors were specified in `confound_names`. Using both " + "introduces near-identical DCT regressors, resulting in multicollinearity issues. " + "`high_pass` has been set to None; cosine regressors from fMRIPrep will be used for " + "high-pass filtering instead." + ) + assert msg in caplog.text + assert extractor.signal_clean_info["masker_init"]["high_pass"] is None + assert "cosine*" in extractor.signal_clean_info["confound_names"] + caplog.clear() + + confound_names = ["trans*", "rot*"] + with caplog.at_level(logging.WARNING): + extractor = TimeseriesExtractor( + high_pass=0.008, + confound_names=confound_names, + ) + + assert "Both `high_pass` and cosine regressors were specified" not in caplog.text + assert extractor.signal_clean_info["masker_init"]["high_pass"] == 0.008 + caplog.clear() + + confound_names = ["cosine*", "trans*", "rot*"] + with caplog.at_level(logging.WARNING): + extractor = TimeseriesExtractor( + high_pass=None, + confound_names=confound_names, + ) + + assert "Both `high_pass` and cosine regressors were specified" not in caplog.text + assert extractor.signal_clean_info["masker_init"]["high_pass"] is None + + def test_delete_property(setup_environment_1, get_vars): """ Ensures deleter property works. @@ -526,7 +573,6 @@ def test_basic_extraction( standardize=True, use_confounds=use_confounds, low_pass=0.15, - high_pass=0.008, confound_names=["cosine*", "rot*"], ) @@ -1861,11 +1907,11 @@ def test_extended_censor_with_dummy_scans(setup_environment_1, get_vars): censored_data = copy.deepcopy(extractor_censored.subject_timeseries) # Remove dummy scans to ensure nuisance regression is the same - extractor_non_censored = TimeseriesExtractor(dummy_scans=dummy_scans) + extractor_non_censored = TimeseriesExtractor(dummy_scans=dummy_scans, standardize=False) extractor_non_censored.get_bold(bids_dir=bids_dir, task="rest", pipeline_name=pipeline_name) # Shift back by three expected_removal = np.array([35, 36, 37, 38, 39]) - dummy_scans - np.array_equal( + assert np.array_equal( censored_data["01"]["run-001"], np.delete( extractor_non_censored.subject_timeseries["01"]["run-001"], expected_removal, axis=0 diff --git a/tests/test_utils.py b/tests/test_utils.py index d620ba2a..4510389d 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -86,6 +86,7 @@ def test_fetch_preset_parcel_approach(): assert parcel_approach["Custom"]["metadata"]["n_regions"] == 7 +@pytest.mark.flaky(reruns=5) @pytest.mark.skipif( os.getenv("GITHUB_ACTIONS") and sys.platform != "linux" and sys.version_info[:2] != (3, 12), reason="Restrict file fetching in Github Actions testing to specific version and platform", diff --git a/tests/utils.py b/tests/utils.py index 23ac1120..3963e27b 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -153,7 +153,7 @@ def simulate_confounds(bids_dir, pipeline_name, use_new_compcor_format=False): json_object = json.dumps(comp_dict, indent=1) - with open(confounds_file.replace("tsv", "json"), "w") as f: + with open(confounds_file.rsplit(".tsv", 1)[0] + ".json", "w") as f: f.write(json_object) confounds_df.to_csv(confounds_file, sep="\t", index=None)