HDXDSYS-2116 Investigate and fix HDX Analysis Scripts failure

.coveragerc

@@ -1,14 +1,17 @@
 [run]
-relative_files = True
+source = src
 
-[report]
-omit =
-    */run.py
-    */python?.?/*
-    */venv/*
-    */site-packages/*
-    */tests/*
-    *__init__*
+omit = */_version.py
 
-exclude_lines =
-    if __name__ == '__main__':
+[report]
+exclude_also =
+    from ._version
+    def __repr__
+    if self.debug:
+    if settings.DEBUG
+    raise AssertionError
+    raise NotImplementedError
+    if 0:
+    if __name__ == .__main__.:
+    if TYPE_CHECKING:
+    @(abc\.)?abstractmethod

.github/workflows/run-python-script-datasetsinfo.yml

@@ -9,19 +9,19 @@ on:
 
 jobs:
   run:
-
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python 3.x
       uses: actions/setup-python@v5
       with:
-        python-version: 3.x
+        python-version: "3.x"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install .
     - name: Run script
       env:
         HDX_KEY: ${{ secrets.HDX_BOT_SCRAPERS_API_TOKEN }}
@@ -31,7 +31,7 @@ jobs:
         MIXPANEL_PROJECT_ID: ${{ secrets.MIXPANEL_PROJECT_ID }}
         MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
       run: |
-        python get_datasets_info.py --output_dir=datasets_info
+        python -m hdx.analysis_script.datasets --output_dir=datasets_info
     - name: Deploy 🚀
       if: always()
       uses: JamesIves/github-pages-deploy-action@v4

.github/workflows/run-python-script-orgstats.yml

@@ -9,19 +9,19 @@ on:
 
 jobs:
   run:
-
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python 3.x
       uses: actions/setup-python@v5
       with:
-        python-version: 3.x
+        python-version: "3.x"
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        pip install .
     - name: Run script
       env:
         HDX_KEY: ${{ secrets.HDX_BOT_SCRAPERS_API_TOKEN }}
@@ -31,7 +31,7 @@ jobs:
         MIXPANEL_PROJECT_ID: ${{ secrets.MIXPANEL_PROJECT_ID }}
         MIXPANEL_TOKEN: ${{ secrets.MIXPANEL_TOKEN }}
       run: |
-        python get_org_stats.py --output_dir=org_stats
+        python -m hdx.analysis_script.orgs --output_dir=org_stats
     - name: Deploy 🚀
       if: always()
       uses: JamesIves/github-pages-deploy-action@v4

.github/workflows/run-python-tests.yml

@@ -1,4 +1,4 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# This workflow will install Python dependencies, lint and run tests
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
 name: Run tests
@@ -8,15 +8,19 @@ on:
   push:
     branches-ignore:
       - gh-pages
+      - 'dependabot/**'
   pull_request:
     branches-ignore:
       - gh-pages
 
-
 jobs:
   build:
     runs-on: ubuntu-latest
 
+    permissions:
+      checks: write
+      pull-requests: write
+
     steps:
     - uses: actions/checkout@v4
     - name: Set up Python
@@ -26,25 +30,24 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 coveralls
-        if [ -f test-requirements.txt ]; then pip install  --upgrade -r test-requirements.txt; fi
-    - name: Lint with flake8
+    - name: Install Hatch
+      uses: pypa/hatch@install
+    - name: Test with hatch/pytest
       run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test with pytest
+        hatch test
+    - name: Check styling
+      if: always()
       run: |
-        pytest --junitxml=junit/test-results.xml --cov-config .coveragerc --cov-report= --cov=.
-    - name: Publish in Coveralls
-      uses: AndreMiras/coveralls-python-action@develop
-      with:
-        github-token: ${{ secrets.GITHUB_TOKEN }}
-        flag-name: tests
+        hatch fmt --check
     - name: Publish Unit Test Results
       uses: EnricoMi/publish-unit-test-result-action@v2
       if: always()
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
-        junit_files: junit/*.xml
+        junit_files: test-results.xml
+    - name: Publish in Coveralls
+      uses: coverallsapp/github-action@v2
+      with:
+        github-token: ${{ secrets.GITHUB_TOKEN }}
+        flag-name: tests
+        format: lcov

.gitignore

@@ -8,7 +8,6 @@ __pycache__/
 
 # Distribution / packaging
 .Python
-env/
 build/
 develop-eggs/
 dist/
@@ -20,9 +19,12 @@ lib64/
 parts/
 sdist/
 var/
+wheels/
+share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
+MANIFEST
 
 # PyInstaller
 #  Usually these files are written by a python script from a template
@@ -37,15 +39,17 @@ pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 .tox/
+.nox/
 .coverage
 .coverage.*
 .cache
 nosetests.xml
 coverage.xml
-*,cover
+*.cover
+*.py,cover
 .hypothesis/
-tests/*.log
 .pytest_cache/
+cover/
 
 # Translations
 *.mo
@@ -54,6 +58,8 @@ tests/*.log
 # Django stuff:
 *.log
 local_settings.py
+db.sqlite3
+db.sqlite3-journal
 
 # Flask stuff:
 instance/
@@ -63,38 +69,99 @@ instance/
 .scrapy
 
 # Sphinx documentation
-documentation/_*/
-documentation/source/
+docs/_build/
 
 # PyBuilder
+.pybuilder/
 target/
 
-# IPython Notebook
+# Jupyter Notebook
 .ipynb_checkpoints
 
-# pyenv
-.python-version
+# IPython
+profile_default/
+ipython_config.py
 
-# celery beat schedule file
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
 celerybeat-schedule
+celerybeat.pid
 
-# dotenv
-.env
+# SageMath parsed files
+*.sage.py
 
-# virtualenv
+# Environments
+.env
+.venv
+env/
 venv/
 ENV/
+env.bak/
+venv.bak/
 
 # Spyder project settings
 .spyderproject
+.spyproject
 
 # Rope project settings
 .ropeproject
 
-# IntelliJ
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
 
-# setuptools_scm
+## Project
+# Version file
 _version.py
 
-.vscode/settings.json
+#OS Files
+*DS_Store

.pre-commit-config.yaml

@@ -0,0 +1,31 @@
+default_language_version:
+    python: python3.13
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-ast
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.10
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.6.5
+    hooks:
+      # Run the pip compile
+      - id: pip-compile
+        name: pip-compile requirements.txt
+        files: pyproject.toml
+        args: [ pyproject.toml, --resolver=backtracking, --upgrade, -q,
+                -o, requirements.txt ]
+      - id: pip-compile
+        name: pip-compile requirements-test.txt
+        files: pyproject.toml
+        args: [ pyproject.toml, --resolver=backtracking, --upgrade, -q,
+                --extra, test, -c, requirements.txt, -o, requirements-test.txt ]

README.md

@@ -34,7 +34,7 @@ pip install -r requirements.txt
 pip install -r test-requirements.txt
 ```
 
-`hdx-analysis-scripts` uses the `hdx-python-api` library, configuration for which is done in the usual way [described here](https://hdx-python-api.readthedocs.io/en/latest/). 
+`hdx-analysis-scripts` uses the `hdx-python-api` library, configuration for which is done in the usual way [described here](https://hdx-python-api.readthedocs.io/en/latest/).
 
 For local use the user agent (`AnalysisScripts`) is specified in the `~/.useragents.yaml` file.
 ```yaml
@@ -52,5 +52,3 @@ MIXPANEL_TOKEN
 ```
 
 Mixpanel accounts for users authenticate by sending a login email to registered users on login request.
-
-