Upstream sync for Ckan 2.11 compatibility

.flake8

@@ -17,4 +17,6 @@ max-line-length=127
 
 # List ignore rules one per line.
 ignore =
+    C901
+    E501
     W503

.github/workflows/publish.yml

@@ -31,77 +31,32 @@ on:
 
 
 jobs:
-  lint:
-    if: github.repository == 'ckan/ckanext-xloader'
+
+  validateVersion:
     runs-on: ubuntu-latest
+    if: github.repository == 'ckan/ckanext-xloader'
     steps:
       - uses: actions/checkout@v4
+
       - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
-      - name: Install requirements
-        run: pip install flake8 pycodestyle
-      - name: Check syntax
-        run: flake8 . --count --select=E901,E999,F821,F822,F823 --show-source --statistics --extend-exclude ckan
 
-  test:
-    needs: lint
-    strategy:
-      matrix:
-        include: #ckan-image see https://github.com/ckan/ckan-docker-base, ckan-version controls other image tags
-          - ckan-version: "2.11"
-            ckan-image: "2.11-py3.10"
-          - ckan-version: "2.10"
-            ckan-image: "2.10-py3.10"
-          - ckan-version: "2.9"
-            ckan-image: "2.9-py3.9"
-          #- ckan-version: "master" Publish does not care about master
-          #  ckan-image: "master"
-      fail-fast: false
-
-    name: CKAN ${{ matrix.ckan-version }}
-    runs-on: ubuntu-latest
-    container:
-      image: ckan/ckan-dev:${{ matrix.ckan-image }}
-      options: --user root
-    services:
-      solr:
-        image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9
-      postgres:
-        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
-        env:
-          POSTGRES_USER: postgres
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_DB: postgres
-        ports:
-          - 5432:5432
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
-      redis:
-          image: redis:3
-    env:
-      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
-      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
-      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
-      CKAN_SOLR_URL: http://solr:8983/solr/ckan
-      CKAN_REDIS_URL: redis://redis:6379/1
+      - name: Validate tag version
+        if: ${{ startsWith(github.ref, 'refs/tags') }}
+        run: |
+          TAG_VALUE=${GITHUB_REF/refs\/tags\//}
+          PYTHON_VERSION=$(grep -E '\bversion\s?=\s?"[^"]+"' pyproject.toml  | awk -F '"' '{print $2}')
+          echo "Tag version is [$TAG_VALUE], Python version is [$PYTHON_VERSION]"
+          if [ "$TAG_VALUE" != "$PYTHON_VERSION" ]; then
+            echo "Version mismatch; tag version is [$TAG_VALUE] but Python version is [$PYTHON_VERSION]" >> $GITHUB_STEP_SUMMARY
+            exit 1
+          fi
 
-    steps:
-    - uses: actions/checkout@v4
-    - if: ${{ matrix.ckan-version == 2.9 }}
-      run: pip install "setuptools>=44.1.0,<71"
-    - name: Install requirements
-      run: |
-        pip install -r requirements.txt
-        pip install -r dev-requirements.txt
-        pip install -e .
-        pip install -U requests[security]
-        # Replace default path to CKAN core config file with the one on the container
-        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
-    - name: Setup extension (CKAN >= 2.9)
-      run: |
-        ckan -c test.ini db init
-    - name: Run tests
-      run: pytest --ckan-ini=test.ini --cov=ckanext.xloader --disable-warnings ckanext/xloader/tests
+  test:
+    needs: validateVersion
+    name: Test
+    uses: ./.github/workflows/test.yml # Call the reusable workflow
 
   publishSkipped:
     if: github.repository != 'ckan/ckanext-xloader'
@@ -138,10 +93,10 @@ jobs:
           echo "reponame=${reponame}" >> $GITHUB_OUTPUT
 
           if [ "$env.ENVIRONMENT" == "testpypi" ]; then
-            url="https://test.pypi.com/p/$reponame"
+            url="https://test.pypi.org/project/$reponame/$TAG_VALUE/"
             echo "environment=${env.ENVIRONMENT}" >> $GITHUB_OUTPUT
           else
-            url="https://pypi.com/p/$reponame"
+            url="https://pypi.org/project/$reponame/$TAG_VALUE/"
             echo "environment=pypi" >> $GITHUB_OUTPUT
           fi
 
@@ -150,16 +105,6 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Validate tag version
-        if: ${{ startsWith(github.ref, 'refs/tags') }}
-        run: |
-          PYTHON_VERSION=$(grep -E "\bversion='[^']+'" setup.py | awk -F "'" '{print $2}')
-          echo "Tag version is [${{ steps.version.outputs.version }}], Python version is [$PYTHON_VERSION]"
-          if [ "${{ steps.version.outputs.version }}" != "$PYTHON_VERSION" ]; then
-            echo "Version mismatch; tag version is [${{ steps.version.outputs.version }}] but Python version is [$PYTHON_VERSION]" >> $GITHUB_STEP_SUMMARY
-            exit 1
-          fi
-
       - name: Build package ${{ steps.version.outputs.reponame }} @ ${{ steps.version.outputs.version }}
         run: |
           pip install build

.github/workflows/test.yml

@@ -5,6 +5,7 @@ on:
   pull_request:
     branches:
       - master
+  workflow_call:
 
 jobs:
   lint:
@@ -30,19 +31,15 @@ jobs:
             experimental: false
           - ckan-version: "2.10"
             ckan-image: "2.10-py3.10"
-            solr-version: "9"
-            experimental: false
-          - ckan-version: "2.9"
-            ckan-image: "2.9-py3.9"
-            solr-version: "8"
             experimental: false
+            solr-version: "9"
           - ckan-version: "master"
             ckan-image: "master"
             solr-version: "9"
             experimental: true  # master is unstable, good to know if we are compatible or not
       fail-fast: false
 
-    name: CKAN ${{ matrix.ckan-version }}
+    name: ${{ matrix.experimental && '**Fail_Ignored** ' || '' }} CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
     container:
       image: ckan/ckan-dev:${{ matrix.ckan-image }}
@@ -60,7 +57,7 @@ jobs:
           - 5432:5432
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
       redis:
-          image: redis:3
+          image: redis:7
     env:
       CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
       CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
@@ -72,12 +69,7 @@ jobs:
     - uses: actions/checkout@v4
       continue-on-error: ${{ matrix.experimental }}
 
-    - name: Pin setuptools for ckan 2.9 only
-      if: ${{ matrix.ckan-version == 2.9 }}
-      run: pip install "setuptools>=44.1.0,<71"
-      continue-on-error: ${{ matrix.experimental }}
-
-    - name: Install requirements
+    - name: ${{ matrix.experimental && '**Fail_Ignored** ' || '' }} Install requirements
       continue-on-error: ${{ matrix.experimental }}
       run: |
         pip install -r requirements.txt
@@ -90,16 +82,20 @@ jobs:
         apt-get update
         apt-get install unzip -y
 
-    - name: Setup extension
+    - name: ${{ matrix.experimental && '**Fail_Ignored** ' || '' }} Setup extension
       continue-on-error: ${{ matrix.experimental }}
       run: |
         ckan -c test.ini db init
+        ckan -c test.ini user add ckan_admin email=ckan_admin@localhost password="AbCdEf12345!@#%"
+        ckan -c test.ini sysadmin add ckan_admin
+        ckan config-tool test.ini "ckanext.xloader.api_token=$(ckan -c test.ini user token add ckan_admin xloader | tail -n 1 | tr -d '\t')"
+        ckan -c test.ini user list
 
-    - name: Run tests
+    - name: ${{ matrix.experimental && '**Fail_Ignored** ' || '' }} Run tests
       continue-on-error: ${{ matrix.experimental }}
       run: pytest --ckan-ini=test.ini --cov=ckanext.xloader --disable-warnings ckanext/xloader/tests --junit-xml=/tmp/artifacts/junit/results.xml
 
-    - name: Test Summary
+    - name: ${{ matrix.experimental && '**Fail_Ignored** ' || '' }} Test Summary
       uses: test-summary/action@v2
       continue-on-error: ${{ matrix.experimental }}
       with:
@@ -108,7 +104,7 @@ jobs:
 
     - name: SonarQube Scan
       uses: sonarsource/sonarqube-scan-action@master
-      with: 
+      with:
         args: >
           -Dsonar.projectKey=ckanext-xloader
           -Dsonar.sources=ckanext/xloader

CHANGELOG

@@ -1,14 +1,67 @@
-1.1.2 2024-10-25
+CHANGELOG
+=========
+See: https://github.com/ckan/ckanext-xloader/releases if this file has drifted.
+
+2.0.1 2025-03-04
+================
+
+## Fix
+
+* #244 Static webassets not included in package
+* #245 support apitoken_header_name in 2.11.x.
+* #241 loading R/W datasource resources via api (not hardcoded)
+
+2.0.0 2024-12-10
 ================
 
-Feat:
+## Major
+Dropped CKAN 2.9.x and Python2.
+
+
+## Feat:
+* Adds Strip White Space fields to the Data Dictionary (defualts to `True` for each field).
+  This will strip surrounding white space from data values prior to inserting them into the database.
+* Adds support for ckanext-validation. Config `ckanext.xloader.validation.requires_successful_report`
+  controls whether a resource requires a successful validation report to be XLoadered.
+  By default, a resource would also require a Validation Schema, which can be turned off with
+  `ckanext.xloader.validation.enforce_schema`.
 * Frontend Status Badges by @JVickery-TBS in https://github.com/ckan/ckanext-xloader/pull/224
 
-Fix:
+
+## Fix:
+* Properly handle REDIS queue timeouts to close/delete any temporary files.
 * Fix automated PyPI publishing by @ThrawnCA in https://github.com/ckan/ckanext-xloader/pull/231
 
+## What's Changed
+* Update README, migrate it to Markdown by @amercader in https://github.com/ckan/ckanext-xloader/pull/235
+* chore: switch to pyproject.toml by @duttonw in https://github.com/ckan/ckanext-xloader/pull/236
+* Validation Extension Support (Squashed) by @JVickery-TBS in https://github.com/ckan/ckanext-xloader/pull/237
+* Strip White Space from Cell Values (Squashed) by @JVickery-TBS in https://github.com/ckan/ckanext-xloader/pull/238
+* RQ Job Timeout Handling (Squashed) by @JVickery-TBS in https://github.com/ckan/ckanext-xloader/pull/239
+* SQLAlchemy v2 support by @smotornyuk in https://github.com/ckan/ckanext-xloader/pull/225
+
+**Full Changelog**: https://github.com/ckan/ckanext-xloader/compare/1.2.0...2.0.0
+
+1.2.0 2024-11-21
+================
+
+## What's Changed
+* Fix PyPI publishing by @ThrawnCA in https://github.com/ckan/ckanext-xloader/pull/233
+* Enhancement/Bugfix: Downstream qld-gov-au fix's by @duttonw in https://github.com/ckan/ckanext-xloader/pull/232
+  * feat: @JVickery-TBS work on validation integration (await successful validation prior to doing datastore work via 'IPipeValidation'
+  * fix: handle gracefully if tabulator load fails by trying 'direct load'
+  * fix: Excel blank header row bug
+  * fix: Datastore truncate, restart identity so numbering restarts from 0 again (when imported data has same columns and types
+  * fix: parital fix on DB deadlock by adding timeouts on DDL events
+  * test: test_simple_large_file, test_with_blanks, test_with_empty_lines, test_with_extra_blank_cells
+  * test: test_require_validation, test_enforce_validation_schema
+  * chore: min version requirements for cve's,
+     *   requests>=2.32.0
+     *   urllib3>=2.2.2
+     *   zipp>=3.19.1
+
+**Full Changelog**: https://github.com/ckan/ckanext-xloader/compare/1.1.2...1.2.0
 
-**Full Changelog**: https://github.com/ckan/ckanext-xloader/compare/1.1.1...1.1.2
 
 1.1.1 2024-10-16
 ================
@@ -21,7 +74,7 @@ Fix:
 1.1.0 2024-10-16
 ================
 
-Fixes: 
+Fixes:
 * feat: Add pypi cicd publish via github action via environment controls by @duttonw in https://github.com/ckan/ckanext-xloader/pull/228
 
 
@@ -32,7 +85,7 @@ Fixes:
 ================
 
 
-Fixes: 
+Fixes:
 
 * add README note about running on separate server, #191 by @ThrawnCA in https://github.com/ckan/ckanext-xloader/pull/192
 * Use IDomainObjectModification Implementation by @JVickery-TBS in https://github.com/ckan/ckanext-xloader/pull/198

MANIFEST.in

@@ -1,6 +1,9 @@
 include *requirements*.txt
 include CHANGELOG
 include LICENSE
-include README.rst
+include README.md
 include ckanext/xloader/config_declaration.yaml
 recursive-include ckanext/xloader/templates *.html
+recursive-include ckanext/xloader/webassets *.css
+recursive-include ckanext/xloader/webassets *.yml
+recursive-include ckanext/xloader/webassets *.js