improve python-client tests

.env.example

@@ -1,2 +1,5 @@
 RSPACE_URL=
-RSPACE_API_KEY=
+RSPACE_API_KEY=
+
+RSPACE_USERNAME=
+RSPACE_PASSWORD=

.github/scripts/get_rspace_api_key.py

@@ -0,0 +1,94 @@
+import os
+import re
+import sys
+from playwright.sync_api import sync_playwright
+from dotenv import load_dotenv
+
+load_dotenv()
+
+RSPACE_URL = os.getenv("RSPACE_URL")
+RSPACE_USERNAME = os.getenv("RSPACE_USERNAME")
+RSPACE_PASSWORD = os.getenv("RSPACE_PASSWORD")
+
+
+def main():
+    if not RSPACE_URL or not RSPACE_USERNAME or not RSPACE_PASSWORD:
+        raise RuntimeError(
+            "Missing required environment variables: RSPACE_URL, RSPACE_USERNAME, RSPACE_PASSWORD"
+        )
+
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        context = browser.new_context()
+        page = context.new_page()
+
+        print("Step 1: Navigating to RSpace...", file=sys.stderr)
+        page.goto(RSPACE_URL, wait_until="networkidle")
+
+        print("Step 2: Logging in...", file=sys.stderr)
+        page.get_by_role("textbox", name="User").fill(RSPACE_USERNAME)
+        page.get_by_role("textbox", name="Password").fill(RSPACE_PASSWORD)
+        page.get_by_role("button", name="Log in").click()
+
+        # Wait for login to complete and redirect
+        page.wait_for_load_state("networkidle")
+        print("Step 3: Login successful, navigating to My RSpace...", file=sys.stderr)
+
+        # Step 2: Navigate to Profile → Manage API Key
+        page.get_by_role("link", name="My RSpace").click()
+        page.wait_for_load_state("networkidle")
+
+        print("Step 4: Looking for Generate/Regenerate key button...", file=sys.stderr)
+        # Wait for the button to appear (either "Generate key" or "Regenerate key")
+        page.wait_for_selector("a#apiKeyRegenerateBtn", timeout=10000)
+        page.click("a#apiKeyRegenerateBtn")
+
+        print("Step 5: Confirming password...", file=sys.stderr)
+        dialog = page.get_by_role("dialog", name="Confirm password")
+        dialog.wait_for()
+        dialog.get_by_role("textbox", name="Please confirm your password").fill(RSPACE_PASSWORD)
+        dialog.get_by_role("button", name="OK").click()
+        dialog.wait_for(state="detached")
+
+
+        print("Step 6: Waiting for key to be displayed...", file=sys.stderr)
+        page.wait_for_load_state("networkidle")
+
+        # Wait for the key to appear in the page
+        page.wait_for_selector("#apiKeyInfo")
+
+        # Extract the key from the displayed text
+        # Format: "Key: {32-char-string}"
+        key_locator = page.locator("div.api-menu__key")
+        if key_locator.count() == 0:
+            raise RuntimeError("Selector 'div.api-menu__key' not found on page — page structure may have changed")
+
+        info_text = key_locator.inner_text()
+        print(f"Key element text length: {len(info_text)}", file=sys.stderr)
+
+        match = re.search(r"Key:\s*([A-Za-z0-9]{32})", info_text)
+        if not match:
+            raise RuntimeError(f"API key regex did not match for selector 'div.api-menu__key' — text length {len(info_text)}")
+
+        api_key = match.group(1)
+        print("Successfully extracted API key", file=sys.stderr)
+
+        # Write to GITHUB_OUTPUT so the key never touches stdout/stderr.
+        # Writing to GITHUB_OUTPUT is the standard GitHub Actions mechanism for passing
+        # step outputs; the file is ephemeral and runner-scoped.
+        github_output = os.getenv("GITHUB_OUTPUT")
+        if github_output:
+            with open(github_output, "a") as f:
+                f.write(f"rspace_api_key={api_key}\n")
+        else:
+            raise RuntimeError("GITHUB_OUTPUT is not set; refusing to emit API key to stdout.")
+
+        browser.close()
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as exc:
+        print(f"Error generating API key: {exc}", file=sys.stderr)
+        sys.exit(1)

.github/workflows/codeql-and-tests.yml

@@ -1,14 +1,20 @@
-name: CodeQL and Unit Test
+name: codeQL and Test
 
 on:
+  push:
+    branches: [ master ]
   pull_request:
     branches: [ master ]
   workflow_dispatch:  # allow manual trigger
 
+permissions:
+  contents: read
+
 jobs:
   analyze:
     name: CodeQL Analyze
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
     permissions:
       actions: read
       contents: read
@@ -20,35 +26,147 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@v3
+        uses: github/codeql-action/init@v4
         with:
           languages: ${{ matrix.language }}
 
       - name: Autobuild
-        uses: github/codeql-action/autobuild@v3
+        uses: github/codeql-action/autobuild@v4
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v3 
+        uses: github/codeql-action/analyze@v4
 
   test:
-    name: Unit Test
-    runs-on: ubuntu-latest
+    name: unit test
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
     needs: analyze
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Set up Poetry
-        run: |
-          curl -sSL https://install.python-poetry.org | python3 -
-          echo 'export PATH="$HOME/.local/bin:$PATH"' >> $GITHUB_ENV
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.8.4
+          virtualenvs-create: true
+          virtualenvs-in-project: true
 
       - name: Install dependencies
-        run: poetry install
+        run: poetry install --no-interaction
+
+      - name: Run unit tests
+        run: poetry run pytest -m "not integration"
+
+  integration-test:
+    name: integration test
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    needs: test
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: 1.8.4
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Install dependencies
+        run: poetry install --no-interaction
+
+      - name: Clone rspace-docker
+        run: git clone https://github.com/rspace-os/rspace-docker.git /tmp/rspace-docker
+
+      - name: Download latest RSpace WAR
+        run: |
+          set -euo pipefail
+
+          release_json=$(curl -fsSL https://api.github.com/repos/rspace-os/rspace-web/releases/latest)
+          latest_tag=$(echo "$release_json" | jq -r '.tag_name')
+          war_url=$(echo "$release_json" | jq -r '.assets[]? | select(.name | test("^researchspace-.*\\.war$")) | .browser_download_url' | head -n1)
+
+          if [ -z "$latest_tag" ] || [ "$latest_tag" = "null" ]; then
+            echo "Unable to determine latest release tag for rspace-web"
+            exit 1
+          fi
+
+          if [ -z "$war_url" ] || [ "$war_url" = "null" ]; then
+            war_url="https://github.com/rspace-os/rspace-web/releases/download/${latest_tag}/researchspace-${latest_tag}.war"
+          fi
+
+          echo "Latest RSpace tag: $latest_tag"
+          curl -fsSL "$war_url" -o /tmp/rspace-docker/rspace.war
+
+      - name: Free up disk space
+        run: |
+          df -h
+          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL || true
+          docker image prune -af || true
+          df -h
+
+      - name: Start RSpace
+        working-directory: /tmp/rspace-docker
+        run: docker compose up -d
+
+      - name: Wait for RSpace to be ready
+        timeout-minutes: 10
+        run: |
+          set -euo pipefail
+
+          echo "Waiting for RSpace on http://localhost:8080 ..."
+          for i in {1..40}; do
+            if timeout 2 bash -c "cat < /dev/null > /dev/tcp/localhost/8080" 2>/dev/null; then
+              echo "RSpace is up"
+              exit 0
+            fi
+            echo "  attempt $i/40..."
+            sleep 15
+          done
+          echo "RSpace failed to start"
+          exit 1
+
+      - name: Install Playwright Browsers
+        run: poetry run python -m playwright install chromium --with-deps
+
+      - name: Generate RSpace API Key
+        id: generate_key
+        run: poetry run python .github/scripts/get_rspace_api_key.py
+        env:
+          RSPACE_URL: http://localhost:8080
+          RSPACE_USERNAME: ${{ secrets.RSPACE_USERNAME }}
+          RSPACE_PASSWORD: ${{ secrets.RSPACE_PASSWORD }}
+
+      - name: Mask API key
+        run: echo "::add-mask::${{ steps.generate_key.outputs.rspace_api_key }}"
 
-      - name: Run Unit Test
-        run: poetry run pytest rspace_client/tests
+      - name: Run integration tests
+        timeout-minutes: 20
+        env:
+          RSPACE_URL: http://localhost:8080
+          RSPACE_API_KEY: ${{ steps.generate_key.outputs.rspace_api_key }}
+        run: poetry run pytest -m integration -v

DEVELOPING.md

@@ -1,6 +1,6 @@
 ## Development
 
-Python 3.7 or later is required. We aim to support only active versions of Python.
+Python 3.9 or later is required. We aim to support only active versions of Python.
 
 ### Setup
 
@@ -21,23 +21,30 @@ to install all project dependencies into your virtual environment.
 
 ### Running tests
 
-Tests are a mixture of plain unit tests and integration tests making calls to an RSpace server.
-To run all tests, set these environment variables,replacing with your own values
+Tests are a mixture of plain unit tests and integration tests that make calls to a live RSpace server.
+
+#### Unit tests only
 
 ```
-bash> export RSPACE_URL=https:/<your-rspace-domain>
-bash> export RSPACE_API_KEY=abcdefgh...
+poetry run pytest -m "not integration"
 ```
 
-If these aren't set, integration tests will be skipped.
+#### Integration tests
+
+Integration tests require credentials for a live RSpace instance. Create a `.env` file in the project root:
+
+```
+RSPACE_URL=https://<your-rspace-domain>
+RSPACE_API_KEY=<your-api-key>
+```
 
-Tests can be invoked:
+Then run:
 
 ```
-poetry run pytest rspace_client/tests
+poetry run pytest -m integration
 ```
 
-They should be run with a new RSpace account that does not belong to any groups.
+Integration tests should be run with a new RSpace account that does not belong to any groups.
 
 ### Writing Tests
 

pyproject.toml

@@ -25,12 +25,20 @@ python = "^3.9"
 requests = "^2.25.1"
 beautifulsoup4 = "^4.9.3"
 fs = "^2.4.16"
+setuptools = "<82"
 
 [tool.poetry.group.dev.dependencies]
 python-dotenv = "^1.1.1"
 black = "^21.6b0"
-pytest = "^6.2.4"
+pytest = "^8.0.0"
+playwright = "^1.58.0"
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
+
+[tool.pytest.ini_options]
+testpaths = ["rspace_client/tests"]
+markers = [
+    "integration: marks tests as integration tests requiring a live RSpace server (deselect with '-m not integration')",
+]

rspace_client/inv/inv.py

@@ -809,7 +809,8 @@ def __init__(
         subsample_count: int = None,
         total_quantity: Quantity = None,
         attachments=None,
-        barcodes: Optional[List[Barcode]] = None
+        barcodes: Optional[List[Barcode]] = None,
+        location: "TargetLocation" = None,
     ):
         super().__init__(name, "SAMPLE", tags, description, extra_fields)
         ## converts arguments into JSON POST syntax
@@ -828,10 +829,11 @@ def __init__(
             self.data["templateId"] = sample_template_id
         if fields is not None:
             self.data["fields"] = fields
+        if location is not None:
+            self.data.update(location.data)
         ## fail early
-        if attachments is not None:
-            if not isinstance(attachments, list):
-                raise ValueError("attachments must be a list of open files")
+        if attachments is not None and not isinstance(attachments, list):
+            raise ValueError("attachments must be a list of open files")
         if barcodes is not None:
             self.data["barcodes"] = [barcode.to_dict() for barcode in barcodes]