diff --git a/.github/workflows/build_and_test_push.yaml b/.github/workflows/build_and_test_push.yaml index 0b81f2a..4ed6a63 100644 --- a/.github/workflows/build_and_test_push.yaml +++ b/.github/workflows/build_and_test_push.yaml @@ -27,218 +27,4 @@ jobs: - name: Run Tests run: go test -v github.com/uc-cdis/gen3-client/tests - build: - env: - goarch: amd64 - needs: test - runs-on: ubuntu-latest - strategy: - matrix: - include: - - goos: linux - goarch: amd64 - zipfile: dataclient_linux.zip - - goos: darwin - goarch: amd64 - zipfile: dataclient_osx.zip - - goos: windows - goarch: amd64 - zipfile: dataclient_win64.zip - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - - name: Setup Go 1.17 - uses: actions/setup-go@v4 - with: - go-version: '1.17' - - - name: Run Setup Script - run: | - bash .github/scripts/before_install.sh - env: - GITHUB_BRANCH: ${{ github.ref_name }} - ACCESS_KEY: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} - SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} - - - - name: Run Build Script - run: | - bash .github/scripts/build.sh - env: - GOOS: ${{ matrix.goos }} - GOARCH: ${{ env.goarch }} - GITHUB_BRANCH: ${{ github.ref_name }} - GITHUB_PULL_REQUEST: ${{ github.event_name == 'pull_request' }} - - - name: Upload Artifacts - uses: actions/upload-artifact@v4 - with: - name: build-artifact-${{ matrix.goos }} - path: ~/shared/${{ matrix.zipfile }} - retention-days: 3 - - - sign: - needs: build - runs-on: macos-latest - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - name: Download OSX Artifact - uses: actions/download-artifact@v4 - with: - name: build-artifact-darwin - path: ./dist - - name: Unzip OSX Artifact and remove zip file - run: | - cd ./dist - ls - unzip dataclient_osx.zip - rm dataclient_osx.zip - - - - - name: Build executable - shell: bash - env: - APPLE_CERT_PASSWORD: ${{ secrets.APPLE_CERT_PASSWORD }} - APPLE_NOTARY_UUID: ${{ secrets.APPLE_NOTARY_UUID }} - APPLE_NOTARY_KEY: ${{ secrets.APPLE_NOTARY_KEY}} - APPLE_NOTARY_DATA: ${{ secrets.APPLE_NOTARY_DATA }} - APPLE_CERT_DATA: ${{ secrets.APPLE_CERT_DATA }} - APPLICATION_CERT_PASSWORD: ${{ secrets.APPLICATION_CERT_PASSWORD }} - APPLICATION_CERT_DATA: ${{ secrets.APPLICATION_CERT_DATA }} - APPLE_TEAM_ID: WYQ7U7YUC9 - - run: | - # Setup - SIGNFILE="$(pwd)/dist/gen3-client" - - # Export certs - echo "$APPLE_CERT_DATA" | base64 --decode > /tmp/certs.p12 - echo "$APPLE_NOTARY_DATA" | base64 --decode > /tmp/notary.p8 - echo "$APPLICATION_CERT_DATA" | base64 --decode > /tmp/app_certs.p12 - - # Create keychain - security create-keychain -p actions macos-build.keychain - security default-keychain -s macos-build.keychain - security unlock-keychain -p actions macos-build.keychain - security set-keychain-settings -t 3600 -u macos-build.keychain - - # Import certs to keychain - security import /tmp/certs.p12 -k ~/Library/Keychains/macos-build.keychain -P "$APPLE_CERT_PASSWORD" -T /usr/bin/codesign -T /usr/bin/productsign - security import /tmp/app_certs.p12 -k ~/Library/Keychains/macos-build.keychain -P "$APPLICATION_CERT_PASSWORD" -T /usr/bin/codesign -T /usr/bin/productsign - - # Key signing - security set-key-partition-list -S apple-tool:,apple: -s -k actions macos-build.keychain - - # Verify keychain things - security find-identity -v macos-build.keychain | grep "$APPLE_TEAM_ID" | grep "Developer ID Application" - security find-identity -v macos-build.keychain | grep "$APPLE_TEAM_ID" | grep "Developer ID Installer" - - # Force the codesignature - codesign --force --options=runtime --keychain "/Users/runner/Library/Keychains/macos-build.keychain-db" -s "$APPLE_TEAM_ID" "$SIGNFILE" - # Verify the code signature - codesign -v "$SIGNFILE" --verbose - - mkdir -p ./dist/pkg - cp ./dist/gen3-client ./dist/pkg/gen3-client - pkgbuild --identifier "org.uc-cdis.gen3-client.pkg" --timestamp --install-location /Applications --root ./dist/pkg installer.pkg - pwd - ls - productbuild --resources ./resources --distribution ./distribution.xml gen3-client.pkg - productsign --sign "$APPLE_TEAM_ID" --timestamp gen3-client.pkg gen3-client_signed.pkg - - xcrun notarytool store-credentials "notarytool-profile" --issuer $APPLE_NOTARY_UUID --key-id $APPLE_NOTARY_KEY --key /tmp/notary.p8 - xcrun notarytool submit gen3-client_signed.pkg --keychain-profile "notarytool-profile" --wait - xcrun stapler staple gen3-client_signed.pkg - mv gen3-client_signed.pkg dataclient_osx.pkg - - - name: Upload signed artifact - uses: actions/upload-artifact@v4 - with: - name: build-artifact-darwin-signed - path: dataclient_osx.pkg - - sync_signed_to_aws: - runs-on: ubuntu-latest - needs: sign - - steps: - - name: Checkout Code - uses: actions/checkout@v4 - - name: Run Setup Script - run: | - bash ./.github/scripts/before_install.sh - env: - GITHUB_BRANCH: ${{ github.ref_name }} - ACCESS_KEY: ${{ secrets.AWS_S3_ACCESS_KEY_ID }} - SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_ACCESS_KEY }} - - - name: Download OSX Artifact - uses: actions/download-artifact@v4 - with: - name: build-artifact-darwin-signed - - - name: Sync to AWS - env: - GITHUB_BRANCH: ${{ github.ref_name }} - run: | - rm ~/shared/dataclient_osx.zip - zip dataclient_osx_signed.zip dataclient_osx.pkg - mv dataclient_osx_signed.zip ~/shared/ - aws s3 sync ~/shared s3://cdis-dc-builds/$GITHUB_BRANCH - - - get_tagged_branch: - if: startsWith(github.ref, 'refs/tags/') - runs-on: ubuntu-latest - needs: [build,sign] - outputs: - branch: ${{ steps.check_step.outputs.branch }} - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Get current branch - id: check_step - # 1. Get the list of branches ref where this tag exists - # 2. Remove 'origin/' from that result - # 3. Put that string in output - # => We can now use function 'contains(list, item)'' - run: | - raw=$(git branch -r --contains ${{ github.ref }}) - branch="$(echo ${raw//origin\//} | tr -d '\n')" - echo "{name}=branch" >> $GITHUB_OUTPUT - echo "Branches where this tag exists : $branch." - - - deploy: - needs: get_tagged_branch - if: startsWith(github.ref, 'refs/tags/') && contains(${{ needs.get_tagged_branch.outputs.branch }}, 'master') - runs-on: ubuntu-latest - steps: - - name: Download Linux Artifact - uses: actions/download-artifact@v4 - with: - name: build-artifact-linux - - - name: Download OSX Artifact - uses: actions/download-artifact@v4 - with: - name: build-artifact-darwin-signed - - - name: Download Windows Artifact - uses: actions/download-artifact@v4 - with: - name: build-artifact-windows - - - name: Create Release gh cli - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_TAG: ${{ github.ref_name }} - run: gh release create "$GH_TAG" dataclient_linux.zip dataclient_osx.pkg dataclient_win64.zip --repo="$GITHUB_REPOSITORY" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..7d3ce32 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,36 @@ +name: Data Client CI + +on: + pull_request: + push: + workflow_dispatch: + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +jobs: + lint-and-test: + name: Lint and Unit Tests + runs-on: ubuntu-latest + + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Download dependencies + run: GOTOOLCHAIN=auto go mod download + + - name: Run go vet + run: GOTOOLCHAIN=auto go vet ./... + + - name: Run unit tests + run: GOTOOLCHAIN=auto go test -v ./... diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml new file mode 100644 index 0000000..9164185 --- /dev/null +++ b/.github/workflows/coverage.yaml @@ -0,0 +1,123 @@ +name: "Test Coverage Check" + +on: + pull_request: + branches: + - master + push: + branches: + - master + +jobs: + coverage: + name: Test Coverage + runs-on: ubuntu-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.24.2' + + - name: Run Tests with Coverage + run: | + CORE_PACKAGES=$(go list ./... | grep -Ev '/(cmd|mocks|tests)$') + go test -coverprofile=coverage.out -covermode=atomic $CORE_PACKAGES + continue-on-error: true + + - name: Generate Coverage Report + id: coverage + run: | + CORE_PACKAGES=$(go list ./... | grep -Ev '/(cmd|mocks|tests)$') + + # Get overall coverage + OVERALL=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//') + echo "overall=$OVERALL" >> $GITHUB_OUTPUT + + # Generate detailed report + echo "## Test Coverage Report" > coverage-report.md + echo "" >> coverage-report.md + echo "**Overall Coverage:** ${OVERALL}%" >> coverage-report.md + echo "" >> coverage-report.md + echo "### Package Coverage" >> coverage-report.md + echo "" >> coverage-report.md + echo "| Package | Coverage |" >> coverage-report.md + echo "|---------|----------|" >> coverage-report.md + + # Extract package coverage + go test -coverprofile=/dev/null -covermode=atomic $CORE_PACKAGES 2>&1 | \ + awk '/^ok[[:space:]]/ { + pkg=$2; + cov=$5; + gsub(/github.com\/calypr\/data-client\//, "", pkg); + print "| " pkg " | " cov " |" + }' >> coverage-report.md + + cat coverage-report.md + + - name: Check Coverage Thresholds + run: | + set -euo pipefail + + CORE_PACKAGES=$(go list ./... | grep -Ev '/(cmd|mocks|tests)$') + + OVERALL=$(go tool cover -func=coverage.out | awk '/^total:/ {gsub(/%/, "", $3); print $3}') + if ! awk -v val="$OVERALL" -v min=30 'BEGIN { if (val + 0 < min) exit 1; exit 0 }'; then + echo "Overall coverage ${OVERALL}% is below the required minimum of 30%" + exit 1 + fi + + while read -r pkg cov; do + cov=${cov%%%} + if ! awk -v val="$cov" -v min=20 'BEGIN { if (val + 0 < min) exit 1; exit 0 }'; then + echo "Package ${pkg} coverage ${cov}% is below the required minimum of 20%" + exit 1 + fi + done < <( + go test -coverprofile=/dev/null -covermode=atomic $CORE_PACKAGES 2>&1 | \ + awk '/^ok[[:space:]]/ { + pkg=$2; + cov=$5; + gsub(/github.com\/calypr\/data-client\//, "", pkg); + print pkg, cov + }' + ) + + - name: Upload Coverage to Codecov (Optional) + uses: codecov/codecov-action@v4 + if: always() + with: + files: ./coverage.out + flags: unittests + name: codecov-umbrella + fail_ci_if_error: false + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + - name: Comment PR with Coverage + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const coverage = fs.readFileSync('coverage-report.md', 'utf8'); + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: coverage + }); + + - name: Upload Coverage Artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: coverage-report + path: | + coverage.out + coverage-report.md + retention-days: 30 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..c23cf06 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,34 @@ +name: Release + +on: + push: + tags: + - '*' + workflow_dispatch: + +permissions: + contents: write + +jobs: + goreleaser: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + - + name: Set up Go + uses: actions/setup-go@v5 + - + name: Run GoReleaser + uses: goreleaser/goreleaser-action@v6 + with: + # either 'goreleaser' (default) or 'goreleaser-pro' + distribution: goreleaser + # 'latest', 'nightly', or a semver + version: 'latest' + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 849f232..453ca8f 100644 --- a/.gitignore +++ b/.gitignore @@ -27,4 +27,9 @@ # Build artifacts /build/ -checksums.txt \ No newline at end of file +/bin/ +checksums.txt +# Local caches and binaries +/.gocache/ +/.tmp/ +/data-client diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..177c5c6 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "ga4gh/data-repository-service-schemas"] + path = ga4gh/data-repository-service-schemas + url = https://github.com/kellrott/data-repository-service-schemas.git + branch = feature/get-by-checksum diff --git a/Makefile b/Makefile index e524d95..1d556f7 100644 --- a/Makefile +++ b/Makefile @@ -10,9 +10,17 @@ MAIN_PACKAGE := . # The directory where the final binary will be placed BIN_DIR := ./bin +# Coverage thresholds +COVERAGE_THRESHOLD := 30 +PACKAGE_COVERAGE_THRESHOLD := 20 +CORE_PACKAGES := $(shell go list ./... | grep -Ev '/(cmd|mocks|tests)$$') + +# OpenAPI generation now lives in syfon. +SYFON_DIR ?= ../syfon + # --- Targets --- -.PHONY: all build test generate tidy clean help +.PHONY: all build test test-coverage coverage-html coverage-check generate tidy clean help # The default target run when you type 'make' all: build @@ -28,19 +36,80 @@ test: @echo "--> Running all tests..." @go test -v ./... +## test-coverage: Runs tests with coverage profiling +test-coverage: + @echo "--> Running tests with coverage..." + @go test -coverprofile=coverage.out -covermode=atomic $(CORE_PACKAGES) + @echo "--> Coverage report generated: coverage.out" + @go tool cover -func=coverage.out | tail -1 + +## coverage-html: Generates HTML coverage report +coverage-html: test-coverage + @echo "--> Generating HTML coverage report..." + @go tool cover -html=coverage.out -o coverage.html + @echo "--> HTML coverage report generated: coverage.html" + +## coverage-check: Verifies coverage meets minimum thresholds +coverage-check: test-coverage + @echo "--> Checking coverage thresholds..." + @set -euo pipefail; \ + OVERALL=$$(go tool cover -func=coverage.out | awk '/^total:/ {gsub(/%/, "", $$3); print $$3}'); \ + if ! awk -v val="$$OVERALL" -v min=$(COVERAGE_THRESHOLD) 'BEGIN { if (val + 0 < min) exit 1; exit 0 }'; then \ + echo "Overall coverage $$OVERALL% is below the required minimum of $(COVERAGE_THRESHOLD)%"; \ + exit 1; \ + fi; \ + go test -coverprofile=/dev/null -covermode=atomic $(CORE_PACKAGES) 2>&1 | \ + awk '/^ok[[:space:]]/ { \ + pkg=$$2; \ + cov=$$5; \ + gsub(/github.com\\/calypr\\/data-client\\//, "", pkg); \ + print pkg, cov; \ + }' | \ + while read -r pkg cov; do \ + case "$$pkg" in \ + cmd|mocks|tests) continue ;; \ + esac; \ + cov=$${cov%%%}; \ + if ! awk -v val="$$cov" -v min=$(PACKAGE_COVERAGE_THRESHOLD) 'BEGIN { if (val + 0 < min) exit 1; exit 0 }'; then \ + echo "Package $$pkg coverage $$cov% is below the required minimum of $(PACKAGE_COVERAGE_THRESHOLD)%"; \ + exit 1; \ + fi; \ + done + ## generate: Runs go generate commands to create mocks, embedded assets, etc. generate: @echo "--> Running code generation (go generate)..." @go generate ./... +## gen: Generates Go models from OpenAPI specs +gen: + @set -euo pipefail; \ + if [[ ! -d "$(SYFON_DIR)" ]]; then \ + echo "ERROR: syfon repo not found at $(SYFON_DIR)"; \ + exit 1; \ + fi; \ + echo "--> OpenAPI generation is centralized in syfon"; \ + $(MAKE) -C "$(SYFON_DIR)" gen + +.PHONY: gen-internal +gen-internal: + @set -euo pipefail; \ + if [[ ! -d "$(SYFON_DIR)" ]]; then \ + echo "ERROR: syfon repo not found at $(SYFON_DIR)"; \ + exit 1; \ + fi; \ + echo "--> OpenAPI generation is centralized in syfon make gen"; \ + $(MAKE) -C "$(SYFON_DIR)" gen + ## tidy: Cleans up module dependencies and formats go files tidy: @echo "--> Tidying go.mod and formatting files..." @go mod tidy @go fmt ./... -## clean: Removes the compiled binary +## clean: Removes the compiled binary and coverage files clean: @echo "--> Cleaning up..." @rm -f $(BIN_DIR)/$(TARGET_NAME) - + @rm -f coverage.out coverage.html + @rm -rf .tmp diff --git a/README.md b/README.md index d8cfe6d..4d16202 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # data-client -[![Build Status](https://travis-ci.org/uc-cdis/cdis-data-client.svg?branch=master)](https://travis-ci.org/uc-cdis/cdis-data-client) -[![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/uc-cdis/cdis-data-client?sort=semver)](https://github.com/uc-cdis/cdis-data-client/releases) +[![CI](https://github.com/calypr/data-client/actions/workflows/ci.yaml/badge.svg?branch=master)](https://github.com/calypr/data-client/actions/workflows/ci.yaml) +[![Coverage](https://codecov.io/gh/calypr/data-client/branch/develop/graph/badge.svg)](https://app.codecov.io/gh/calypr/data-client/tree/develop) +[![Go Report Card](https://goreportcard.com/badge/github.com/calypr/data-client)](https://goreportcard.com/report/github.com/calypr/data-client) +[![Release](https://img.shields.io/github/v/release/calypr/data-client?sort=semver)](https://github.com/calypr/data-client/releases) `data-client` is a command-line tool for downloading, uploading, and submitting data files to and from a Gen3 data commons. diff --git a/build.sh b/build.sh deleted file mode 100755 index 213b40a..0000000 --- a/build.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# -# Adapted from 'How To Build Go Executables for Multiple Platforms on Ubuntu 16.04' -# By Marko Mudrinić -# -# Usage: -# ./build.sh - -if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then - echo "usage: $0" - echo "output: zipped executables to ./build directory" -fi - -package=$1 - -if [[ -z "$package" ]]; then - package='gen3-client' -fi - -package_split=(${package//\// }) -package_name=${package_split[-1]} - -platforms=( - "darwin/arm64" - "darwin/amd64" - "linux/amd64" - "windows/amd64" -) - -mkdir -p ./build -> checksums.txt -for platform in "${platforms[@]}" -do - platform_split=(${platform//\// }) - GOOS=${platform_split[0]} - GOARCH=${platform_split[1]} - output_name=$package_name'-'$GOOS'-'$GOARCH - exe_name=$package_name - - if [ $GOOS = "windows" ]; then - exe_name+='.exe' - - elif [ $GOOS = "darwin" ]; then - if [ $GOARCH = "arm64" ]; then - output_name=$package_name'-macos' - - elif [ $GOARCH = "amd64" ]; then - output_name=$package_name'-macos-intel' - fi - fi - - printf 'Building %s...' "$output_name" - env GOOS=$GOOS GOARCH=$GOARCH go build -o ./build/$exe_name . - cd build - zip -r -q $output_name $exe_name - sha256sum $output_name.zip >> checksums.txt - cd .. - - if [ $? -ne 0 ]; then - echo 'An error has occurred! Aborting the script execution...' - exit 1 - fi - echo 'OK' -done - -# Clean up build artifacts -rm build/{$package_name,$package_name.exe} - diff --git a/bump-tag.sh b/bump-tag.sh new file mode 100644 index 0000000..b38169e --- /dev/null +++ b/bump-tag.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# File: `bump-patch.sh` +set -euo pipefail + +# Find latest tag excluding major v0 +LATEST_TAG=$(git tag --list --sort=-v:refname | grep -v '^v0' | head -n1 || true) +if [ -z "$LATEST_TAG" ]; then + echo "No suitable tag found (excluding v0). Aborting." >&2 + exit 1 +fi + +# check that the working directory is clean +if [ -n "$(git status --porcelain)" ]; then + echo "Working directory is not clean. Please commit or stash changes before running this script." >&2 + exit 1 +fi + +usage() { + cat <<-EOF +Usage: $0 [--major | --minor | --patch] + +LATEST_TAG: $LATEST_TAG + +Options: + --major Bump major (MAJOR+1, MINOR=0, PATCH=0) + --minor Bump minor (MINOR+1, PATCH=0) + --patch Bump patch (PATCH+1) [default] +EOF + exit 1 +} + +# Parse options +opt_major=false +opt_minor=false +opt_patch=false +count=0 + +while [ $# -gt 0 ]; do + case "$1" in + --major) + opt_major=true + count=$((count + 1)) + shift + ;; + --minor) + opt_minor=true + count=$((count + 1)) + shift + ;; + --patch) + opt_patch=true + count=$((count + 1)) + shift + ;; + --help|-h) + usage + ;; + *) + echo "Unknown option: $1" >&2 + usage + ;; + esac +done + +# Default to patch if no option provided +if [ "$count" -eq 0 ]; then + opt_patch=true +fi + +# Disallow specifying more than one +if [ "$count" -gt 1 ]; then + echo "Specify only one of --major, --minor, or --patch" >&2 + exit 1 +fi + + +# Parse semver vMAJOR.MINOR.PATCH +if [[ "$LATEST_TAG" =~ ^v?([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then + MAJOR="${BASH_REMATCH[1]}" + MINOR="${BASH_REMATCH[2]}" + PATCH="${BASH_REMATCH[3]}" +else + echo "Latest tag '$LATEST_TAG' is not in semver format. Aborting." >&2 + exit 1 +fi + +# Compute new version +if [ "$opt_major" = true ]; then + NEW_MAJOR=$((MAJOR + 1)) + NEW_MINOR=0 + NEW_PATCH=0 + NEW_TAG="${NEW_MAJOR}.${NEW_MINOR}.${NEW_PATCH}" + NEW_FILE_VER="${NEW_MAJOR}.${NEW_MINOR}.${NEW_PATCH}" +elif [ "$opt_minor" = true ]; then + NEW_MAJOR=$MAJOR + NEW_MINOR=$((MINOR + 1)) + NEW_PATCH=0 + NEW_TAG="${NEW_MAJOR}.${NEW_MINOR}.${NEW_PATCH}" + NEW_FILE_VER="${NEW_MAJOR}.${NEW_MINOR}.${NEW_PATCH}" +else + # patch + NEW_MAJOR=$MAJOR + NEW_MINOR=$MINOR + NEW_PATCH=$((PATCH + 1)) + NEW_TAG="${NEW_MAJOR}.${NEW_MINOR}.${NEW_PATCH}" + NEW_FILE_VER="${NEW_MAJOR}.${NEW_MINOR}.${NEW_PATCH}" +fi + +BRANCH="$(git rev-parse --abbrev-ref HEAD)" + +echo "Latest branch: $BRANCH" +echo "Latest tag: $LATEST_TAG" +echo "New tag: $NEW_TAG (files will use ${NEW_FILE_VER})" + +# Update internal version file +if [ -f cmd/gitversion.go ]; then + # sed on mac is -i '' + sed -E -i '' -e "s/(gitversion *= *\")[^\"]+(\")/\1${NEW_FILE_VER}\2/" cmd/gitversion.go + git add cmd/gitversion.go +fi + +# Commit, tag and push +git commit -m "chore(release): bump to ${NEW_TAG}" || echo "No changes to commit" +git tag -a "${NEW_TAG}" -m "Release ${NEW_TAG}" +echo "Created tag. Please push tag ${NEW_TAG} on branch ${BRANCH}." + +echo git push origin "${BRANCH}" +echo git push origin "${NEW_TAG}" diff --git a/client/common/common.go b/client/common/common.go deleted file mode 100644 index 8eea7bf..0000000 --- a/client/common/common.go +++ /dev/null @@ -1,205 +0,0 @@ -package common - -import ( - "fmt" - "io" - "log" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/hashicorp/go-multierror" - "github.com/vbauerster/mpb/v8" -) - -// DefaultUseShepherd sets whether gen3client will attempt to use the Shepherd / Object Management API -// endpoints if available. -// The user can override this default using the `data-client configure` command. -const DefaultUseShepherd = false - -// DefaultMinShepherdVersion is the minimum version of Shepherd that the gen3client will use. -// Before attempting to use Shepherd, the client will check for Shepherd's version, and if the version is -// below this number the gen3client will instead warn the user and fall back to fence/indexd. -// The user can override this default using the `data-client configure` command. -const DefaultMinShepherdVersion = "2.0.0" - -// ShepherdEndpoint is the endpoint postfix for SHEPHERD / the Object Management API -const ShepherdEndpoint = "/mds" - -// ShepherdVersionEndpoint is the endpoint used to check what version of Shepherd a commons has deployed -const ShepherdVersionEndpoint = "/mds/version" - -// IndexdIndexEndpoint is the endpoint postfix for INDEXD index -const IndexdIndexEndpoint = "/index/index" - -// FenceUserEndpoint is the endpoint postfix for FENCE user -const FenceUserEndpoint = "/user/user" - -// FenceDataEndpoint is the endpoint postfix for FENCE data -const FenceDataEndpoint = "/user/data" - -// FenceAccessTokenEndpoint is the endpoint postfix for FENCE access token -const FenceAccessTokenEndpoint = "/user/credentials/api/access_token" - -// FenceDataUploadEndpoint is the endpoint postfix for FENCE data upload -const FenceDataUploadEndpoint = FenceDataEndpoint + "/upload" - -// FenceDataDownloadEndpoint is the endpoint postfix for FENCE data download -const FenceDataDownloadEndpoint = FenceDataEndpoint + "/download" - -// FenceDataMultipartInitEndpoint is the endpoint postfix for FENCE multipart init -const FenceDataMultipartInitEndpoint = FenceDataEndpoint + "/multipart/init" - -// FenceDataMultipartUploadEndpoint is the endpoint postfix for FENCE multipart upload -const FenceDataMultipartUploadEndpoint = FenceDataEndpoint + "/multipart/upload" - -// FenceDataMultipartCompleteEndpoint is the endpoint postfix for FENCE multipart complete -const FenceDataMultipartCompleteEndpoint = FenceDataEndpoint + "/multipart/complete" - -// PathSeparator is os dependent path separator char -const PathSeparator = string(os.PathSeparator) - -// DefaultTimeout is used to set timeout value for http client -const DefaultTimeout = 120 * time.Second - -// FileUploadRequestObject defines a object for file upload -type FileUploadRequestObject struct { - FilePath string - Filename string - FileMetadata FileMetadata - GUID string - PresignedURL string - Request *http.Request - Progress *mpb.Progress - Bar *mpb.Bar - Bucket string `json:"bucket,omitempty"` -} - -// FileDownloadResponseObject defines a object for file download -type FileDownloadResponseObject struct { - DownloadPath string - Filename string - GUID string - URL string - Range int64 - Overwrite bool - Skip bool - Response *http.Response - Writer io.Writer -} - -// FileMetadata defines the metadata accepted by the new object management API, Shepherd -type FileMetadata struct { - Authz []string `json:"authz"` - Aliases []string `json:"aliases"` - // Metadata is an encoded JSON string of any arbitrary metadata the user wishes to upload. - Metadata map[string]any `json:"metadata"` -} - -// RetryObject defines a object for retry upload -type RetryObject struct { - FilePath string - Filename string - FileMetadata FileMetadata - GUID string - RetryCount int - Multipart bool - Bucket string -} - -// ParseRootPath parses dirname that has "~" in the beginning -func ParseRootPath(filePath string) (string, error) { - if filePath != "" && filePath[0] == '~' { - homeDir, err := os.UserHomeDir() - if err != nil { - return "", err - } - return homeDir + filePath[1:], nil - } - return filePath, nil -} - -// GetAbsolutePath parses input file path to its absolute path and removes the "~" in the beginning -func GetAbsolutePath(filePath string) (string, error) { - fullFilePath, err := ParseRootPath(filePath) - if err != nil { - return "", err - } - fullFilePath, err = filepath.Abs(fullFilePath) - return fullFilePath, err -} - -// ParseFilePaths generates all possible file paths -func ParseFilePaths(filePath string, metadataEnabled bool) ([]string, error) { - fullFilePath, err := GetAbsolutePath(filePath) - if err != nil { - return []string{}, err - } - initialPaths, err := filepath.Glob(fullFilePath) - if err != nil { - return []string{}, err - } - - var multiErr *multierror.Error - var finalFilePaths []string - for _, p := range cleanupHiddenFiles(initialPaths) { - file, err := os.Open(p) - if err != nil { - multiErr = multierror.Append(multiErr, fmt.Errorf("file open error for %s: %w", p, err)) - continue - } - - func(filePath string, file *os.File) { - defer file.Close() - - fi, _ := file.Stat() - if fi.IsDir() { - err = filepath.Walk(filePath, func(path string, fileInfo os.FileInfo, err error) error { - if err != nil { - return err - } - isHidden, err := IsHidden(path) - if err != nil { - return err - } - isMetadata := false - if metadataEnabled { - isMetadata = strings.HasSuffix(path, "_metadata.json") - } - if !fileInfo.IsDir() && !isHidden && !isMetadata { - finalFilePaths = append(finalFilePaths, path) - } - return nil - }) - if err != nil { - multiErr = multierror.Append(multiErr, fmt.Errorf("directory walk error for %s: %w", filePath, err)) - } - } else { - finalFilePaths = append(finalFilePaths, filePath) - } - }(p, file) - } - - return finalFilePaths, multiErr.ErrorOrNil() -} - -func cleanupHiddenFiles(filePaths []string) []string { - i := 0 - for _, filePath := range filePaths { - isHidden, err := IsHidden(filePath) - if err != nil { - log.Println("Error occurred when checking hidden files: " + err.Error()) - continue - } - - if isHidden { - log.Printf("File %s is a hidden file and will be skipped\n", filePath) - continue - } - filePaths[i] = filePath - i++ - } - return filePaths[:i] -} diff --git a/client/common/isHidden_notwindows.go b/client/common/isHidden_notwindows.go deleted file mode 100644 index 321601c..0000000 --- a/client/common/isHidden_notwindows.go +++ /dev/null @@ -1,16 +0,0 @@ -//go:build !windows -// +build !windows - -package common - -import ( - "path/filepath" -) - -func IsHidden(filePath string) (bool, error) { - filename := filepath.Base(filePath) - if filename[0:1] == "." || filename[0:1] == "~" { // also takes care of temp files - return true, nil - } - return false, nil -} diff --git a/client/common/isHidden_windows.go b/client/common/isHidden_windows.go deleted file mode 100644 index b719ae2..0000000 --- a/client/common/isHidden_windows.go +++ /dev/null @@ -1,30 +0,0 @@ -//go:build windows -// +build windows - -package common - -import ( - "errors" - "path/filepath" - "runtime" - "syscall" -) - -func IsHidden(filePath string) (bool, error) { - filename := filepath.Base(filePath) - if runtime.GOOS == "windows" { - if filename[0:1] == "." || filename[0:1] == "~" { - return true, nil - } - pointer, err := syscall.UTF16PtrFromString(filePath) - if err != nil { - return false, err - } - attributes, err := syscall.GetFileAttributes(pointer) - if err != nil { - return false, err - } - return attributes&syscall.FILE_ATTRIBUTE_HIDDEN != 0, nil - } - return false, errors.New("Unable to check if file is hidden under non-Windows OS") -} diff --git a/client/common/logHelper.go b/client/common/logHelper.go deleted file mode 100644 index a117bbc..0000000 --- a/client/common/logHelper.go +++ /dev/null @@ -1,24 +0,0 @@ -package common - -import ( - "encoding/json" - "os" -) - -func LoadFailedLog(path string) (map[string]RetryObject, error) { - data, err := os.ReadFile(path) - if err != nil { - return nil, err - } - var m map[string]RetryObject - if err := json.Unmarshal(data, &m); err != nil { - return nil, err - } - return m, nil -} - -func AlreadySucceededFromFile(filePath string) bool { - // Simple: check if any succeeded log contains this path - // Or just return false — safer to re-upload than skip - return false -} diff --git a/client/g3cmd/auth.go b/client/g3cmd/auth.go deleted file mode 100644 index 2dbd361..0000000 --- a/client/g3cmd/auth.go +++ /dev/null @@ -1,76 +0,0 @@ -package g3cmd - -import ( - "context" - "encoding/json" - "log" - "sort" - - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - "github.com/spf13/cobra" -) - -func init() { - var profile string - var authCmd = &cobra.Command{ - Use: "auth", - Short: "Return resource access privileges from profile", - Long: `Gets resource access privileges for specified profile.`, - Example: `./data-client auth --profile=`, - Run: func(cmd *cobra.Command, args []string) { - // don't initialize transmission logs for non-uploading related commands - - logger, logCloser := logs.New(profile, logs.WithConsole()) - defer logCloser() - - g3i, err := client.NewGen3Interface(context.Background(), profile, logger) - if err != nil { - log.Fatalf("Fatal NewGen3Interface error: %s\n", err) - } - - host, resourceAccess, err := g3i.CheckPrivileges() - if err != nil { - g3i.Logger().Fatalf("Fatal authentication error: %s\n", err) - } else { - if len(resourceAccess) == 0 { - g3i.Logger().Printf("\nYou don't currently have access to any resources at %s\n", host) - } else { - g3i.Logger().Printf("\nYou have access to the following resource(s) at %s:\n", host) - - // Sort by resource name - resources := make([]string, 0, len(resourceAccess)) - for resource := range resourceAccess { - resources = append(resources, resource) - } - sort.Strings(resources) - - for _, project := range resources { - // Sort by access name if permissions are from Fence - permissions := resourceAccess[project].([]any) - _, isFencePermission := permissions[0].(string) - if isFencePermission { - access := make([]string, 0, len(permissions)) - for _, permission := range permissions { - access = append(access, permission.(string)) - } - sort.Strings(access) - g3i.Logger().Printf("%s %s\n", project, access) - } else { - // Permissions from Arborist already sorted, just pretty print them - marshalledPermissions, err := json.MarshalIndent(permissions, "", " ") - if err != nil { - g3i.Logger().Printf("%s (error occurred when marshalling permissions): %s\n", project, err) - } - g3i.Logger().Printf("%s %s\n", project, marshalledPermissions) - } - } - } - } - }, - } - - authCmd.Flags().StringVar(&profile, "profile", "", "Specify the profile to check your access privileges") - authCmd.MarkFlagRequired("profile") // nolint: errcheck - RootCmd.AddCommand(authCmd) -} diff --git a/client/g3cmd/delete.go b/client/g3cmd/delete.go deleted file mode 100644 index 5be6795..0000000 --- a/client/g3cmd/delete.go +++ /dev/null @@ -1,34 +0,0 @@ -package g3cmd - -import ( - "log" - - "github.com/spf13/cobra" -) - -//Not support yet, place holder only - -var deleteCmd = &cobra.Command{ // nolint:deadcode,unused,varcheck - Use: "delete", - Short: "Send DELETE HTTP Request for given URI", - Long: `Deletes a given URI from the database. -If no profile is specified, "default" profile is used for authentication.`, - Example: `./data-client delete --uri=v0/submission/bpa/test/entities/example_id - ./data-client delete --profile=user1 --uri=v0/submission/bpa/test/entities/1af1d0ab-efec-4049-98f0-ae0f4bb1bc64`, - Run: func(cmd *cobra.Command, args []string) { - log.Fatalf("Not supported!") - // request := new(jwt.Request) - // configure := new(jwt.Configure) - // function := new(jwt.Functions) - - // function.Config = configure - // function.Request = request - - // fmt.Println(jwt.ResponseToString( - // function.DoRequestWithSignedHeader(RequestDelete, profile, "txt", uri))) - }, -} - -func init() { - // RootCmd.AddCommand(deleteCmd) -} diff --git a/client/g3cmd/download-multiple.go b/client/g3cmd/download-multiple.go deleted file mode 100644 index d8dbca8..0000000 --- a/client/g3cmd/download-multiple.go +++ /dev/null @@ -1,495 +0,0 @@ -package g3cmd - -import ( - "bufio" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "log" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - - "github.com/calypr/data-client/client/common" - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - "github.com/vbauerster/mpb/v8" - "github.com/vbauerster/mpb/v8/decor" - - "github.com/spf13/cobra" -) - -// mockgen -destination=../mocks/mock_gen3interface.go -package=mocks . Gen3Interface - -func AskGen3ForFileInfo(g3i client.Gen3Interface, guid string, protocol string, downloadPath string, filenameFormat string, rename bool, renamedFiles *[]RenamedOrSkippedFileInfo) (string, int64) { - var fileName string - var fileSize int64 - - // If the commons has the newer Shepherd API deployed, get the filename and file size from the Shepherd API. - // Otherwise, fall back on Indexd and Fence. - hasShepherd, err := g3i.CheckForShepherdAPI() - if err != nil { - g3i.Logger().Println("Error occurred when checking for Shepherd API: " + err.Error()) - g3i.Logger().Println("Falling back to Indexd...") - } - if hasShepherd { - endPointPostfix := common.ShepherdEndpoint + "/objects/" + guid - _, res, err := g3i.GetResponse(endPointPostfix, "GET", "", nil) - if err != nil { - g3i.Logger().Println("Error occurred when querying filename from Shepherd: " + err.Error()) - g3i.Logger().Println("Using GUID for filename instead.") - if filenameFormat != "guid" { - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: "N/A", NewFilename: guid}) - } - return guid, 0 - } - - decoded := struct { - Record struct { - FileName string `json:"file_name"` - Size int64 `json:"size"` - } - }{} - err = json.NewDecoder(res.Body).Decode(&decoded) - if err != nil { - g3i.Logger().Println("Error occurred when reading response from Shepherd: " + err.Error()) - g3i.Logger().Println("Using GUID for filename instead.") - if filenameFormat != "guid" { - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: "N/A", NewFilename: guid}) - } - return guid, 0 - } - defer res.Body.Close() - - fileName = decoded.Record.FileName - fileSize = decoded.Record.Size - - } else { - // Attempt to get the filename from Indexd - endPointPostfix := common.IndexdIndexEndpoint + "/" + guid - indexdMsg, err := g3i.DoRequestWithSignedHeader(endPointPostfix, "", nil) - if err != nil { - g3i.Logger().Println("Error occurred when querying filename from IndexD: " + err.Error()) - g3i.Logger().Println("Using GUID for filename instead.") - if filenameFormat != "guid" { - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: "N/A", NewFilename: guid}) - } - return guid, 0 - } - - if filenameFormat == "guid" { - return guid, indexdMsg.Size - } - - actualFilename := indexdMsg.FileName - if actualFilename == "" { - if len(indexdMsg.URLs) > 0 { - // Indexd record has no file name but does have URLs, try to guess file name from URL - var indexdURL = indexdMsg.URLs[0] - if protocol != "" { - for _, url := range indexdMsg.URLs { - if strings.HasPrefix(url, protocol) { - indexdURL = url - } - } - } - - actualFilename = guessFilenameFromURL(indexdURL) - if actualFilename == "" { - g3i.Logger().Println("Error occurred when guessing filename for object " + guid) - g3i.Logger().Println("Using GUID for filename instead.") - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: "N/A", NewFilename: guid}) - return guid, indexdMsg.Size - } - } else { - // Neither file name nor URLs exist in the Indexd record - // Indexd record is busted for that file, just return as we are renaming the file for now - // The download logic will handle the errors - g3i.Logger().Println("Neither file name nor URLs exist in the Indexd record of " + guid) - g3i.Logger().Println("The attempt of downloading file is likely to fail! Check Indexd record!") - g3i.Logger().Println("Using GUID for filename instead.") - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: "N/A", NewFilename: guid}) - return guid, indexdMsg.Size - } - } - - fileName = actualFilename - fileSize = indexdMsg.Size - } - - if filenameFormat == "original" { - if !rename { // no renaming in original mode - return fileName, fileSize - } - newFilename := processOriginalFilename(downloadPath, fileName) - if fileName != newFilename { - *renamedFiles = append(*renamedFiles, RenamedOrSkippedFileInfo{GUID: guid, OldFilename: fileName, NewFilename: newFilename}) - } - return newFilename, fileSize - } - // filenameFormat == "combined" - combinedFilename := guid + "_" + fileName - return combinedFilename, fileSize -} - -func guessFilenameFromURL(URL string) string { - splittedURLWithFilename := strings.Split(URL, "/") - actualFilename := splittedURLWithFilename[len(splittedURLWithFilename)-1] - return actualFilename -} - -func processOriginalFilename(downloadPath string, actualFilename string) string { - _, err := os.Stat(downloadPath + actualFilename) - if os.IsNotExist(err) { - return actualFilename - } - extension := filepath.Ext(actualFilename) - filename := strings.TrimSuffix(actualFilename, extension) - counter := 2 - for { - newFilename := filename + "_" + strconv.Itoa(counter) + extension - _, err := os.Stat(downloadPath + newFilename) - if os.IsNotExist(err) { - return newFilename - } - counter++ - } -} - -func validateLocalFileStat(logger logs.Logger, downloadPath string, filename string, filesize int64, skipCompleted bool) common.FileDownloadResponseObject { - fi, err := os.Stat(downloadPath + filename) // check filename for local existence - if err != nil { - if os.IsNotExist(err) { - return common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename} // no local file, normal full length download - } - logger.Printf("Error occurred when getting information for file \"%s\": %s\n", downloadPath+filename, err.Error()) - logger.Println("Will try to download the whole file") - return common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename} // errorred when trying to get local FI, normal full length download - } - - // have existing local file and may want to skip, check more conditions - if !skipCompleted { - return common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename, Overwrite: true} // not skipping any local files, normal full length download - } - - localFilesize := fi.Size() - if localFilesize == filesize { - return common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename, Skip: true} // both filename and filesize matches, consider as completed - } - if localFilesize > filesize { - return common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename, Overwrite: true} // local filesize is greater than INDEXD record, overwrite local existing - } - // local filesize is less than INDEXD record, try ranged download - return common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename, Range: localFilesize} -} - -func batchDownload(g3 client.Gen3Interface, progress *mpb.Progress, batchFDRSlice []common.FileDownloadResponseObject, protocolText string, workers int, errCh chan error) int { - fdrs := make([]common.FileDownloadResponseObject, 0) - for _, fdrObject := range batchFDRSlice { - err := GetDownloadResponse(g3, &fdrObject, protocolText) - if err != nil { - errCh <- err - continue - } - - fileFlag := os.O_CREATE | os.O_RDWR - if fdrObject.Range != 0 { - fileFlag = os.O_APPEND | os.O_RDWR - } else if fdrObject.Overwrite { - fileFlag = os.O_TRUNC | os.O_RDWR - } - - subDir := filepath.Dir(fdrObject.Filename) - if subDir != "." && subDir != "/" { - err = os.MkdirAll(fdrObject.DownloadPath+subDir, 0766) - if err != nil { - errCh <- err - continue - } - } - file, err := os.OpenFile(fdrObject.DownloadPath+fdrObject.Filename, fileFlag, 0666) - if err != nil { - errCh <- errors.New("Error occurred during opening local file: " + err.Error()) - continue - } - total := fdrObject.Response.ContentLength + fdrObject.Range - bar := progress.AddBar(total, - mpb.PrependDecorators( - decor.Name(fdrObject.Filename+" "), - decor.CountersKibiByte("% .1f / % .1f"), - ), - mpb.AppendDecorators( - decor.Percentage(), - decor.AverageSpeed(decor.SizeB1024(0), " % .1f"), - ), - ) - if fdrObject.Range > 0 { - bar.SetCurrent(fdrObject.Range) - } - writer := bar.ProxyWriter(file) - fdrObject.Writer = writer - fdrs = append(fdrs, fdrObject) - defer file.Close() - defer fdrObject.Response.Body.Close() - } - - fdrCh := make(chan common.FileDownloadResponseObject, len(fdrs)) - wg := sync.WaitGroup{} - succeeded := 0 - var err error - for range workers { - wg.Add(1) - go func() { - for fdr := range fdrCh { - if _, err = io.Copy(fdr.Writer, fdr.Response.Body); err != nil { - errCh <- errors.New("io.Copy error: " + err.Error()) - return - } - succeeded++ - } - wg.Done() - }() - } - - for _, fdr := range fdrs { - fdrCh <- fdr - } - close(fdrCh) - - wg.Wait() - return succeeded -} - -// AskForConfirmation asks user for confirmation before proceed, will wait if user entered garbage -func AskForConfirmation(logger logs.Logger, s string) bool { - reader := bufio.NewReader(os.Stdin) - - for { - logger.Printf("%s [y/n]: ", s) - - response, err := reader.ReadString('\n') - if err != nil { - logger.Fatal("Error occurred during parsing user's confirmation: " + err.Error()) - } - - switch strings.ToLower(strings.TrimSpace(response)) { - case "y", "yes": - return true - case "n", "no": - return false - default: - return false // Example of defaulting to false - } - } -} - -func downloadFile(g3i client.Gen3Interface, objects []ManifestObject, downloadPath string, filenameFormat string, rename bool, noPrompt bool, protocol string, numParallel int, skipCompleted bool) error { - if numParallel < 1 { - return fmt.Errorf("invalid value for option \"numparallel\": must be a positive integer! Please check your input") - } - - downloadPath, err := common.ParseRootPath(downloadPath) - if err != nil { - return fmt.Errorf("downloadFile Error: %s", err.Error()) - } - if !strings.HasSuffix(downloadPath, "/") { - downloadPath += "/" - } - filenameFormat = strings.ToLower(strings.TrimSpace(filenameFormat)) - if (filenameFormat == "guid" || filenameFormat == "combined") && rename { - g3i.Logger().Println("NOTICE: flag \"rename\" only works if flag \"filename-format\" is \"original\"") - rename = false - } - - if filenameFormat != "original" && filenameFormat != "guid" && filenameFormat != "combined" { - return fmt.Errorf("invalid option found! option \"filename-format\" can either be \"original\", \"guid\" or \"combined\" only") - } - if filenameFormat == "guid" || filenameFormat == "combined" { - g3i.Logger().Printf("WARNING: in \"guid\" or \"combined\" mode, duplicated files under \"%s\" will be overwritten\n", downloadPath) - if !noPrompt && !AskForConfirmation(g3i.Logger(), "Proceed?") { - g3i.Logger().Fatal("Aborted by user") - } - } else if !rename { - g3i.Logger().Printf("WARNING: flag \"rename\" was set to false in \"original\" mode, duplicated files under \"%s\" will be overwritten\n", downloadPath) - if !noPrompt && !AskForConfirmation(g3i.Logger(), "Proceed?") { - g3i.Logger().Fatal("Aborted by user") - } - } else { - g3i.Logger().Printf("NOTICE: flag \"rename\" was set to true in \"original\" mode, duplicated files under \"%s\" will be renamed by appending a counter value to the original filenames\n", downloadPath) - } - - protocolText := "" - if protocol != "" { - protocolText = "?protocol=" + protocol - } - - err = os.MkdirAll(downloadPath, 0766) - if err != nil { - return fmt.Errorf("cannot create folder %s", downloadPath) - } - - renamedFiles := make([]RenamedOrSkippedFileInfo, 0) - skippedFiles := make([]RenamedOrSkippedFileInfo, 0) - fdrObjects := make([]common.FileDownloadResponseObject, 0) - - g3i.Logger().Printf("Total number of objects in manifest: %d\n", len(objects)) - g3i.Logger().Println("Preparing file info for each file, please wait...") - fileInfoProgress := mpb.New(mpb.WithOutput(os.Stdout)) - fileInfoBar := fileInfoProgress.AddBar(int64(len(objects)), - mpb.PrependDecorators( - decor.Name("Preparing files "), - decor.CountersNoUnit("%d / %d"), - ), - mpb.AppendDecorators(decor.Percentage()), - ) - for _, obj := range objects { - if obj.ObjectID == "" { - g3i.Logger().Println("Found empty object_id (GUID), skipping this entry") - continue - } - var fdrObject common.FileDownloadResponseObject - filename := obj.Filename - filesize := obj.Filesize - // only queries Gen3 services if any of these 2 values doesn't exists in manifest - if filename == "" || filesize == 0 { - filename, filesize = AskGen3ForFileInfo(g3i, obj.ObjectID, protocol, downloadPath, filenameFormat, rename, &renamedFiles) - } - fdrObject = common.FileDownloadResponseObject{DownloadPath: downloadPath, Filename: filename} - if !rename { - fdrObject = validateLocalFileStat(g3i.Logger(), downloadPath, filename, filesize, skipCompleted) - } - fdrObject.GUID = obj.ObjectID - fdrObjects = append(fdrObjects, fdrObject) - fileInfoBar.Increment() - } - fileInfoProgress.Wait() - g3i.Logger().Println("File info prepared successfully") - - totalCompeleted := 0 - workers, _, errCh, _ := initBatchUploadChannels(numParallel, len(fdrObjects)) - downloadProgress := mpb.New(mpb.WithOutput(os.Stdout)) - batchFDRSlice := make([]common.FileDownloadResponseObject, 0) - for _, fdrObject := range fdrObjects { - if fdrObject.Skip { - g3i.Logger().Printf("File \"%s\" (GUID: %s) has been skipped because there is a complete local copy\n", fdrObject.Filename, fdrObject.GUID) - skippedFiles = append(skippedFiles, RenamedOrSkippedFileInfo{GUID: fdrObject.GUID, OldFilename: fdrObject.Filename}) - continue - } - - if len(batchFDRSlice) < workers { - batchFDRSlice = append(batchFDRSlice, fdrObject) - } else { - totalCompeleted += batchDownload(g3i, downloadProgress, batchFDRSlice, protocolText, workers, errCh) - batchFDRSlice = make([]common.FileDownloadResponseObject, 0) - batchFDRSlice = append(batchFDRSlice, fdrObject) - } - } - totalCompeleted += batchDownload(g3i, downloadProgress, batchFDRSlice, protocolText, workers, errCh) // download remainders - downloadProgress.Wait() - - g3i.Logger().Printf("%d files downloaded.\n", totalCompeleted) - - if len(renamedFiles) > 0 { - g3i.Logger().Printf("%d files have been renamed as the following:\n", len(renamedFiles)) - for _, rfi := range renamedFiles { - g3i.Logger().Printf("File \"%s\" (GUID: %s) has been renamed as: %s\n", rfi.OldFilename, rfi.GUID, rfi.NewFilename) - } - } - if len(skippedFiles) > 0 { - g3i.Logger().Printf("%d files have been skipped\n", len(skippedFiles)) - } - if len(errCh) > 0 { - close(errCh) - g3i.Logger().Printf("%d files have encountered an error during downloading, detailed error messages are:\n", len(errCh)) - for err := range errCh { - g3i.Logger().Println(err.Error()) - } - } - return nil -} - -func init() { - var manifestPath string - var downloadPath string - var filenameFormat string - var rename bool - var noPrompt bool - var protocol string - var numParallel int - var skipCompleted bool - - var downloadMultipleCmd = &cobra.Command{ - Use: "download-multiple", - Short: "Download multiple of files from a specified manifest", - Long: `Get presigned URLs for multiple of files specified in a manifest file and then download all of them.`, - Example: `./data-client download-multiple --profile= --manifest= --download-path=`, - Run: func(cmd *cobra.Command, args []string) { - // don't initialize transmission logs for non-uploading related commands - - logger, logCloser := logs.New(profile, logs.WithConsole(), logs.WithFailedLog(), logs.WithScoreboard(), logs.WithSucceededLog()) - defer logCloser() - - g3i, err := client.NewGen3Interface(context.Background(), profile, logger) - if err != nil { - log.Fatalf("Failed to parse config on profile %s, %v", profile, err) - } - - manifestPath, _ = common.GetAbsolutePath(manifestPath) - manifestFile, err := os.Open(manifestPath) - if err != nil { - g3i.Logger().Fatalf("Failed to open manifest file %s, %v\n", manifestPath, err) - } - defer manifestFile.Close() - manifestFileStat, err := manifestFile.Stat() - if err != nil { - g3i.Logger().Fatalf("Failed to get manifest file stats %s, %v\n", manifestPath, err) - } - g3i.Logger().Println("Reading manifest...") - manifestFileSize := manifestFileStat.Size() - manifestProgress := mpb.New(mpb.WithOutput(os.Stdout)) - manifestFileBar := manifestProgress.AddBar(manifestFileSize, - mpb.PrependDecorators( - decor.Name("Manifest "), - decor.CountersKibiByte("% .1f / % .1f"), - ), - mpb.AppendDecorators(decor.Percentage()), - ) - - manifestFileReader := manifestFileBar.ProxyReader(manifestFile) - - manifestBytes, err := io.ReadAll(manifestFileReader) - if err != nil { - g3i.Logger().Fatalf("Failed reading manifest %s, %v\n", manifestPath, err) - } - manifestProgress.Wait() - - var objects []ManifestObject - err = json.Unmarshal(manifestBytes, &objects) - if err != nil { - g3i.Logger().Fatalf("Error has occurred during unmarshalling manifest object: %v\n", err) - } - - err = downloadFile(g3i, objects, downloadPath, filenameFormat, rename, noPrompt, protocol, numParallel, skipCompleted) - if err != nil { - g3i.Logger().Fatal(err.Error()) - } - }, - } - - downloadMultipleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") - downloadMultipleCmd.MarkFlagRequired("profile") //nolint:errcheck - downloadMultipleCmd.Flags().StringVar(&manifestPath, "manifest", "", "The manifest file to read from. A valid manifest can be acquired by using the \"Download Manifest\" button in Data Explorer from a data common's portal") - downloadMultipleCmd.MarkFlagRequired("manifest") //nolint:errcheck - downloadMultipleCmd.Flags().StringVar(&downloadPath, "download-path", ".", "The directory in which to store the downloaded files") - downloadMultipleCmd.Flags().StringVar(&filenameFormat, "filename-format", "original", "The format of filename to be used, including \"original\", \"guid\" and \"combined\"") - downloadMultipleCmd.Flags().BoolVar(&rename, "rename", false, "Only useful when \"--filename-format=original\", will rename file by appending a counter value to its filename if set to true, otherwise the same filename will be used") - downloadMultipleCmd.Flags().BoolVar(&noPrompt, "no-prompt", false, "If set to true, will not display user prompt message for confirmation") - downloadMultipleCmd.Flags().StringVar(&protocol, "protocol", "", "Specify the preferred protocol with --protocol=s3") - downloadMultipleCmd.Flags().IntVar(&numParallel, "numparallel", 1, "Number of downloads to run in parallel") - downloadMultipleCmd.Flags().BoolVar(&skipCompleted, "skip-completed", false, "If set to true, will check for filename and size before download and skip any files in \"download-path\" that matches both") - RootCmd.AddCommand(downloadMultipleCmd) -} diff --git a/client/g3cmd/download-single.go b/client/g3cmd/download-single.go deleted file mode 100644 index 6038f23..0000000 --- a/client/g3cmd/download-single.go +++ /dev/null @@ -1,60 +0,0 @@ -package g3cmd - -import ( - "context" - "log" - - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - "github.com/spf13/cobra" -) - -func init() { - var guid string - var downloadPath string - var protocol string - var filenameFormat string - var rename bool - var noPrompt bool - var skipCompleted bool - var profile string - - var downloadSingleCmd = &cobra.Command{ - Use: "download-single", - Short: "Download a single file from a GUID", - Long: `Gets a presigned URL for a file from a GUID and then downloads the specified file.`, - Example: `./data-client download-single --profile= --guid=206dfaa6-bcf1-4bc9-b2d0-77179f0f48fc`, - Run: func(cmd *cobra.Command, args []string) { - // don't initialize transmission logs for non-uploading related commands - - logger, logCloser := logs.New(profile, logs.WithConsole(), logs.WithFailedLog(), logs.WithSucceededLog(), logs.WithScoreboard()) - defer logCloser() - - g3I, err := client.NewGen3Interface(context.Background(), profile, logger) - if err != nil { - log.Fatalf("Failed to parse config on profile %s, %v", profile, err) - } - - obj := ManifestObject{ - ObjectID: guid, - } - objects := []ManifestObject{obj} - err = downloadFile(g3I, objects, downloadPath, filenameFormat, rename, noPrompt, protocol, 1, skipCompleted) - if err != nil { - g3I.Logger().Println(err.Error()) - } - }, - } - - downloadSingleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") - downloadSingleCmd.MarkFlagRequired("profile") //nolint:errcheck - downloadSingleCmd.Flags().StringVar(&guid, "guid", "", "Specify the guid for the data you would like to work with") - downloadSingleCmd.MarkFlagRequired("guid") //nolint:errcheck - downloadSingleCmd.Flags().StringVar(&downloadPath, "download-path", ".", "The directory in which to store the downloaded files") - downloadSingleCmd.Flags().StringVar(&filenameFormat, "filename-format", "original", "The format of filename to be used, including \"original\", \"guid\" and \"combined\"") - downloadSingleCmd.Flags().BoolVar(&rename, "rename", false, "Only useful when \"--filename-format=original\", will rename file by appending a counter value to its filename if set to true, otherwise the same filename will be used") - downloadSingleCmd.Flags().BoolVar(&noPrompt, "no-prompt", false, "If set to true, will not display user prompt message for confirmation") - downloadSingleCmd.Flags().StringVar(&protocol, "protocol", "", "Specify the preferred protocol with --protocol=gs") - downloadSingleCmd.Flags().BoolVar(&skipCompleted, "skip-completed", false, "If set to true, will check for filename and size before download and skip any files in \"download-path\" that matches both") - RootCmd.AddCommand(downloadSingleCmd) -} diff --git a/client/g3cmd/generate-tsv.go b/client/g3cmd/generate-tsv.go deleted file mode 100644 index 9abff77..0000000 --- a/client/g3cmd/generate-tsv.go +++ /dev/null @@ -1,17 +0,0 @@ -package g3cmd - -import ( - "github.com/spf13/cobra" -) - -func init() { - var generateTSVCmd = &cobra.Command{ - Use: "generate-tsv", - Short: "Generate a file upload tsv from a template", - Long: `Fills in a Gen3 data file template with information from a directory of files.`, - Deprecated: "please use an older version of data-client", - Run: func(cmd *cobra.Command, args []string) {}, - } - - RootCmd.AddCommand(generateTSVCmd) -} diff --git a/client/g3cmd/gitversion.go b/client/g3cmd/gitversion.go deleted file mode 100644 index cb3a308..0000000 --- a/client/g3cmd/gitversion.go +++ /dev/null @@ -1,6 +0,0 @@ -package g3cmd - -var ( - gitcommit = "N/A" - gitversion = "2023.11" -) diff --git a/client/g3cmd/retry-upload.go b/client/g3cmd/retry-upload.go deleted file mode 100644 index edd5c52..0000000 --- a/client/g3cmd/retry-upload.go +++ /dev/null @@ -1,215 +0,0 @@ -package g3cmd - -import ( - "context" - "os" - "path/filepath" - "time" - - "github.com/calypr/data-client/client/common" - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - - "github.com/spf13/cobra" -) - -func handleFailedRetry(g3i client.Gen3Interface, ro common.RetryObject, retryObjCh chan common.RetryObject, err error) { - logger := g3i.Logger() - - // Record failure in JSON log - logger.Failed(ro.FilePath, ro.Filename, ro.FileMetadata, ro.GUID, ro.RetryCount, ro.Multipart) - - if err != nil { - logger.Println("Error:", err) - } - - if ro.RetryCount < MaxRetryCount { - retryObjCh <- ro - return - } - - // Max retries reached — clean up - if ro.GUID != "" { - if msg, err := DeleteRecord(g3i, ro.GUID); err == nil { - logger.Println(msg) - } else { - logger.Println("Cleanup failed:", err) - } - } - - // Final failure - sb, err := logs.FromSBContext(context.Background()) - if err != nil { - logger.Println(err) - } - sb.IncrementSB(MaxRetryCount + 1) - - if len(retryObjCh) == 0 { - close(retryObjCh) - logger.Println("Retry channel closed — all done") - } -} - -func retryUpload(g3i client.Gen3Interface, failedLogMap map[string]common.RetryObject) { - logger := g3i.Logger() - - sb, err := logs.FromSBContext(context.Background()) - if err != nil { - logger.Println(err) - } - - if len(failedLogMap) == 0 { - logger.Println("No failed files to retry.") - return - } - - logger.Println("Starting retry-upload...") - retryObjCh := make(chan common.RetryObject, len(failedLogMap)) - - // Load failed entries (skip already succeeded ones) - for _, ro := range failedLogMap { - // Simple check: if succeeded log exists and contains this path, skip - if common.AlreadySucceededFromFile(ro.FilePath) { - logger.Printf("Already uploaded: %s — skipping\n", ro.FilePath) - continue - } - retryObjCh <- ro - } - - if len(retryObjCh) == 0 { - logger.Println("All failed files were already successfully uploaded in a previous run.") - return - } - - for ro := range retryObjCh { - ro.RetryCount++ - logger.Printf("#%d retry — %s\n", ro.RetryCount, ro.FilePath) - logger.Printf("Waiting %.0f seconds...\n", GetWaitTime(ro.RetryCount).Seconds()) - time.Sleep(GetWaitTime(ro.RetryCount)) - - // Optional: delete old record - if ro.GUID != "" { - if msg, err := DeleteRecord(g3i, ro.GUID); err == nil { - logger.Println(msg) - } - } - - // Fix missing filename if needed - if ro.Filename == "" { - absPath, _ := common.GetAbsolutePath(ro.FilePath) - ro.Filename = filepath.Base(absPath) - } - - var err error - if ro.Multipart { - // Multipart retry - req := common.FileUploadRequestObject{ - FilePath: ro.FilePath, - Filename: ro.Filename, - GUID: ro.GUID, - } - err = MultipartUpload(context.Background(), g3i, req, ro.Bucket, true) - if err == nil { - logger.Succeeded(ro.FilePath, req.GUID) - sb.IncrementSB(ro.RetryCount - 1) // success on this retry - continue - } - } else { - // Single-part retry - var presignedURL, guid string - presignedURL, guid, err = GeneratePresignedURL(g3i, ro.Filename, ro.FileMetadata, ro.Bucket) - if err != nil { - handleFailedRetry(g3i, ro, retryObjCh, err) - continue - } - - file, err := os.Open(ro.FilePath) - if err != nil { - handleFailedRetry(g3i, ro, retryObjCh, err) - continue - } - stat, _ := file.Stat() - file.Close() - - if stat.Size() > FileSizeLimit { - ro.Multipart = true - retryObjCh <- ro - continue - } - - fur := common.FileUploadRequestObject{ - FilePath: ro.FilePath, - Filename: ro.Filename, - FileMetadata: ro.FileMetadata, - GUID: guid, - PresignedURL: presignedURL, - } - - fur, err = GenerateUploadRequest(g3i, fur, nil, nil) - if err != nil { - handleFailedRetry(g3i, ro, retryObjCh, err) - continue - } - - err = uploadFile(g3i, fur, ro.RetryCount) - if err != nil { - handleFailedRetry(g3i, ro, retryObjCh, err) - continue - } - - logger.Succeeded(ro.FilePath, fur.GUID) - sb.IncrementSB(ro.RetryCount - 1) - } - - if len(retryObjCh) == 0 { - close(retryObjCh) - } - } -} - -func init() { - var failedLogPath, profile string - - var retryUploadCmd = &cobra.Command{ - Use: "retry-upload", - Short: "Retry failed uploads from a failed_log.json", - Long: `Re-uploads files listed in a failed log using exponential backoff and progress bars.`, - Example: `./data-client retry-upload --profile=myprofile --failed-log-path=/path/to/failed_log.json`, - Run: func(cmd *cobra.Command, args []string) { - Logger, closer := logs.New(profile, - logs.WithConsole(), - logs.WithMessageFile(), - logs.WithFailedLog(), - logs.WithSucceededLog(), - ) - defer closer() - - g3, err := client.NewGen3Interface(context.Background(), profile, Logger) - if err != nil { - Logger.Fatalf("Failed to initialize client: %v", err) - } - - logger := g3.Logger() - - // Create scoreboard with our logger injected - sb := logs.NewSB(MaxRetryCount, logger) - - // Load failed log - failedMap, err := common.LoadFailedLog(failedLogPath) - if err != nil { - logger.Fatalf("Cannot read failed log: %v", err) - } - - retryUpload(g3, failedMap) - sb.PrintSB() - }, - } - - retryUploadCmd.Flags().StringVar(&profile, "profile", "", "Profile to use") - retryUploadCmd.MarkFlagRequired("profile") - - retryUploadCmd.Flags().StringVar(&failedLogPath, "failed-log-path", "", "Path to failed_log.json") - retryUploadCmd.MarkFlagRequired("failed-log-path") - - RootCmd.AddCommand(retryUploadCmd) -} diff --git a/client/g3cmd/root.go b/client/g3cmd/root.go deleted file mode 100644 index 8bc7ab9..0000000 --- a/client/g3cmd/root.go +++ /dev/null @@ -1,124 +0,0 @@ -package g3cmd - -import ( - "encoding/json" - "net/http" - "os" - "strconv" - "time" - - "github.com/calypr/data-client/client/jwt" - "github.com/calypr/data-client/client/logs" - "github.com/spf13/cobra" - "golang.org/x/mod/semver" -) - -var profile string - -// Package-level variable to hold the closer function -// (Assuming logs.Closer is a type that can hold a function, like func() error) -var logCloser func() - -// Or just: -// var logCloser io.Closer // if closer implements io.Closer - -// RootCmd represents the base command when called without any subcommands -var RootCmd = &cobra.Command{ - Use: "data-client", - Short: "Use the data-client to interact with a Gen3 Data Commons", - Long: "Gen3 Client for downloading, uploading and submitting data to data commons.\ndata-client version: " + gitversion + ", commit: " + gitcommit, - Version: gitversion, -} - -// Execute adds all child commands to the root command sets flags appropriately -// This is called by main.main(). It only needs to happen once to the rootCmd. -func Execute() { - if logCloser != nil { - defer func() { - logCloser() - }() - } - - if err := RootCmd.Execute(); err != nil { - os.Stderr.WriteString("Error: " + err.Error() + "\n") - os.Exit(1) - } -} - -func init() { - cobra.OnInitialize(initConfig) - - // Define flags and configuration settings. - RootCmd.PersistentFlags().StringVar(&profile, "profile", "", "Specify profile to use") - _ = RootCmd.MarkFlagRequired("profile") -} - -type GitHubRelease struct { - TagName string `json:"tag_name"` -} - -func initConfig() { - // The logger is needed throughout the application, so we don't store it here, - // but the closer must be stored. - logger, closer := logs.New(profile, - logs.WithConsole(), - logs.WithMessageFile(), - logs.WithFailedLog(), - logs.WithSucceededLog(), - ) - - // 2. ASSIGN CLOSER TO PACKAGE VARIABLE - logCloser = closer - - // The rest of the function remains the same, except for removing the 'defer resp.Body.Close()' - // from the initConfig body, as that was unrelated to the logs closer. - // The rest of your original logic follows... - - conf := jwt.Configure{} - // init local config file - err := conf.InitConfigFile() - if err != nil { - logger.Fatal("Error occurred when trying to init config file: " + err.Error()) - } - - // version checker - if os.Getenv("GEN3_CLIENT_VERSION_CHECK") != "false" && - gitversion != "" && gitversion != "N/A" { - - const ( - owner = "uc-cdis" - repository = "cdis-data-client" - // The official GitHub API endpoint for the latest release - apiURL = "https://api.github.com/repos/" + owner + "/" + repository + "/releases/latest" - ) - - client := http.Client{Timeout: 5 * time.Second} - resp, err := client.Get(apiURL) - if err != nil { - logger.Println("Error occurred when fetching latest version (HTTP request failed): " + err.Error()) - // Continue execution, as version check failure is non-fatal - return - } - - // This defer is correct and should remain, as it cleans up the HTTP response body - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - logger.Println("Error occurred when fetching latest version (GitHub API returned status " + strconv.Itoa(resp.StatusCode) + ")") - return - } - - var release GitHubRelease - if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { - logger.Println("Error occurred when decoding latest version response: " + err.Error()) - return - } - - latestVersionTag := release.TagName - - if semver.Compare(gitversion, latestVersionTag) < 0 { - logger.Println("A new version of data-client is available! The latest version is " + latestVersionTag + ". You are using version " + gitversion) - logger.Println("Please download the latest data-client release from https://github.com/uc-cdis/cdis-data-client/releases/latest") - } - } -} diff --git a/client/g3cmd/upload-multipart.go b/client/g3cmd/upload-multipart.go deleted file mode 100644 index fa6d26f..0000000 --- a/client/g3cmd/upload-multipart.go +++ /dev/null @@ -1,299 +0,0 @@ -package g3cmd - -import ( - "bytes" - "context" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "sort" - "strings" - "sync" - "time" - - "github.com/calypr/data-client/client/common" - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - "github.com/spf13/cobra" - "github.com/vbauerster/mpb/v8" - "github.com/vbauerster/mpb/v8/decor" -) - -const ( - minChunkSize = 5 * 1024 * 1024 // S3 minimum part size - maxMultipartParts = 10000 - maxConcurrentUploads = 10 - maxRetries = 5 -) - -func NewUploadMultipartCmd() *cobra.Command { - var ( - filePath string - guid string - bucketName string - ) - - cmd := &cobra.Command{ - Use: "upload-multipart", - Short: "Upload a single file using multipart upload", - Long: `Uploads a large file to object storage using multipart upload. -This method is resilient to network interruptions and supports resume capability.`, - Example: `./data-client upload-multipart --profile=myprofile --file-path=./large.bam -./data-client upload-multipart --profile=myprofile --file-path=./data.bam --guid=existing-guid`, - RunE: func(cmd *cobra.Command, args []string) error { - profile, _ := cmd.Flags().GetString("profile") - - return UploadSingleFile(profile, bucketName, filePath, guid) - }, - } - - cmd.Flags().StringVar(&filePath, "file-path", "", "Path to the file to upload") - cmd.Flags().StringVar(&guid, "guid", "", "Optional existing GUID (otherwise generated)") - cmd.Flags().StringVar(&bucketName, "bucket", "", "Target bucket (defaults to configured DATA_UPLOAD_BUCKET)") - - _ = cmd.MarkFlagRequired("profile") - _ = cmd.MarkFlagRequired("file-path") - - return cmd -} - -func UploadSingleFile(profile, bucket, filePath, guid string) error { - - logger, closer := logs.New(profile, logs.WithSucceededLog(), logs.WithFailedLog(), logs.WithScoreboard()) - defer closer() - g3, err := client.NewGen3Interface( - context.Background(), - profile, - logger, - ) - if err != nil { - return fmt.Errorf("failed to initialize Gen3 interface: %w", err) - } - - absPath, err := common.GetAbsolutePath(filePath) - if err != nil { - return fmt.Errorf("invalid file path: %w", err) - } - - fileInfo := common.FileUploadRequestObject{ - FilePath: absPath, - Filename: filepath.Base(absPath), - GUID: guid, - FileMetadata: common.FileMetadata{}, - } - - return MultipartUpload(context.TODO(), g3, fileInfo, bucket, true) -} - -// MultipartUpload is now clean, context-aware, and uses modern progress bars -func MultipartUpload(ctx context.Context, g3 client.Gen3Interface, req common.FileUploadRequestObject, bucketName string, showProgress bool) error { - g3.Logger().Printf("File Upload Request: %#v\n", req) - - file, err := os.Open(req.FilePath) - if err != nil { - return fmt.Errorf("cannot open file %s: %w", req.FilePath, err) - } - defer file.Close() - - stat, err := file.Stat() - if err != nil { - return fmt.Errorf("cannot stat file: %w", err) - } - - g3.Logger().Printf("File Name: '%s', File Size: '%d'\n", stat.Name(), stat.Size()) - - if stat.Size() == 0 { - return fmt.Errorf("file is empty: %s", req.Filename) - } - - // Initialize multipart upload - uploadID, finalGUID, err := InitMultipartUpload(g3, req, bucketName) - if err != nil { - return fmt.Errorf("failed to initiate multipart upload: %w", err) - } - req.GUID = finalGUID // update with server-provided GUID - - key := finalGUID + "/" + req.Filename - chunkSize := optimalChunkSize(stat.Size()) - - numChunks := int((stat.Size() + chunkSize - 1) / chunkSize) - parts := make([]MultipartPartObject, 0, numChunks) - - // Progress bar setup (modern mpb) - var p *mpb.Progress - var bar *mpb.Bar - if showProgress { - p = mpb.New(mpb.WithOutput(os.Stdout)) - bar = p.AddBar(stat.Size(), - mpb.PrependDecorators( - decor.Name(req.Filename+" "), - decor.CountersKibiByte("%.1f / %.1f"), - ), - mpb.AppendDecorators( - decor.Percentage(), - decor.AverageSpeed(decor.SizeB1024(0), " % .1f"), - ), - ) - } - - // Channel for chunk indices - chunks := make(chan int, numChunks) - for i := 1; i <= numChunks; i++ { - chunks <- i - } - close(chunks) - - var ( - wg sync.WaitGroup - mu sync.Mutex - uploadErrors []error - ) - - worker := func() { - defer wg.Done() - buf := make([]byte, chunkSize) - - for partNum := range chunks { - offset := int64(partNum-1) * chunkSize - end := offset + chunkSize - end = min(end, stat.Size()) - size := end - offset - - // Read chunk - if _, err := file.Seek(offset, io.SeekStart); err != nil { - mu.Lock() - uploadErrors = append(uploadErrors, fmt.Errorf("seek failed for part %d: %w", partNum, err)) - mu.Unlock() - continue - } - n, err := io.ReadFull(file, buf[:size]) - if err != nil && err != io.ErrUnexpectedEOF { - mu.Lock() - uploadErrors = append(uploadErrors, fmt.Errorf("read failed for part %d: %w", partNum, err)) - mu.Unlock() - continue - } - - reader := bytes.NewReader(buf[:n]) - - // Get presigned URL + upload with retry - var etag string - if err := retryWithBackoff(ctx, maxRetries, func() error { - url, err := GenerateMultipartPresignedURL(g3, key, uploadID, partNum, bucketName) - if err != nil { - return err - } - - return uploadPart(url, reader, &etag) - }); err != nil { - mu.Lock() - uploadErrors = append(uploadErrors, fmt.Errorf("part %d failed after retries: %w", partNum, err)) - mu.Unlock() - continue - } - - // Success - mu.Lock() - etag = strings.Trim(etag, `"`) - parts = append(parts, MultipartPartObject{PartNumber: partNum, ETag: etag}) - g3.Logger().Printf("Appended part %d with ETag %s\n", partNum, etag) - if bar != nil { - bar.IncrBy(n) - } - mu.Unlock() - } - } - - // Launch workers - for range maxConcurrentUploads { - wg.Add(1) - go worker() - } - wg.Wait() - - if p != nil { - p.Wait() - } - - if len(uploadErrors) > 0 { - return fmt.Errorf("multipart upload failed: %d parts failed: %v", len(uploadErrors), uploadErrors) - } - - // Sort parts by PartNumber - sort.Slice(parts, func(i, j int) bool { - return parts[i].PartNumber < parts[j].PartNumber - }) - - g3.Logger().Printf("Completing multipart upload with %d parts for file %s\n", len(parts), req.Filename) - for _, part := range parts { - g3.Logger().Printf(" Part %d: ETag=%s\n", part.PartNumber, part.ETag) - } - - if err := CompleteMultipartUpload(g3, key, uploadID, parts, bucketName); err != nil { - return fmt.Errorf("failed to complete multipart upload: %w", err) - } - - g3.Logger().Printf("Successfully uploaded %s as %s (%d)", req.Filename, finalGUID, stat.Size()) - return nil -} - -// Helper: exponential backoff retry -func retryWithBackoff(ctx context.Context, attempts int, fn func() error) error { - var err error - for i := range attempts { - if err = fn(); err == nil { - return nil - } - if i == attempts-1 { - break - } - select { - case <-ctx.Done(): - return ctx.Err() - case <-time.After(backoffDuration(i)): - } - } - return fmt.Errorf("after %d attempts: %w", attempts, err) -} - -func backoffDuration(attempt int) time.Duration { - return min(time.Duration(1< --manifest= --upload-path= --bucket= --force-multipart= --include-subdirname= --batch=`, - Run: func(cmd *cobra.Command, args []string) { - fmt.Printf("Notice: this is the upload method which requires the user to provide GUIDs. In this method files will be uploaded to specified GUIDs.\nIf your intention is to upload files without pre-existing GUIDs, consider to use \"./data-client upload\" instead.\n\n") - - logger, closer := logs.New(profile, logs.WithSucceededLog(), logs.WithFailedLog(), logs.WithScoreboard()) - defer closer() - - // Instantiate interface to Gen3 - g3i, err := client.NewGen3Interface(context.Background(), profile, logger) - if err != nil { - g3i.Logger().Fatalf("Failed to parse config on profile %s, %v", profile, err) - } - - host, err := g3i.GetHost() - if err != nil { - g3i.Logger().Fatal("Error occurred during parsing config file for hostname: " + err.Error()) - } - dataExplorerURL := host.Scheme + "://" + host.Host + "/explorer" - - var objects []ManifestObject - - manifestFile, err := os.Open(manifestPath) - if err != nil { - g3i.Logger().Println("Failed to open manifest file") - g3i.Logger().Fatal("A valid manifest can be acquired by using the \"Download Manifest\" button on " + dataExplorerURL) - } - defer manifestFile.Close() - switch { - case strings.EqualFold(filepath.Ext(manifestPath), ".json"): - manifestBytes, err := os.ReadFile(manifestPath) - if err != nil { - g3i.Logger().Printf("Failed reading manifest %s, %v\n", manifestPath, err) - g3i.Logger().Fatal("A valid manifest can be acquired by using the \"Download Manifest\" button on " + dataExplorerURL) - } - err = json.Unmarshal(manifestBytes, &objects) - if err != nil { - g3i.Logger().Fatal("Unmarshalling manifest failed with error: " + err.Error()) - } - default: - g3i.Logger().Println("Unsupported manifast format") - g3i.Logger().Fatal("A valid manifest can be acquired by using the \"Download Manifest\" button on " + dataExplorerURL) - } - - absUploadPath, err := common.GetAbsolutePath(uploadPath) - if err != nil { - g3i.Logger().Fatalf("Error when parsing file paths: %s", err.Error()) - } - - // Create unified upload request objects - uploadRequestObjects := make([]common.FileUploadRequestObject, 0, len(objects)) - - for _, object := range objects { - var localFilePath string - // Determine the local file path - if object.Filename != "" { - // conform to fence naming convention - localFilePath, err = getFullFilePath(absUploadPath, object.Filename) - } else { - // Otherwise, here we are assuming the local filename will be the same as GUID - localFilePath, err = getFullFilePath(absUploadPath, object.ObjectID) - } - - if err != nil { - g3i.Logger().Println(err.Error()) - continue - } - - fileInfo, err := ProcessFilename(g3i.Logger(), absUploadPath, localFilePath, object.ObjectID, includeSubDirName, false) - if err != nil { - g3i.Logger().Println("Process filename error: " + err.Error()) - g3i.Logger().Failed(localFilePath, filepath.Base(localFilePath), common.FileMetadata{}, object.ObjectID, 0, false) - continue - } - - // Convert FileInfo to the unified common.FileUploadRequestObject - furObject := common.FileUploadRequestObject{ - FilePath: fileInfo.FilePath, - Filename: fileInfo.Filename, - FileMetadata: fileInfo.FileMetadata, - GUID: fileInfo.GUID, - } - uploadRequestObjects = append(uploadRequestObjects, furObject) - } - - // Separate into single-part and multipart objects - singlePartObjects, multipartObjects := separateSingleAndMultipartUploads(g3i, uploadRequestObjects, forceMultipart) - // Pass the unified objects to the upload handlers - if batch { - workers, respCh, errCh, batchFURObjects := initBatchUploadChannels(numParallel, len(singlePartObjects)) - for i, furObject := range singlePartObjects { - // FileInfo processing and path normalization are already done, so we use the object directly - if len(batchFURObjects) < workers { - batchFURObjects = append(batchFURObjects, furObject) - } else { - batchUpload(g3i, batchFURObjects, workers, respCh, errCh, bucketName) - batchFURObjects = []common.FileUploadRequestObject{furObject} - } - if !forceMultipart && i == len(singlePartObjects)-1 && len(batchFURObjects) > 0 { // upload remainders - batchUpload(g3i, batchFURObjects, workers, respCh, errCh, bucketName) - } - } - } else { - processSingleUploads(g3i, singlePartObjects, bucketName, includeSubDirName, absUploadPath) // Assuming updated - } - - if len(multipartObjects) > 0 { - err := processMultipartUpload(g3i, multipartObjects, bucketName, includeSubDirName, absUploadPath) - if err != nil { - g3i.Logger().Fatal(err.Error()) - } - } - - if len(g3i.Logger().GetSucceededLogMap()) == 0 { - retryUpload(g3i, g3i.Logger().GetFailedLogMap()) - } - - g3i.Logger().Scoreboard().PrintSB() - }, - } - - uploadMultipleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") - uploadMultipleCmd.MarkFlagRequired("profile") //nolint:errcheck - uploadMultipleCmd.Flags().StringVar(&manifestPath, "manifest", "", "The manifest file to read from. A valid manifest can be acquired by using the \"Download Manifest\" button in Data Explorer for Common portal") - uploadMultipleCmd.MarkFlagRequired("manifest") //nolint:errcheck - uploadMultipleCmd.Flags().StringVar(&uploadPath, "upload-path", "", "The directory in which contains files to be uploaded") - uploadMultipleCmd.MarkFlagRequired("upload-path") //nolint:errcheck - uploadMultipleCmd.Flags().BoolVar(&batch, "batch", true, "Upload in parallel") - uploadMultipleCmd.Flags().IntVar(&numParallel, "numparallel", 3, "Number of uploads to run in parallel") - uploadMultipleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") - uploadMultipleCmd.Flags().BoolVar(&forceMultipart, "force-multipart", false, "Force to use multipart upload when possible (file size >= 5MB)") - uploadMultipleCmd.Flags().BoolVar(&includeSubDirName, "include-subdirname", true, "Include subdirectory names in file name") - RootCmd.AddCommand(uploadMultipleCmd) -} - -func processSingleUploads(g3i client.Gen3Interface, singleObjects []common.FileUploadRequestObject, bucketName string, includeSubDirName bool, uploadPath string) { - for _, furObject := range singleObjects { - filePath := furObject.FilePath - file, err := os.Open(filePath) - if err != nil { - g3i.Logger().Println("File open error: " + err.Error()) - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - continue - } - startSingleFileUpload(g3i, furObject, file, bucketName) - file.Close() - } -} - -func startSingleFileUpload(g3i client.Gen3Interface, furObject common.FileUploadRequestObject, file *os.File, bucketName string) { - - fi, err := file.Stat() - if err != nil { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - g3i.Logger().Println("File stat error for file" + fi.Name() + ", file may be missing or unreadable because of permissions.\n") - return - } - - respURL, guid, err := GeneratePresignedURL(g3i, furObject.Filename, furObject.FileMetadata, bucketName) - if err != nil { - g3i.Logger().Println(err.Error()) - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, guid, 0, false) - return - } - furObject.GUID = guid - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - furObject.PresignedURL = respURL - - furObject, err = GenerateUploadRequest(g3i, furObject, file, nil) - if err != nil { - file.Close() - g3i.Logger().Printf("Error occurred during request generation: %s\n", err.Error()) - return - } - - err = uploadFile(g3i, furObject, 0) - if err != nil { - g3i.Logger().Println(err.Error()) - } else { - g3i.Logger().Scoreboard().IncrementSB(0) - } - - file.Close() -} - -func processMultipartUpload(g3i client.Gen3Interface, multipartObjects []common.FileUploadRequestObject, bucketName string, includeSubDirName bool, uploadPath string) error { - cred := g3i.GetCredential() - if cred.UseShepherd == "true" || - cred.UseShepherd == "" && common.DefaultUseShepherd == true { - return fmt.Errorf("error: Shepherd currently does not support multipart uploads. For the moment, please disable Shepherd with\n $ data-client configure --profile=%v --use-shepherd=false\nand try again", cred.Profile) - } - g3i.Logger().Println("Multipart uploading...") - - for _, furObject := range multipartObjects { - // No more redundant ProcessFilename call! - // Pass the complete FileUploadRequestObject to the streamlined multipartUpload. - // Enable progress bar for batch uploads (interactive CLI use) - err := MultipartUpload(context.Background(), g3i, furObject, bucketName, true) - - if err != nil { - g3i.Logger().Println(err.Error()) - } else { - g3i.Logger().Scoreboard().IncrementSB(0) - } - } - return nil -} diff --git a/client/g3cmd/upload-single.go b/client/g3cmd/upload-single.go deleted file mode 100644 index 8395370..0000000 --- a/client/g3cmd/upload-single.go +++ /dev/null @@ -1,122 +0,0 @@ -package g3cmd - -// Deprecated: Use upload instead. -import ( - "context" - "errors" - "fmt" - "log" - "os" - "path/filepath" - - "github.com/calypr/data-client/client/common" - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - "github.com/spf13/cobra" -) - -func init() { - var guid string - var filePath string - var bucketName string - - var uploadSingleCmd = &cobra.Command{ - Use: "upload-single", - Short: "Upload a single file to a GUID", - Long: `Gets a presigned URL for which to upload a file associated with a GUID and then uploads the specified file.`, - Example: `./data-client upload-single --profile= --guid=f6923cf3-xxxx-xxxx-xxxx-14ab3f84f9d6 --file=`, - Run: func(cmd *cobra.Command, args []string) { - // initialize transmission logs - err := UploadSingle(profile, guid, filePath, bucketName, true) - if err != nil { - log.Fatalln(err.Error()) - } - }, - } - uploadSingleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") - uploadSingleCmd.MarkFlagRequired("profile") //nolint:errcheck - uploadSingleCmd.Flags().StringVar(&guid, "guid", "", "Specify the guid for the data you would like to work with") - uploadSingleCmd.MarkFlagRequired("guid") //nolint:errcheck - uploadSingleCmd.Flags().StringVar(&filePath, "file", "", "Specify file to upload to with --file=~/path/to/file") - uploadSingleCmd.MarkFlagRequired("file") //nolint:errcheck - uploadSingleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") - RootCmd.AddCommand(uploadSingleCmd) -} - -func UploadSingle(profile string, guid string, filePath string, bucketName string, enableLogs bool) error { - - logger, closer := logs.New(profile, logs.WithSucceededLog(), logs.WithFailedLog()) - if enableLogs { - logger, closer = logs.New( - profile, - logs.WithSucceededLog(), - logs.WithFailedLog(), - logs.WithScoreboard(), - logs.WithConsole(), - ) - } - defer closer() - - // Instantiate interface to Gen3 - g3i, err := client.NewGen3Interface( - context.Background(), - profile, - logger, - ) - if err != nil { - return fmt.Errorf("failed to parse config on profile %s: %w", profile, err) - } - - filePaths, err := common.ParseFilePaths(filePath, false) - if len(filePaths) > 1 { - return errors.New("more than 1 file location has been found. Do not use \"*\" in file path or provide a folder as file path") - } - if err != nil { - return errors.New("file path parsing error: " + err.Error()) - } - if len(filePaths) == 1 { - filePath = filePaths[0] - } - filename := filepath.Base(filePath) - if _, err := os.Stat(filePath); os.IsNotExist(err) { - g3i.Logger().Failed(filePath, filename, common.FileMetadata{}, "", 0, false) - sb := g3i.Logger().Scoreboard() - sb.IncrementSB(len(sb.Counts)) - sb.PrintSB() - return fmt.Errorf("[ERROR] The file you specified \"%s\" does not exist locally\n", filePath) - } - - file, err := os.Open(filePath) - if err != nil { - sb := g3i.Logger().Scoreboard() - sb.IncrementSB(len(sb.Counts)) - sb.PrintSB() - g3i.Logger().Failed(filePath, filename, common.FileMetadata{}, "", 0, false) - g3i.Logger().Println("File open error: " + err.Error()) - return fmt.Errorf("[ERROR] when opening file path %s, an error occurred: %s\n", filePath, err.Error()) - } - defer file.Close() - - furObject := common.FileUploadRequestObject{FilePath: filePath, Filename: filename, GUID: guid, Bucket: bucketName} - - furObject, err = GenerateUploadRequest(g3i, furObject, file, nil) - if err != nil { - file.Close() - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, common.FileMetadata{}, furObject.GUID, 0, false) - sb := g3i.Logger().Scoreboard() - sb.IncrementSB(len(sb.Counts)) - sb.PrintSB() - g3i.Logger().Fatalf("Error occurred during request generation: %s", err.Error()) - return fmt.Errorf("[ERROR] Error occurred during request generation for file %s: %s\n", filePath, err.Error()) - } - err = uploadFile(g3i, furObject, 0) - if err != nil { - sb := g3i.Logger().Scoreboard() - sb.IncrementSB(len(sb.Counts)) - return fmt.Errorf("[ERROR] Error uploading file %s: %s\n", filePath, err.Error()) - } else { - g3i.Logger().Scoreboard().IncrementSB(0) - } - g3i.Logger().Scoreboard().PrintSB() - return nil -} diff --git a/client/g3cmd/upload.go b/client/g3cmd/upload.go deleted file mode 100644 index 2e50a87..0000000 --- a/client/g3cmd/upload.go +++ /dev/null @@ -1,155 +0,0 @@ -package g3cmd - -import ( - "context" - "log" - "os" - "path/filepath" - - "github.com/calypr/data-client/client/common" - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - "github.com/spf13/cobra" -) - -func init() { - var bucketName string - var includeSubDirName bool - var uploadPath string - var batch bool - var forceMultipart bool - var numParallel int - var hasMetadata bool - var uploadCmd = &cobra.Command{ - Use: "upload", - Short: "Upload file(s) to object storage.", - Long: `Gets a presigned URL for each file and then uploads the specified file(s).`, - Example: "For uploading a single file:\n./data-client upload --profile= --upload-path=\n" + - "For uploading all files within an folder:\n./data-client upload --profile= --upload-path=\n" + - "Can also support regex such as:\n./data-client upload --profile= --upload-path=\n" + - "Or:\n./data-client upload --profile= --upload-path=\n" + - "This command can also upload file metadata using the --metadata flag. If the --metadata flag is passed, the data-client will look for a file called [filename]_metadata.json in the same folder, which contains the metadata to upload.\n" + - "For example, if uploading the file `folder/my_file.bam`, the data-client will look for a metadata file at `folder/my_file_metadata.json`.\n" + - "For the format of the metadata files, see the README.", - Run: func(cmd *cobra.Command, args []string) { - - Logger, logCloser := logs.New(profile, logs.WithSucceededLog(), logs.WithScoreboard(), logs.WithFailedLog()) - defer logCloser() - // Instantiate interface to Gen3 - g3i, err := client.NewGen3Interface(context.Background(), profile, Logger) - if err != nil { - log.Fatalf("Failed to parse config on profile %s, %v", profile, err) - } - - logger := g3i.Logger() - if hasMetadata { - hasShepherd, err := g3i.CheckForShepherdAPI() - if err != nil { - logger.Printf("WARNING: Error when checking for Shepherd API: %v", err) - } else { - if !hasShepherd { - logger.Fatalf("ERROR: Metadata upload (`--metadata`) is not supported in the environment you are uploading to. Double check that you are uploading to the right profile.") - } - } - } - - uploadPath, _ = common.GetAbsolutePath(uploadPath) - filePaths, err := common.ParseFilePaths(uploadPath, hasMetadata) - if err != nil { - logger.Fatalf("Error when parsing file paths: %s", err.Error()) - } - uploadRequestObjects := make([]common.FileUploadRequestObject, 0, len(filePaths)) - - logger.Println("\nThe following file(s) has been found in path \"" + uploadPath + "\" and will be uploaded:") - for _, filePath := range filePaths { - // Use ProcessFilename to create the unified object (GUID is empty here, as this command requests a new GUID) - // ProcessFilename signature: (uploadPath, filePath, objectId, includeSubDirName, includeMetadata) - furObject, err := ProcessFilename(g3i.Logger(), uploadPath, filePath, "", includeSubDirName, hasMetadata) - - // Handle case where ProcessFilename fails (e.g., metadata parsing error) - if err != nil { - // Use the data available for logging the failure - g3i.Logger().Failed(filePath, filepath.Base(filePath), common.FileMetadata{}, "", 0, false) - logger.Println("Error processing file path or metadata: " + err.Error()) - continue - } - - // Optional: Display file path before proceeding - file, _ := os.Open(filePath) - if fi, _ := file.Stat(); !fi.IsDir() { - logger.Println("\t" + filePath) - } - file.Close() - - uploadRequestObjects = append(uploadRequestObjects, furObject) - } - // fmt.Fprintln(os.Stderr) - logger.Println() - - if len(uploadRequestObjects) == 0 { - logger.Println("No valid file upload requests were created.") - return - } - - singlePartObjects, multipartObjects := separateSingleAndMultipartUploads(g3i, uploadRequestObjects, forceMultipart) - if batch { - workers, respCh, errCh, batchFURObjects := initBatchUploadChannels(numParallel, len(singlePartObjects)) - - for _, furObject := range singlePartObjects { - if len(batchFURObjects) < workers { - batchFURObjects = append(batchFURObjects, furObject) - } else { - batchUpload(g3i, batchFURObjects, workers, respCh, errCh, bucketName) - batchFURObjects = []common.FileUploadRequestObject{furObject} - } - } - if len(batchFURObjects) > 0 { - batchUpload(g3i, batchFURObjects, workers, respCh, errCh, bucketName) - } - - if len(errCh) > 0 { - close(errCh) - for err := range errCh { - if err != nil { - logger.Printf("Error occurred during uploading: %s\n", err.Error()) - } - } - } - } else { - for _, furObject := range singlePartObjects { - file, err := os.Open(furObject.FilePath) - if err != nil { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - logger.Println("File open error: " + err.Error()) - continue - } - startSingleFileUpload(g3i, furObject, file, bucketName) - } - } - - if len(multipartObjects) > 0 { - err := processMultipartUpload(g3i, multipartObjects, bucketName, includeSubDirName, uploadPath) - if err != nil { - logger.Println(err.Error()) - } - } - if len(g3i.Logger().GetSucceededLogMap()) == 0 { - retryUpload(g3i, g3i.Logger().GetFailedLogMap()) - } - - g3i.Logger().Scoreboard().PrintSB() - }, - } - - uploadCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") - uploadCmd.MarkFlagRequired("profile") //nolint:errcheck - uploadCmd.Flags().StringVar(&uploadPath, "upload-path", "", "The directory or file in which contains file(s) to be uploaded") - uploadCmd.MarkFlagRequired("upload-path") //nolint:errcheck - uploadCmd.Flags().BoolVar(&batch, "batch", false, "Upload in parallel") - uploadCmd.Flags().IntVar(&numParallel, "numparallel", 3, "Number of uploads to run in parallel") - uploadCmd.Flags().BoolVar(&includeSubDirName, "include-subdirname", true, "Include subdirectory names in file name") - uploadCmd.Flags().BoolVar(&forceMultipart, "force-multipart", false, "Force to use multipart upload if possible") - uploadCmd.Flags().BoolVar(&hasMetadata, "metadata", false, "Search for and upload file metadata alongside the file") - uploadCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") - RootCmd.AddCommand(uploadCmd) -} diff --git a/client/g3cmd/utils.go b/client/g3cmd/utils.go deleted file mode 100644 index d65f488..0000000 --- a/client/g3cmd/utils.go +++ /dev/null @@ -1,686 +0,0 @@ -package g3cmd - -import ( - "bytes" - "encoding/json" - "errors" - "fmt" - "io" - "math" - "net/http" - "net/url" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "github.com/calypr/data-client/client/common" - client "github.com/calypr/data-client/client/gen3Client" - "github.com/calypr/data-client/client/logs" - - "github.com/vbauerster/mpb/v8" - "github.com/vbauerster/mpb/v8/decor" -) - -// ManifestObject represents an object from manifest that downloaded from windmill / data-portal -type ManifestObject struct { - ObjectID string `json:"object_id"` - SubjectID string `json:"subject_id"` - Filename string `json:"file_name"` - Filesize int64 `json:"file_size"` -} - -// InitRequestObject represents the payload that sends to FENCE for getting a singlepart upload presignedURL or init a multipart upload for new object file -type InitRequestObject struct { - Filename string `json:"file_name"` - Bucket string `json:"bucket,omitempty"` - GUID string `json:"guid,omitempty"` -} - -// ShepherdInitRequestObject represents the payload that sends to Shepherd for getting a singlepart upload presignedURL or init a multipart upload for new object file -type ShepherdInitRequestObject struct { - Filename string `json:"file_name"` - Authz struct { - Version string `json:"version"` - ResourcePaths []string `json:"resource_paths"` - } `json:"authz"` - Aliases []string `json:"aliases"` - // Metadata is an encoded JSON string of any arbitrary metadata the user wishes to upload. - Metadata map[string]any `json:"metadata"` -} - -// MultipartUploadRequestObject represents the payload that sends to FENCE for getting a presignedURL for a part -type MultipartUploadRequestObject struct { - Key string `json:"key"` - UploadID string `json:"uploadId"` - PartNumber int `json:"partNumber"` - Bucket string `json:"bucket,omitempty"` -} - -// MultipartCompleteRequestObject represents the payload that sends to FENCE for completeing a multipart upload -type MultipartCompleteRequestObject struct { - Key string `json:"key"` - UploadID string `json:"uploadId"` - Parts []MultipartPartObject `json:"parts"` - Bucket string `json:"bucket,omitempty"` -} - -// MultipartPartObject represents a part object -type MultipartPartObject struct { - PartNumber int `json:"PartNumber"` - ETag string `json:"ETag"` -} - -// FileInfo is a helper struct for including subdirname as filename -type FileInfo struct { - FilePath string - Filename string - FileMetadata common.FileMetadata - ObjectId string -} - -// RenamedOrSkippedFileInfo is a helper struct for recording renamed or skipped files -type RenamedOrSkippedFileInfo struct { - GUID string - OldFilename string - NewFilename string -} - -const ( - // B is bytes - B int64 = iota - // KB is kilobytes - KB int64 = 1 << (10 * iota) - // MB is megabytes - MB - // GB is gigabytes - GB - // TB is terrabytes - TB -) - -var unitMap = map[int64]string{ - B: "B", - KB: "KB", - MB: "MB", - GB: "GB", - TB: "TB", -} - -// FileSizeLimit is the maximun single file size for non-multipart upload (5GB) -const FileSizeLimit = 5 * GB - -// MultipartFileSizeLimit is the maximun single file size for multipart upload (5TB) -const MultipartFileSizeLimit = 5 * TB -const minMultipartChunkSize = 5 * MB - -// MaxRetryCount is the maximum retry number per record -const MaxRetryCount = 5 -const maxWaitTime = 300 - -// InitMultipartUpload helps sending requests to FENCE to init a multipart upload -func InitMultipartUpload(g3 client.Gen3Interface, furObject common.FileUploadRequestObject, bucketName string) (string, string, error) { - // Use Filename and GUID directly from the unified request object - multipartInitObject := InitRequestObject{Filename: furObject.Filename, Bucket: bucketName, GUID: furObject.GUID} - - objectBytes, err := json.Marshal(multipartInitObject) - if err != nil { - return "", "", errors.New("Error has occurred during marshalling data for multipart upload initialization, detailed error message: " + err.Error()) - } - - msg, err := g3.DoRequestWithSignedHeader(common.FenceDataMultipartInitEndpoint, "application/json", objectBytes) - - if err != nil { - if strings.Contains(err.Error(), "404") { - return "", "", errors.New(err.Error() + "\nPlease check to ensure FENCE version is at 2.8.0 or beyond") - } - return "", "", errors.New("Error has occurred during multipart upload initialization, detailed error message: " + err.Error()) - } - if msg.UploadID == "" || msg.GUID == "" { - return "", "", errors.New("unknown error has occurred during multipart upload initialization. Please check logs from Gen3 services") - } - return msg.UploadID, msg.GUID, err -} - -// GenerateMultipartPresignedURL helps sending requests to FENCE to get a presigned URL for a part during a multipart upload -func GenerateMultipartPresignedURL(g3 client.Gen3Interface, key string, uploadID string, partNumber int, bucketName string) (string, error) { - multipartUploadObject := MultipartUploadRequestObject{Key: key, UploadID: uploadID, PartNumber: partNumber, Bucket: bucketName} - objectBytes, err := json.Marshal(multipartUploadObject) - if err != nil { - return "", errors.New("Error has occurred during marshalling data for multipart upload presigned url generation, detailed error message: " + err.Error()) - } - - msg, err := g3.DoRequestWithSignedHeader(common.FenceDataMultipartUploadEndpoint, "application/json", objectBytes) - - if err != nil { - return "", errors.New("Error has occurred during multipart upload presigned url generation, detailed error message: " + err.Error()) - } - if msg.PresignedURL == "" { - return "", errors.New("unknown error has occurred during multipart upload presigned url generation. Please check logs from Gen3 services") - } - return msg.PresignedURL, err -} - -// CompleteMultipartUpload helps sending requests to FENCE to complete a multipart upload -func CompleteMultipartUpload(g3 client.Gen3Interface, key string, uploadID string, parts []MultipartPartObject, bucketName string) error { - multipartCompleteObject := MultipartCompleteRequestObject{Key: key, UploadID: uploadID, Parts: parts, Bucket: bucketName} - objectBytes, err := json.Marshal(multipartCompleteObject) - if err != nil { - return errors.New("Error has occurred during marshalling data for multipart upload, detailed error message: " + err.Error()) - } - - _, err = g3.DoRequestWithSignedHeader(common.FenceDataMultipartCompleteEndpoint, "application/json", objectBytes) - if err != nil { - return errors.New("Error has occurred during completing multipart upload, detailed error message: " + err.Error()) - } - return nil -} - -// GetDownloadResponse helps grabbing a response for downloading a file specified with GUID -func GetDownloadResponse(g3 client.Gen3Interface, fdrObject *common.FileDownloadResponseObject, protocolText string) error { - // Attempt to get the file download URL from Shepherd if it's deployed in this commons, - // otherwise fall back to Fence. - var fileDownloadURL string - hasShepherd, err := g3.CheckForShepherdAPI() - if err != nil { - g3.Logger().Println("Error occurred when checking for Shepherd API: " + err.Error()) - g3.Logger().Println("Falling back to Indexd...") - } else if hasShepherd { - endPointPostfix := common.ShepherdEndpoint + "/objects/" + fdrObject.GUID + "/download" - _, r, err := g3.GetResponse(endPointPostfix, "GET", "", nil) - if err != nil { - return errors.New("Error occurred when getting download URL for object " + fdrObject.GUID + " from endpoint " + endPointPostfix + " . Details: " + err.Error()) - } - defer r.Body.Close() - if r.StatusCode != 200 { - buf := new(bytes.Buffer) - buf.ReadFrom(r.Body) // nolint:errcheck - body := buf.String() - return errors.New("Error when getting download URL at " + endPointPostfix + " for file " + fdrObject.GUID + " : Shepherd returned non-200 status code " + strconv.Itoa(r.StatusCode) + " . Request body: " + body) - } - // Unmarshal into json - urlResponse := struct { - URL string `json:"url"` - }{} - err = json.NewDecoder(r.Body).Decode(&urlResponse) - if err != nil { - return errors.New("Error occurred when getting download URL for object " + fdrObject.GUID + " from endpoint " + endPointPostfix + " . Details: " + err.Error()) - } - fileDownloadURL = urlResponse.URL - if fileDownloadURL == "" { - return errors.New("Unknown error occurred when getting download URL for object " + fdrObject.GUID + " from endpoint " + endPointPostfix + " : No URL found in response body. Check the Shepherd logs") - } - } else { - endPointPostfix := common.FenceDataDownloadEndpoint + "/" + fdrObject.GUID + protocolText - msg, err := g3.DoRequestWithSignedHeader(endPointPostfix, "", nil) - - if err != nil || msg.URL == "" { - errorMsg := "Error occurred when getting download URL for object " + fdrObject.GUID - if err != nil { - errorMsg += "\n Details of error: " + err.Error() - } - return errors.New(errorMsg) - } - fileDownloadURL = msg.URL - } - - // TODO: for now we don't print fdrObject.URL in error messages since it is sensitive - // Later after we had log level we could consider for putting URL into debug logs... - fdrObject.URL = fileDownloadURL - if fdrObject.Range != 0 && !strings.Contains(fdrObject.URL, "X-Amz-Signature") && !strings.Contains(fdrObject.URL, "X-Goog-Signature") { // Not S3 or GS URLs and we want resume, send HEAD req first to check if server supports range - resp, err := http.Head(fdrObject.URL) - if err != nil { - errorMsg := "Error occurred when sending HEAD req to URL associated with GUID " + fdrObject.GUID - errorMsg += "\n Details of error: " + sanitizeErrorMsg(err.Error(), fdrObject.URL) - return errors.New(errorMsg) - } - if resp.Header.Get("Accept-Ranges") != "bytes" { // server does not support range, download without range header - fdrObject.Range = 0 - } - } - - headers := map[string]string{} - if fdrObject.Range != 0 { - headers["Range"] = "bytes=" + strconv.FormatInt(fdrObject.Range, 10) + "-" - } - resp, err := g3.MakeARequest(http.MethodGet, fdrObject.URL, "", "", headers, nil, true) - if err != nil { - errorMsg := "Error occurred when making request to URL associated with GUID " + fdrObject.GUID - errorMsg += "\n Details of error: " + sanitizeErrorMsg(err.Error(), fdrObject.URL) - return errors.New(errorMsg) - } - if resp.StatusCode != 200 && resp.StatusCode != 206 { - errorMsg := "Got a non-200 or non-206 response when making request to URL associated with GUID " + fdrObject.GUID - errorMsg += "\n HTTP status code for response: " + strconv.Itoa(resp.StatusCode) - return errors.New(errorMsg) - } - fdrObject.Response = resp - return nil -} - -func sanitizeErrorMsg(errorMsg string, sensitiveURL string) string { - return strings.ReplaceAll(errorMsg, sensitiveURL, "") -} - -// GeneratePresignedURL helps sending requests to Shepherd/Fence and parsing the response in order to get presigned URL for the new upload flow -func GeneratePresignedURL(g3 client.Gen3Interface, filename string, fileMetadata common.FileMetadata, bucketName string) (string, string, error) { - // Attempt to get the presigned URL of this file from Shepherd if it's deployed, otherwise fall back to Fence. - hasShepherd, err := g3.CheckForShepherdAPI() - if err != nil { - g3.Logger().Println("Error occurred when checking for Shepherd API: " + err.Error()) - g3.Logger().Println("Falling back to Fence...") - } else if hasShepherd { - purObject := ShepherdInitRequestObject{ - Filename: filename, - Authz: struct { - Version string `json:"version"` - ResourcePaths []string `json:"resource_paths"` - }{ - "0", - fileMetadata.Authz, - }, - Aliases: fileMetadata.Aliases, - Metadata: fileMetadata.Metadata, - } - objectBytes, err := json.Marshal(purObject) - if err != nil { - return "", "", errors.New("Error occurred when creating upload request for file " + filename + ". Details: " + err.Error()) - } - endPointPostfix := common.ShepherdEndpoint + "/objects" - _, r, err := g3.GetResponse(endPointPostfix, "POST", "", objectBytes) - if err != nil { - return "", "", errors.New("Error occurred when requesting upload URL from " + endPointPostfix + " for file " + filename + ". Details: " + err.Error()) - } - defer r.Body.Close() - if r.StatusCode != 201 { - buf := new(bytes.Buffer) - buf.ReadFrom(r.Body) // nolint:errcheck - body := buf.String() - return "", "", errors.New("Error when requesting upload URL at " + endPointPostfix + " for file " + filename + ": Shepherd returned non-200 status code " + strconv.Itoa(r.StatusCode) + ". Request body: " + body) - } - res := struct { - GUID string `json:"guid"` - URL string `json:"upload_url"` - }{} - err = json.NewDecoder(r.Body).Decode(&res) - if err != nil { - return "", "", errors.New("Error occurred when creating upload URL for file " + filename + ": . Details: " + err.Error()) - } - if res.URL == "" || res.GUID == "" { - return "", "", errors.New("unknown error has occurred during presigned URL or GUID generation. Please check logs from Gen3 services") - } - return res.URL, res.GUID, nil - } - - // Otherwise, fall back to Fence - purObject := InitRequestObject{Filename: filename, Bucket: bucketName} - objectBytes, err := json.Marshal(purObject) - if err != nil { - return "", "", errors.New("Error occurred when marshalling object: " + err.Error()) - } - msg, err := g3.DoRequestWithSignedHeader(common.FenceDataUploadEndpoint, "application/json", objectBytes) - - if err != nil { - return "", "", errors.New("Something went wrong. Maybe you don't have permission to upload data or Fence is misconfigured. Detailed error message: " + err.Error()) - } - if msg.URL == "" || msg.GUID == "" { - return "", "", errors.New("unknown error has occurred during presigned URL or GUID generation. Please check logs from Gen3 services") - } - return msg.URL, msg.GUID, err -} - -// GenerateUploadRequest helps preparing the HTTP request for upload and the progress bar for single part upload -func GenerateUploadRequest(g3 client.Gen3Interface, furObject common.FileUploadRequestObject, file *os.File, progress *mpb.Progress) (common.FileUploadRequestObject, error) { - if furObject.PresignedURL == "" { - endPointPostfix := common.FenceDataUploadEndpoint + "/" + furObject.GUID + "?file_name=" + url.QueryEscape(furObject.Filename) - - // ensure bucket is set - if furObject.Bucket != "" { - endPointPostfix += "&bucket=" + furObject.Bucket - } - msg, err := g3.DoRequestWithSignedHeader(endPointPostfix, "application/json", nil) - if err != nil && !strings.Contains(err.Error(), "No GUID found") { - return furObject, errors.New("Upload error: " + err.Error()) - } - if msg.URL == "" { - return furObject, errors.New("Upload error: error in generating presigned URL for " + furObject.Filename) - } - furObject.PresignedURL = msg.URL - } - - fi, err := file.Stat() - if err != nil { - return furObject, errors.New("File stat error for file" + furObject.Filename + ", file may be missing or unreadable because of permissions.\n") - } - - if fi.Size() > FileSizeLimit { - return furObject, errors.New("The file size of file " + furObject.Filename + " exceeds the limit allowed and cannot be uploaded. The maximum allowed file size is " + FormatSize(FileSizeLimit) + ".\n") - } - - if progress == nil { - progress = mpb.New(mpb.WithOutput(os.Stdout)) - } - bar := progress.AddBar(fi.Size(), - mpb.PrependDecorators( - decor.Name(furObject.Filename+" "), - decor.CountersKibiByte("% .1f / % .1f"), - ), - mpb.AppendDecorators( - decor.Percentage(), - decor.AverageSpeed(decor.SizeB1024(0), " % .1f"), - ), - ) - pr, pw := io.Pipe() - - go func() { - var writer io.Writer - defer pw.Close() - defer file.Close() - - writer = bar.ProxyWriter(pw) - if _, err = io.Copy(writer, file); err != nil { - err = errors.New("io.Copy error: " + err.Error() + "\n") - } - if err = pw.Close(); err != nil { - err = errors.New("Pipe writer close error: " + err.Error() + "\n") - } - }() - if err != nil { - return furObject, err - } - - req, err := http.NewRequest(http.MethodPut, furObject.PresignedURL, pr) - req.ContentLength = fi.Size() - - furObject.Request = req - furObject.Progress = progress - furObject.Bar = bar - - return furObject, err -} - -// DeleteRecord helps sending requests to FENCE to delete a record from INDEXD as well as its storage locations -func DeleteRecord(g3 client.Gen3Interface, guid string) (string, error) { - return g3.DeleteRecord(guid) -} - -func separateSingleAndMultipartUploads(g3i client.Gen3Interface, objects []common.FileUploadRequestObject, forceMultipart bool) ([]common.FileUploadRequestObject, []common.FileUploadRequestObject) { - fileSizeLimit := FileSizeLimit // 5GB - if forceMultipart { - fileSizeLimit = minMultipartChunkSize // 5MB - } - singlepartObjects := make([]common.FileUploadRequestObject, 0) - multipartObjects := make([]common.FileUploadRequestObject, 0) - - for _, object := range objects { - filePath := object.FilePath - - // Check if file exists locally - if _, err := os.Stat(filePath); os.IsNotExist(err) { - g3i.Logger().Printf("The file you specified \"%s\" does not exist locally\n", filePath) - g3i.Logger().Failed(object.FilePath, object.Filename, object.FileMetadata, object.GUID, 0, false) - continue - } - - // Use a closure to handle file operations and cleanup - func(obj common.FileUploadRequestObject) { - file, err := os.Open(filePath) - if err != nil { - g3i.Logger().Println("File open error occurred when validating file path: " + err.Error()) - g3i.Logger().Failed(obj.FilePath, obj.Filename, obj.FileMetadata, obj.GUID, 0, false) - return - } - defer file.Close() - - fi, err := file.Stat() - if err != nil { - g3i.Logger().Println("File stat error occurred when validating file path: " + err.Error()) - g3i.Logger().Failed(obj.FilePath, obj.Filename, obj.FileMetadata, obj.GUID, 0, false) - return - } - if fi.IsDir() { - return - } - - _, ok := g3i.Logger().GetSucceededLogMap()[filePath] - if ok { - g3i.Logger().Println("File \"" + filePath + "\" has been found in local submission history and has been skipped to prevent duplicated submissions.") - return - } - - // Add to failed log initially, it will be removed on success - // This is an existing pattern, keeping it here. - g3i.Logger().Failed(obj.FilePath, obj.Filename, obj.FileMetadata, obj.GUID, 0, false) - - if fi.Size() > MultipartFileSizeLimit { - g3i.Logger().Printf("The file size of %s has exceeded the limit allowed and cannot be uploaded. The maximum allowed file size is %s\n", fi.Name(), FormatSize(MultipartFileSizeLimit)) - } else if fi.Size() > int64(fileSizeLimit) { - multipartObjects = append(multipartObjects, obj) - } else { - singlepartObjects = append(singlepartObjects, obj) - } - }(object) - } - return singlepartObjects, multipartObjects -} - -// ProcessFilename returns an FileInfo object which has the information about the path and name to be used for upload of a file -func ProcessFilename(logger logs.Logger, uploadPath string, filePath string, objectId string, includeSubDirName bool, includeMetadata bool) (common.FileUploadRequestObject, error) { - var err error - filePath, err = common.GetAbsolutePath(filePath) - if err != nil { - return common.FileUploadRequestObject{}, err - } - - filename := filepath.Base(filePath) // Default to base filename - - var metadata common.FileMetadata - if includeSubDirName { - absUploadPath, err := common.GetAbsolutePath(uploadPath) - if err != nil { - return common.FileUploadRequestObject{}, err - } - - // Ensure absUploadPath is a directory path for relative calculation - // Trim the optional wildcard if present - uploadDir := strings.TrimSuffix(absUploadPath, common.PathSeparator+"*") - fileInfo, err := os.Stat(uploadDir) - if err != nil { - return common.FileUploadRequestObject{}, err - } - if fileInfo.IsDir() { - // Calculate the path of the file relative to the upload directory - relPath, err := filepath.Rel(uploadDir, filePath) - if err != nil { - return common.FileUploadRequestObject{}, err - } - filename = relPath - } - } - - if includeMetadata { - // The metadata path is the file name plus '_metadata.json' - metadataFilePath := strings.TrimSuffix(filePath, filepath.Ext(filePath)) + "_metadata.json" - var metadataFileBytes []byte - if _, err := os.Stat(metadataFilePath); err == nil { - metadataFileBytes, err = os.ReadFile(metadataFilePath) - if err != nil { - return common.FileUploadRequestObject{}, errors.New("Error reading metadata file " + metadataFilePath + ": " + err.Error()) - } - err := json.Unmarshal(metadataFileBytes, &metadata) - if err != nil { - return common.FileUploadRequestObject{}, errors.New("Error parsing metadata file " + metadataFilePath + ": " + err.Error()) - } - } else { - // No metadata file was found for this file -- proceed, but warn the user. - logger.Printf("WARNING: File metadata is enabled, but could not find the metadata file %v for file %v. Execute `data-client upload --help` for more info on file metadata.\n", metadataFilePath, filePath) - } - } - return common.FileUploadRequestObject{FilePath: filePath, Filename: filename, FileMetadata: metadata, GUID: objectId}, nil -} - -func getFullFilePath(filePath string, filename string) (string, error) { - filePath, err := common.GetAbsolutePath(filePath) - if err != nil { - return "", err - } - fi, err := os.Stat(filePath) - if err != nil { - return "", err - } - switch mode := fi.Mode(); { - case mode.IsDir(): - if strings.HasSuffix(filePath, "/") { - return filePath + filename, nil - } - return filePath + "/" + filename, nil - case mode.IsRegular(): - return "", errors.New("in manifest upload mode filePath must be a dir") - default: - return "", errors.New("full file path creation unsuccessful") - } -} - -func uploadFile(g3i client.Gen3Interface, furObject common.FileUploadRequestObject, retryCount int) error { - g3i.Logger().Println("Uploading data ...") - if furObject.Progress != nil { - defer furObject.Progress.Wait() - } - - client := &http.Client{} - resp, err := client.Do(furObject.Request) - if err != nil { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, retryCount, false) - return errors.New("Error occurred during upload: " + err.Error()) - } - if resp.StatusCode != 200 { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, retryCount, false) - return errors.New("Upload request got a non-200 response with status code " + strconv.Itoa(resp.StatusCode)) - } - g3i.Logger().Printf("Successfully uploaded file \"%s\" to GUID %s.\n", furObject.FilePath, furObject.GUID) - g3i.Logger().DeleteFromFailedLog(furObject.FilePath) - g3i.Logger().Succeeded(furObject.FilePath, furObject.GUID) - return nil -} - -func getNumberOfWorkers(numParallel int, inputSliceLen int) int { - workers := numParallel - if workers < 1 || workers > inputSliceLen { - workers = inputSliceLen - } - return workers -} - -func initBatchUploadChannels(numParallel int, inputSliceLen int) (int, chan *http.Response, chan error, []common.FileUploadRequestObject) { - workers := getNumberOfWorkers(numParallel, inputSliceLen) - respCh := make(chan *http.Response, inputSliceLen) - errCh := make(chan error, inputSliceLen) - batchFURSlice := make([]common.FileUploadRequestObject, 0) - return workers, respCh, errCh, batchFURSlice -} - -func batchUpload(g3i client.Gen3Interface, furObjects []common.FileUploadRequestObject, workers int, respCh chan *http.Response, errCh chan error, bucketName string) { - progress := mpb.New(mpb.WithOutput(os.Stdout)) - respURL := "" - var err error - var guid string - - for i := range furObjects { - if furObjects[i].Bucket == "" { - furObjects[i].Bucket = bucketName - } - if furObjects[i].GUID == "" { - respURL, guid, err = GeneratePresignedURL(g3i, furObjects[i].Filename, furObjects[i].FileMetadata, bucketName) - if err != nil { - g3i.Logger().Failed(furObjects[i].FilePath, furObjects[i].Filename, furObjects[i].FileMetadata, guid, 0, false) - errCh <- err - continue - } - furObjects[i].PresignedURL = respURL - furObjects[i].GUID = guid - // update failed log with new guid - g3i.Logger().Failed(furObjects[i].FilePath, furObjects[i].Filename, furObjects[i].FileMetadata, guid, 0, false) - } - file, err := os.Open(furObjects[i].FilePath) - if err != nil { - g3i.Logger().Failed(furObjects[i].FilePath, furObjects[i].Filename, furObjects[i].FileMetadata, furObjects[i].GUID, 0, false) - errCh <- errors.New("File open error: " + err.Error()) - continue - } - defer file.Close() - - furObjects[i], err = GenerateUploadRequest(g3i, furObjects[i], file, progress) - if err != nil { - file.Close() - g3i.Logger().Failed(furObjects[i].FilePath, furObjects[i].Filename, furObjects[i].FileMetadata, furObjects[i].GUID, 0, false) - errCh <- errors.New("Error occurred during request generation: " + err.Error()) - continue - } - } - - furObjectCh := make(chan common.FileUploadRequestObject, len(furObjects)) - - client := &http.Client{} - wg := sync.WaitGroup{} - for range workers { - wg.Add(1) - go func() { - for furObject := range furObjectCh { - if furObject.Request != nil { - resp, err := client.Do(furObject.Request) - if err != nil { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - errCh <- err - } else { - if resp.StatusCode != 200 { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - } else { - respCh <- resp - g3i.Logger().DeleteFromFailedLog(furObject.FilePath) - g3i.Logger().Succeeded(furObject.FilePath, furObject.GUID) - g3i.Logger().Scoreboard().IncrementSB(0) - } - } - } else if furObject.FilePath != "" { - g3i.Logger().Failed(furObject.FilePath, furObject.Filename, furObject.FileMetadata, furObject.GUID, 0, false) - } - } - wg.Done() - }() - } - - for i := range furObjects { - furObjectCh <- furObjects[i] - } - close(furObjectCh) - - wg.Wait() - progress.Wait() -} - -// GetWaitTime calculates the wait time for the next retry based on retry count -func GetWaitTime(retryCount int) time.Duration { - exponentialWaitTime := math.Pow(2, float64(retryCount)) - return time.Duration(math.Min(exponentialWaitTime, float64(maxWaitTime))) * time.Second -} - -// FormatSize helps to parse a int64 size into string -func FormatSize(size int64) string { - var unitSize int64 - switch { - case size >= TB: - unitSize = TB - case size >= GB: - unitSize = GB - case size >= MB: - unitSize = MB - case size >= KB: - unitSize = KB - default: - unitSize = B - } - - return fmt.Sprintf("%.1f"+unitMap[unitSize], float64(size)/float64(unitSize)) -} diff --git a/client/gen3Client/client.go b/client/gen3Client/client.go deleted file mode 100644 index a4e46c7..0000000 --- a/client/gen3Client/client.go +++ /dev/null @@ -1,120 +0,0 @@ -package client - -import ( - "bytes" - "context" - "errors" - "fmt" - "net/http" - "net/url" - - "github.com/calypr/data-client/client/jwt" - "github.com/calypr/data-client/client/logs" -) - -//go:generate mockgen -destination=../mocks/mock_gen3interface.go -package=mocks github.com/calypr/data-client/client/gen3Client Gen3Interface - -// Gen3Interface contains methods used to make authorized http requests to Gen3 services. -// The credential is embedded in the implementation, so it doesn't need to be passed to each method. -type Gen3Interface interface { - CheckPrivileges() (string, map[string]any, error) - CheckForShepherdAPI() (bool, error) - GetResponse(endpointPostPrefix string, method string, contentType string, bodyBytes []byte) (string, *http.Response, error) - DoRequestWithSignedHeader(endpointPostPrefix string, contentType string, bodyBytes []byte) (jwt.JsonMessage, error) - MakeARequest(method string, apiEndpoint string, accessToken string, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) - GetHost() (*url.URL, error) - GetCredential() *jwt.Credential - DeleteRecord(guid string) (string, error) - - Logger() *logs.TeeLogger -} - -// Gen3Client wraps jwt.FunctionInterface and embeds the credential -type Gen3Client struct { - Ctx context.Context - FunctionInterface jwt.FunctionInterface - credential *jwt.Credential - - logger *logs.TeeLogger -} - -func (g *Gen3Client) Logger() *logs.TeeLogger { - return g.logger -} - -// CheckPrivileges wraps the underlying method with embedded credential -func (g *Gen3Client) CheckPrivileges() (string, map[string]any, error) { - return g.FunctionInterface.CheckPrivileges(g.credential) -} - -// CheckForShepherdAPI wraps the underlying method with embedded credential -func (g *Gen3Client) CheckForShepherdAPI() (bool, error) { - return g.FunctionInterface.CheckForShepherdAPI(g.credential) -} - -// GetResponse wraps the underlying method with embedded credential -func (g *Gen3Client) GetResponse(endpointPostPrefix string, method string, contentType string, bodyBytes []byte) (string, *http.Response, error) { - return g.FunctionInterface.GetResponse(g.credential, endpointPostPrefix, method, contentType, bodyBytes) -} - -// DoRequestWithSignedHeader wraps the underlying method with embedded credential -func (g *Gen3Client) DoRequestWithSignedHeader(endpointPostPrefix string, contentType string, bodyBytes []byte) (jwt.JsonMessage, error) { - return g.FunctionInterface.DoRequestWithSignedHeader(g.credential, endpointPostPrefix, contentType, bodyBytes) -} - -// GetHost wraps the underlying method with embedded credential -func (g *Gen3Client) GetHost() (*url.URL, error) { - return g.FunctionInterface.GetHost(g.credential) -} - -// GetCredential returns the embedded credential -func (g *Gen3Client) GetCredential() *jwt.Credential { - return g.credential -} - -// MakeARequest wraps the underlying Request.MakeARequest method -func (g *Gen3Client) MakeARequest(method string, apiEndpoint string, accessToken string, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) { - // Access the underlying Request through the Functions struct - // We need to create a temporary Request instance since we can't access it directly - if functions, ok := g.FunctionInterface.(*jwt.Functions); ok { - return functions.Request.MakeARequest(method, apiEndpoint, accessToken, contentType, headers, body, noTimeout) - } - return nil, errors.New("unable to access MakeARequest method") -} - -// DeleteRecord deletes a record from INDEXD as well as its storage locations -func (g *Gen3Client) DeleteRecord(guid string) (string, error) { - // Use the embedded credential - // Since DeleteRecord is not part of FunctionInterface, we need to access it via type assertion - // or create a new Functions instance. We'll use type assertion first. - if functions, ok := g.FunctionInterface.(*jwt.Functions); ok { - return functions.DeleteRecord(g.credential, guid) - } - - // This should never happen, but handle it gracefully - return "", errors.New("unable to access DeleteRecord method") -} - -// NewGen3Interface returns a Gen3Client that embeds the credential and implements Gen3Interface. -// This eliminates the need to pass credentials around everywhere. -func NewGen3Interface(ctx context.Context, profile string, logger *logs.TeeLogger, opts ...func(*Gen3Client)) (Gen3Interface, error) { - // Note: A tee logger must be passed here otherwise you risk causing panics. - - config := &jwt.Configure{} - request := &jwt.Request{Ctx: ctx, Logs: logger} - client := jwt.NewFunctions(ctx, config, request) - - cred, err := config.ParseConfig(profile) - if err != nil { - return nil, err - } - if valid, err := config.IsValidCredential(cred); !valid { - return nil, fmt.Errorf("invalid credential: %v", err) - } - - return &Gen3Client{ - FunctionInterface: client, - credential: &cred, - logger: logger, - }, nil -} diff --git a/client/jwt/configure.go b/client/jwt/configure.go deleted file mode 100644 index 2b9b7f6..0000000 --- a/client/jwt/configure.go +++ /dev/null @@ -1,321 +0,0 @@ -package jwt - -//go:generate mockgen -destination=../mocks/mock_configure.go -package=mocks github.com/calypr/data-client/client/jwt ConfigureInterface - -import ( - "encoding/json" - "errors" - "fmt" - "net/url" - "os" - "path" - "regexp" - "strings" - "time" - - "github.com/calypr/data-client/client/common" - "github.com/calypr/data-client/client/logs" - "github.com/golang-jwt/jwt/v5" - "gopkg.in/ini.v1" -) - -var ErrProfileNotFound = errors.New("profile not found in config file") - -type Credential struct { - Profile string - KeyId string - APIKey string - AccessToken string - APIEndpoint string - UseShepherd string - MinShepherdVersion string -} - -type Configure struct { - Logs logs.Logger -} - -type ConfigureInterface interface { - ReadFile(string, string) string - ValidateUrl(string) (*url.URL, error) - GetConfigPath() (string, error) - UpdateConfigFile(Credential) error - ParseKeyValue(str string, expr string) (string, error) - ParseConfig(profile string) (Credential, error) - IsValidCredential(Credential) (bool, error) -} - -func (conf *Configure) ReadFile(filePath string, fileType string) string { - //Look in config file - fullFilePath, err := common.GetAbsolutePath(filePath) - if err != nil { - conf.Logs.Println("error occurred when parsing config file path: " + err.Error()) - return "" - } - if _, err := os.Stat(fullFilePath); err != nil { - conf.Logs.Println("File specified at " + fullFilePath + " not found") - return "" - } - - content, err := os.ReadFile(fullFilePath) - if err != nil { - conf.Logs.Println("error occurred when reading file: " + err.Error()) - return "" - } - - contentStr := string(content[:]) - - if fileType == "json" { - contentStr = strings.ReplaceAll(contentStr, "\n", "") - } - return contentStr -} - -func (conf *Configure) ValidateUrl(apiEndpoint string) (*url.URL, error) { - parsedURL, err := url.Parse(apiEndpoint) - if err != nil { - return parsedURL, errors.New("Error occurred when parsing apiendpoint URL: " + err.Error()) - } - if parsedURL.Host == "" { - return parsedURL, errors.New("Invalid endpoint. A valid endpoint looks like: https://www.tests.com") - } - return parsedURL, nil -} - -func (conf *Configure) ReadCredentials(filePath string, fenceToken string) (*Credential, error) { - var profileConfig Credential - if filePath != "" { - jsonContent := conf.ReadFile(filePath, "json") - jsonContent = strings.ReplaceAll(jsonContent, "key_id", "KeyId") - jsonContent = strings.ReplaceAll(jsonContent, "api_key", "APIKey") - err := json.Unmarshal([]byte(jsonContent), &profileConfig) - if err != nil { - errs := fmt.Errorf("Cannot read json file: %s", err.Error()) - conf.Logs.Println(errs.Error()) - return nil, errs - } - } else if fenceToken != "" { - profileConfig.AccessToken = fenceToken - } - return &profileConfig, nil -} - -func (conf *Configure) GetConfigPath() (string, error) { - homeDir, err := os.UserHomeDir() - if err != nil { - return "", err - } - configPath := path.Join(homeDir + common.PathSeparator + ".gen3" + common.PathSeparator + "gen3_client_config.ini") - return configPath, nil -} - -func (conf *Configure) InitConfigFile() error { - /* - Make sure the config exists on start up - */ - configPath, err := conf.GetConfigPath() - if err != nil { - return err - } - - if _, err := os.Stat(path.Dir(configPath)); os.IsNotExist(err) { - osErr := os.Mkdir(path.Join(path.Dir(configPath)), os.FileMode(0777)) - if osErr != nil { - return err - } - _, osErr = os.Create(configPath) - if osErr != nil { - return err - } - } - if _, err := os.Stat(configPath); os.IsNotExist(err) { - _, osErr := os.Create(configPath) - if osErr != nil { - return err - } - } - _, err = ini.Load(configPath) - - return err -} - -func (conf *Configure) UpdateConfigFile(profileConfig Credential) error { - /* - Overwrite the config file with new credential - - Args: - profileConfig: Credential object represents config of a profile - configPath: file path to config file - */ - configPath, err := conf.GetConfigPath() - if err != nil { - errs := fmt.Errorf("error occurred when getting config path: %s", err.Error()) - conf.Logs.Println(errs.Error()) - return errs - } - cfg, err := ini.Load(configPath) - if err != nil { - errs := fmt.Errorf("error occurred when loading config file: %s", err.Error()) - conf.Logs.Println(errs.Error()) - return errs - } - - section := cfg.Section(profileConfig.Profile) - if profileConfig.KeyId != "" { - section.Key("key_id").SetValue(profileConfig.KeyId) - } - if profileConfig.APIKey != "" { - section.Key("api_key").SetValue(profileConfig.APIKey) - } - if profileConfig.AccessToken != "" { - section.Key("access_token").SetValue(profileConfig.AccessToken) - } - if profileConfig.APIEndpoint != "" { - section.Key("api_endpoint").SetValue(profileConfig.APIEndpoint) - } - - section.Key("use_shepherd").SetValue(profileConfig.UseShepherd) - section.Key("min_shepherd_version").SetValue(profileConfig.MinShepherdVersion) - err = cfg.SaveTo(configPath) - if err != nil { - errs := fmt.Errorf("error occurred when saving config file: %s", err.Error()) - return errs - } - return nil -} - -func (conf *Configure) ParseKeyValue(str string, expr string) (string, error) { - r, err := regexp.Compile(expr) - if err != nil { - return "", fmt.Errorf("error occurred when parsing key/value: %v", err.Error()) - } - match := r.FindStringSubmatch(str) - if len(match) == 0 { - return "", fmt.Errorf("No match found") - } - return match[1], nil -} - -func (conf *Configure) ParseConfig(profile string) (Credential, error) { - /* - Looking profile in config file. The config file is a text file located at ~/.gen3 directory. It can - contain more than 1 profile. If there is no profile found, the user is asked to run a command to - create the profile - - The format of config file is described as following - - [profile1] - key_id=key_id_example_1 - api_key=api_key_example_1 - access_token=access_token_example_1 - api_endpoint=http://localhost:8000 - use_shepherd=true - min_shepherd_version=2.0.0 - - [profile2] - key_id=key_id_example_2 - api_key=api_key_example_2 - access_token=access_token_example_2 - api_endpoint=http://localhost:8000 - use_shepherd=false - min_shepherd_version= - - Args: - profile: the specific profile in config file - Returns: - An instance of Credential - */ - - homeDir, err := os.UserHomeDir() - if err != nil { - errs := fmt.Errorf("Error occurred when getting home directory: %s", err.Error()) - return Credential{}, errs - } - configPath := path.Join(homeDir + common.PathSeparator + ".gen3" + common.PathSeparator + "gen3_client_config.ini") - profileConfig := Credential{ - Profile: profile, - KeyId: "", - APIKey: "", - AccessToken: "", - APIEndpoint: "", - } - if _, err := os.Stat(configPath); os.IsNotExist(err) { - return Credential{}, fmt.Errorf("%w Run configure command (with a profile if desired) to set up account credentials \n"+ - "Example: ./data-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org", ErrProfileNotFound) - } - - // If profile not in config file, prompt user to set up config first - cfg, err := ini.Load(configPath) - if err != nil { - errs := fmt.Errorf("Error occurred when reading config file: %s", err.Error()) - return Credential{}, errs - } - sec, err := cfg.GetSection(profile) - if err != nil { - return Credential{}, fmt.Errorf("%w: Need to run \"data-client configure --profile="+profile+" --cred= --apiendpoint=\" first", ErrProfileNotFound) - } - // Read in API key, key ID and endpoint for given profile - profileConfig.KeyId = sec.Key("key_id").String() - profileConfig.APIKey = sec.Key("api_key").String() - profileConfig.AccessToken = sec.Key("access_token").String() - - if profileConfig.KeyId == "" && profileConfig.APIKey == "" && profileConfig.AccessToken == "" { - errs := fmt.Errorf("key_id, api_key and access_token not found in profile.") - return Credential{}, errs - } - profileConfig.APIEndpoint = sec.Key("api_endpoint").String() - if profileConfig.APIEndpoint == "" { - errs := fmt.Errorf("api_endpoint not found in profile.") - return Credential{}, errs - } - // UseShepherd and MinShepherdVersion are optional - profileConfig.UseShepherd = sec.Key("use_shepherd").String() - profileConfig.MinShepherdVersion = sec.Key("min_shepherd_version").String() - - return profileConfig, nil -} - -func (conf *Configure) IsValidCredential(profileConfig Credential) (bool, error) { - /* Checks to see if credential in credential file is still valid */ - const expirationThresholdDays = 10 - // Parse the token without verifying the signature to access the claims. - token, _, err := new(jwt.Parser).ParseUnverified(profileConfig.APIKey, jwt.MapClaims{}) - if err != nil { - return false, fmt.Errorf("ERROR: Invalid token format: %v", err) - } - - claims, ok := token.Claims.(jwt.MapClaims) - if !ok { - return false, fmt.Errorf("Unable to parse claims from provided token %#v", token) - } - - exp, ok := claims["exp"].(float64) - if !ok { - return false, fmt.Errorf("ERROR: 'exp' claim not found or is not a number for claims %s", claims) - } - - iat, ok := claims["iat"].(float64) - if !ok { - return false, fmt.Errorf("ERROR: 'iat' claim not found or is not a number for claims %s", claims) - } - - now := time.Now().UTC() - expTime := time.Unix(int64(exp), 0).UTC() - iatTime := time.Unix(int64(iat), 0).UTC() - - if expTime.Before(now) { - return false, fmt.Errorf("key %s expired %s < %s", profileConfig.APIKey, expTime.Format(time.RFC3339), now.Format(time.RFC3339)) - } - if iatTime.After(now) { - return false, fmt.Errorf("key %s not yet valid %s > %s", profileConfig.APIKey, iatTime.Format(time.RFC3339), now.Format(time.RFC3339)) - } - - delta := expTime.Sub(now) - if delta > 0 && delta.Hours() < float64(expirationThresholdDays*24) { - daysUntilExpiration := int(delta.Hours() / 24) - if daysUntilExpiration > 0 { - return true, fmt.Errorf("WARNING %s: Key will expire in %d days, on %s", profileConfig.APIKey, daysUntilExpiration, expTime.Format(time.RFC3339)) - } - } - return true, nil -} diff --git a/client/jwt/functions.go b/client/jwt/functions.go deleted file mode 100644 index 004d61b..0000000 --- a/client/jwt/functions.go +++ /dev/null @@ -1,370 +0,0 @@ -package jwt - -//go:generate mockgen -destination=../mocks/mock_functions.go -package=mocks github.com/calypr/data-client/client/jwt FunctionInterface -//go:generate mockgen -destination=../mocks/mock_request.go -package=mocks github.com/calypr/data-client/client/jwt RequestInterface - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strconv" - "strings" - - "github.com/calypr/data-client/client/common" - "github.com/calypr/data-client/client/logs" - "github.com/hashicorp/go-version" -) - -func NewFunctions(ctx context.Context, config ConfigureInterface, request RequestInterface) FunctionInterface { - return &Functions{ - Config: config, - Request: request, - } -} - -type Functions struct { - Request RequestInterface - Config ConfigureInterface -} - -type FunctionInterface interface { - CheckPrivileges(profileConfig *Credential) (string, map[string]any, error) - CheckForShepherdAPI(profileConfig *Credential) (bool, error) - GetResponse(profileConfig *Credential, endpointPostPrefix string, method string, contentType string, bodyBytes []byte) (string, *http.Response, error) - DoRequestWithSignedHeader(profileConfig *Credential, endpointPostPrefix string, contentType string, bodyBytes []byte) (JsonMessage, error) - ParseFenceURLResponse(resp *http.Response) (JsonMessage, error) - GetHost(profileConfig *Credential) (*url.URL, error) -} - -type Request struct { - Logs logs.Logger - Ctx context.Context -} - -type RequestInterface interface { - MakeARequest(method string, apiEndpoint string, accessToken string, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) - RequestNewAccessToken(accessTokenEndpoint string, profileConfig *Credential) error - Logger() logs.Logger -} - -func (r *Request) Logger() logs.Logger { - return r.Logs -} - -func (r *Request) MakeARequest(method string, apiEndpoint string, accessToken string, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) { - /* - Make http request with header and body - */ - if headers == nil { - headers = make(map[string]string) - } - if accessToken != "" { - headers["Authorization"] = "Bearer " + accessToken - } - if contentType != "" { - headers["Content-Type"] = contentType - } - var client *http.Client - if noTimeout { - client = &http.Client{} - } else { - client = &http.Client{Timeout: common.DefaultTimeout} - } - var req *http.Request - var err error - if body == nil { - req, err = http.NewRequestWithContext(r.Ctx, method, apiEndpoint, nil) - } else { - req, err = http.NewRequestWithContext(r.Ctx, method, apiEndpoint, body) - } - if err != nil { - return nil, errors.New("Error occurred during generating HTTP request: " + err.Error()) - } - for k, v := range headers { - req.Header.Add(k, v) - } - resp, err := client.Do(req) - if err != nil { - return nil, errors.New("Error occurred during making HTTP request: " + err.Error()) - } - return resp, nil -} - -func (r *Request) RequestNewAccessToken(accessTokenEndpoint string, profileConfig *Credential) error { - /* - Request new access token to replace the expired one. - - Args: - accessTokenEndpoint: the api endpoint for request new access token - Returns: - profileConfig: new credential - err: error - - */ - body := bytes.NewBufferString("{\"api_key\": \"" + profileConfig.APIKey + "\"}") - resp, err := r.MakeARequest("POST", accessTokenEndpoint, "", "application/json", nil, body, false) - var m AccessTokenStruct - // parse resp error codes first for profile configuration verification - if resp != nil && resp.StatusCode != 200 { - return errors.New("Error occurred in RequestNewAccessToken with error code " + strconv.Itoa(resp.StatusCode) + ", check FENCE log for more details.") - } - if err != nil { - return errors.New("Error occurred in RequestNewAccessToken: " + err.Error()) - } - defer resp.Body.Close() - - str := ResponseToString(resp) - err = DecodeJsonFromString(str, &m) - if err != nil { - return errors.New("Error occurred in RequestNewAccessToken: " + err.Error()) - } - - if m.AccessToken == "" { - return errors.New("Could not get new access key from response string: " + str) - } - profileConfig.AccessToken = m.AccessToken - return nil -} - -func (f *Functions) ParseFenceURLResponse(resp *http.Response) (JsonMessage, error) { - msg := JsonMessage{} - - if resp == nil { - return msg, errors.New("Nil response received") - } - - // Capture the body for error reporting before we do anything else - // Using your existing ResponseToString helper - bodyStr := ResponseToString(resp) - - if !(resp.StatusCode == 200 || resp.StatusCode == 201) { - // Prepare a base error that includes the body content - errorMessage := fmt.Sprintf("Status: %d | Response: %s", resp.StatusCode, bodyStr) - - switch resp.StatusCode { - case 401: - return msg, fmt.Errorf("401 Unauthorized: %s", errorMessage) - case 403: - return msg, fmt.Errorf("403 Forbidden: %s (URL: %s)", bodyStr, resp.Request.URL.String()) - case 404: - return msg, fmt.Errorf("404 Not Found: %s (URL: %s)", bodyStr, resp.Request.URL.String()) - case 500: - return msg, fmt.Errorf("500 Internal Server Error: %s", bodyStr) - case 503: - return msg, fmt.Errorf("503 Service Unavailable: %s", bodyStr) - default: - return msg, fmt.Errorf("Unexpected Error (%d): %s", resp.StatusCode, bodyStr) - } - } - - // Logic for successful status codes - if strings.Contains(bodyStr, "Can't find a location for the data") { - return msg, errors.New("The provided GUID is not found") - } - - err := DecodeJsonFromString(bodyStr, &msg) - if err != nil { - return msg, fmt.Errorf("failed to decode JSON: %w (Raw body: %s)", err, bodyStr) - } - - return msg, nil -} -func (f *Functions) CheckForShepherdAPI(profileConfig *Credential) (bool, error) { - // Check if Shepherd is enabled - if profileConfig.UseShepherd == "false" { - return false, nil - } - if profileConfig.UseShepherd != "true" && common.DefaultUseShepherd == false { - return false, nil - } - // If Shepherd is enabled, make sure that the commons has a compatible version of Shepherd deployed. - // Compare the version returned from the Shepherd version endpoint with the minimum acceptable Shepherd version. - var minShepherdVersion string - if profileConfig.MinShepherdVersion == "" { - minShepherdVersion = common.DefaultMinShepherdVersion - } else { - minShepherdVersion = profileConfig.MinShepherdVersion - } - - _, res, err := f.GetResponse(profileConfig, common.ShepherdVersionEndpoint, "GET", "", nil) - if err != nil { - return false, errors.New("Error occurred during generating HTTP request: " + err.Error()) - } - defer res.Body.Close() - if res.StatusCode != 200 { - return false, nil - } - bodyBytes, err := io.ReadAll(res.Body) - if err != nil { - return false, errors.New("Error occurred when reading HTTP request: " + err.Error()) - } - body, err := strconv.Unquote(string(bodyBytes)) - if err != nil { - return false, fmt.Errorf("Error occurred when parsing version from Shepherd: %v: %v", string(body), err) - } - // Compare the version in the response to the target version - ver, err := version.NewVersion(body) - if err != nil { - return false, fmt.Errorf("Error occurred when parsing version from Shepherd: %v: %v", string(body), err) - } - minVer, err := version.NewVersion(minShepherdVersion) - if err != nil { - return false, fmt.Errorf("Error occurred when parsing minimum acceptable Shepherd version: %v: %v", minShepherdVersion, err) - } - if ver.GreaterThanOrEqual(minVer) { - return true, nil - } - return false, fmt.Errorf("Shepherd is enabled, but %v does not have correct Shepherd version. (Need Shepherd version >=%v, got %v)", profileConfig.APIEndpoint, minVer, ver) -} - -func (f *Functions) GetResponse(profileConfig *Credential, endpointPostPrefix string, method string, contentType string, bodyBytes []byte) (string, *http.Response, error) { - - var resp *http.Response - var err error - - if profileConfig.APIKey == "" && profileConfig.AccessToken == "" && profileConfig.APIEndpoint == "" { - return "", resp, fmt.Errorf("No credentials found in the configuration file! Please use \"./data-client configure\" to configure your credentials first %s", profileConfig) - } - - host, _ := url.Parse(profileConfig.APIEndpoint) - prefixEndPoint := host.Scheme + "://" + host.Host - apiEndpoint := host.Scheme + "://" + host.Host + endpointPostPrefix - isExpiredToken := false - if profileConfig.AccessToken != "" { - resp, err = f.Request.MakeARequest(method, apiEndpoint, profileConfig.AccessToken, contentType, nil, bytes.NewBuffer(bodyBytes), false) - if err != nil { - return "", resp, fmt.Errorf("Error while requesting user access token at %v: %v", apiEndpoint, err) - } - - // 401 code is general error code from FENCE. the error message is also not clear for the case - // that the token expired. Temporary solution: get new access token and make another attempt. - if resp != nil && (resp.StatusCode == 401 || resp.StatusCode == 503) { - isExpiredToken = true - } else { - return prefixEndPoint, resp, err - } - } - if profileConfig.AccessToken == "" || isExpiredToken { - err := f.Request.RequestNewAccessToken(prefixEndPoint+common.FenceAccessTokenEndpoint, profileConfig) - if err != nil { - return prefixEndPoint, resp, err - } - err = f.Config.UpdateConfigFile(*profileConfig) - if err != nil { - return prefixEndPoint, resp, err - } - - resp, err = f.Request.MakeARequest(method, apiEndpoint, profileConfig.AccessToken, contentType, nil, bytes.NewBuffer(bodyBytes), false) - if err != nil { - return prefixEndPoint, resp, err - } - } - - return prefixEndPoint, resp, nil -} - -func (f *Functions) GetHost(profileConfig *Credential) (*url.URL, error) { - if profileConfig.APIEndpoint == "" { - return nil, errors.New("No APIEndpoint found in the configuration file! Please use \"./data-client configure\" to configure your credentials first") - } - host, _ := url.Parse(profileConfig.APIEndpoint) - return host, nil -} - -func (f *Functions) DoRequestWithSignedHeader(profileConfig *Credential, endpointPostPrefix string, contentType string, bodyBytes []byte) (JsonMessage, error) { - /* - Do request with signed header. User may have more than one profile and use a profile to make a request - */ - var err error - var msg JsonMessage - - method := "GET" - if bodyBytes != nil { - method = "POST" - } - - _, resp, err := f.GetResponse(profileConfig, endpointPostPrefix, method, contentType, bodyBytes) - if err != nil { - return msg, err - } - defer resp.Body.Close() - - msg, err = f.ParseFenceURLResponse(resp) - return msg, err -} - -func (f *Functions) CheckPrivileges(profileConfig *Credential) (string, map[string]any, error) { - /* - Return user privileges from specified profile - */ - var err error - var data map[string]any - - host, resp, err := f.GetResponse(profileConfig, common.FenceUserEndpoint, "GET", "", nil) - if err != nil { - return "", nil, errors.New("Error occurred when getting response from remote: " + err.Error()) - } - defer resp.Body.Close() - - str := ResponseToString(resp) - err = json.Unmarshal([]byte(str), &data) - if err != nil { - return "", nil, errors.New("Error occurred when unmarshalling response: " + err.Error()) - } - - resourceAccess, ok := data["authz"].(map[string]any) - - // If the `authz` section (Arborist permissions) is empty or missing, try get `project_access` section (Fence permissions) - if len(resourceAccess) == 0 || !ok { - resourceAccess, ok = data["project_access"].(map[string]any) - if !ok { - return "", nil, errors.New("Not possible to read access privileges of user") - } - } - - return host, resourceAccess, err -} - -func (f *Functions) DeleteRecord(profileConfig *Credential, guid string) (string, error) { - var err error - var msg string - - hasShepherd, err := f.CheckForShepherdAPI(profileConfig) - if err != nil { - f.Request.Logger().Printf("WARNING: Error while checking for Shepherd API: %v. Falling back to Fence to delete record.\n", err) - } else if hasShepherd { - endPointPostfix := common.ShepherdEndpoint + "/objects/" + guid - _, resp, err := f.GetResponse(profileConfig, endPointPostfix, "DELETE", "", nil) - if err != nil { - return "", err - } - defer resp.Body.Close() - if resp.StatusCode == 204 { - msg = "Record with GUID " + guid + " has been deleted" - } else if resp.StatusCode == 500 { - err = errors.New("Internal server error occurred when deleting " + guid + "; could not delete stored files, or not able to delete INDEXD record") - } - return msg, err - } - - endPointPostfix := common.FenceDataEndpoint + "/" + guid - - _, resp, err := f.GetResponse(profileConfig, endPointPostfix, "DELETE", "", nil) - if err != nil { - return "", err - } - defer resp.Body.Close() - - if resp.StatusCode == 204 { - msg = "Record with GUID " + guid + " has been deleted" - } else if resp.StatusCode == 500 { - err = errors.New("Internal server error occurred when deleting " + guid + "; could not delete stored files, or not able to delete INDEXD record") - } - - return msg, err -} diff --git a/client/jwt/update.go b/client/jwt/update.go deleted file mode 100644 index b2d9cc9..0000000 --- a/client/jwt/update.go +++ /dev/null @@ -1,78 +0,0 @@ -package jwt - -import ( - "context" - "errors" - "fmt" - "strings" - - "github.com/calypr/data-client/client/common" - "github.com/calypr/data-client/client/logs" - "github.com/hashicorp/go-version" -) - -func UpdateConfig(logger logs.Logger, cred *Credential) error { - var conf Configure - var req Request = Request{Ctx: context.Background()} - - if cred.Profile == "" { - return fmt.Errorf("profile name is required") - } - if cred.APIEndpoint == "" { - return fmt.Errorf("API endpoint is required") - } - - // Normalize endpoint - cred.APIEndpoint = strings.TrimSpace(cred.APIEndpoint) - cred.APIEndpoint = strings.TrimSuffix(cred.APIEndpoint, "/") - - // Validate URL format - parsedURL, err := conf.ValidateUrl(cred.APIEndpoint) - if err != nil { - return fmt.Errorf("invalid apiendpoint URL: %w", err) - } - fenceBase := parsedURL.Scheme + "://" + parsedURL.Host - if existingCfg, err := conf.ParseConfig(cred.Profile); err == nil { - // Only copy optional fields if the user didn't override them via flags - if cred.UseShepherd == "" { - cred.UseShepherd = existingCfg.UseShepherd - } - if cred.MinShepherdVersion == "" { - cred.MinShepherdVersion = existingCfg.MinShepherdVersion - } - } else if !errors.Is(err, ErrProfileNotFound) { - return err - } - - if cred.APIKey != "" { - // Always refresh the access token — ignore any old one that might be in the struct - err = req.RequestNewAccessToken(fenceBase+common.FenceAccessTokenEndpoint, cred) - if err != nil { - if strings.Contains(err.Error(), "401") { - return fmt.Errorf("authentication failed (401) for %s — your API key is invalid, revoked, or expired", fenceBase) - } - if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "no such host") { - return fmt.Errorf("cannot reach Fence at %s — is this a valid Gen3 commons?", fenceBase) - } - return fmt.Errorf("failed to refresh access token: %w", err) - } - } else { - logger.Printf("WARNING: Your profile will only be valid for 24 hours since you have only provided a refresh token for authentication") - } - - // Clean up shepherd flags - cred.UseShepherd = strings.TrimSpace(cred.UseShepherd) - cred.MinShepherdVersion = strings.TrimSpace(cred.MinShepherdVersion) - - if cred.MinShepherdVersion != "" { - if _, err = version.NewVersion(cred.MinShepherdVersion); err != nil { - return fmt.Errorf("invalid min-shepherd-version: %w", err) - } - } - - if err := conf.UpdateConfigFile(*cred); err != nil { - return fmt.Errorf("failed to write config file: %w", err) - } - - return nil -} diff --git a/client/jwt/utils.go b/client/jwt/utils.go deleted file mode 100644 index 466a3a0..0000000 --- a/client/jwt/utils.go +++ /dev/null @@ -1,38 +0,0 @@ -package jwt - -import ( - "bytes" - "encoding/json" - "net/http" -) - -type Message any - -type Response any - -type AccessTokenStruct struct { - AccessToken string `json:"access_token"` -} - -type JsonMessage struct { - URL string `json:"url"` - GUID string `json:"guid"` - UploadID string `json:"uploadId"` - PresignedURL string `json:"presigned_url"` - FileName string `json:"file_name"` - URLs []string `json:"urls"` - Size int64 `json:"size"` -} - -type DoRequest func(*http.Response) *http.Response - -func ResponseToString(resp *http.Response) string { - buf := new(bytes.Buffer) - buf.ReadFrom(resp.Body) // nolint: errcheck - return buf.String() -} - -func DecodeJsonFromString(str string, msg Message) error { - err := json.Unmarshal([]byte(str), &msg) - return err -} diff --git a/client/logs/logger.go b/client/logs/logger.go deleted file mode 100644 index 7a6d53b..0000000 --- a/client/logs/logger.go +++ /dev/null @@ -1,41 +0,0 @@ -package logs - -import ( - "io" -) - -type Logger interface { - Printf(format string, v ...any) - Println(v ...any) - Fatalf(format string, v ...any) - Fatal(v ...any) - Writer() io.Writer -} - -type Option func(*config) - -type config struct { - console bool - messageFile bool - failedLog bool - succeededLog bool - enableScoreboard bool - baseLogger Logger -} - -func WithConsole() Option { return func(c *config) { c.console = true } } -func WithMessageFile() Option { return func(c *config) { c.messageFile = true } } -func WithFailedLog() Option { return func(c *config) { c.failedLog = true } } -func WithSucceededLog() Option { return func(c *config) { c.succeededLog = true } } -func WithScoreboard() Option { return func(c *config) { c.enableScoreboard = true } } -func WithBaseLogger(base Logger) Option { return func(c *config) { c.baseLogger = base } } - -func defaults() *config { - return &config{ - console: true, - messageFile: true, - failedLog: true, - succeededLog: true, - baseLogger: nil, - } -} diff --git a/client/logs/tee_logger.go b/client/logs/tee_logger.go deleted file mode 100644 index bd78d8a..0000000 --- a/client/logs/tee_logger.go +++ /dev/null @@ -1,174 +0,0 @@ -package logs - -import ( - "encoding/json" - "fmt" - "io" // Added for standard logging methods like Fatal - "os" - "sync" - - "github.com/calypr/data-client/client/common" -) - -// --- teeLogger Implementation --- -type TeeLogger struct { - mu sync.RWMutex - writers []io.Writer - scoreboard *Scoreboard - - failedMu sync.Mutex - FailedMap map[string]common.RetryObject // Maps filePath to FileMetadata - failedPath string - - succeededMu sync.Mutex - succeededMap map[string]string // Maps filePath to GUID - succeededPath string -} - -// NewTeeLogger combines initialization and log loading (replacing initSyncLogs) -func NewTeeLogger(logDir, profile string, writers ...io.Writer) *TeeLogger { - t := &TeeLogger{ - mu: sync.RWMutex{}, - writers: writers, - scoreboard: nil, - - FailedMap: make(map[string]common.RetryObject), - succeededMap: make(map[string]string), - } - - return t -} - -// Internal helper function (replaces the global loadJSON) -func loadJSON(path string, v any) { - data, _ := os.ReadFile(path) - if len(data) > 0 { - // Error handling for Unmarshal is often omitted in utility code - // but is good practice. We keep the original style for now. - json.Unmarshal(data, v) - } -} - -// --- Public Logger Methods --- - -// Printf implements part of the standard Logger interface. -func (t *TeeLogger) Printf(format string, v ...any) { - t.write(fmt.Sprintf(format, v...)) -} - -// Println implements part of the standard Logger interface. -func (t *TeeLogger) Println(v ...any) { - t.write(fmt.Sprintln(v...)) -} - -// Fatalf implements part of the standard Logger interface and exits the program. -func (t *TeeLogger) Fatalf(format string, v ...any) { - s := fmt.Sprintf(format, v...) - t.write(s) - os.Exit(1) -} - -// Fatal implements part of the standard Logger interface and exits the program. -func (t *TeeLogger) Fatal(v ...any) { - s := fmt.Sprintln(v...) - t.write(s) - os.Exit(1) -} - -// Writer implements part of the standard Logger interface, returning a multi-writer. -func (t *TeeLogger) Writer() io.Writer { - t.mu.RLock() - defer t.mu.RUnlock() - return io.MultiWriter(t.writers...) -} - -// Scoreboard returns the embedded ScoreboardAccess. -func (t *TeeLogger) Scoreboard() *Scoreboard { - return t.scoreboard -} - -// GetSucceededLogMap returns a copy of the succeeded log map. -func (t *TeeLogger) GetSucceededLogMap() map[string]string { - t.succeededMu.Lock() - defer t.succeededMu.Unlock() - // Return a copy to prevent external modification - copiedMap := make(map[string]string, len(t.succeededMap)) - for k, v := range t.succeededMap { - copiedMap[k] = v - } - return copiedMap -} - -// GetFailedLogMap returns a copy of the failed log map. -func (t *TeeLogger) GetFailedLogMap() map[string]common.RetryObject { - t.failedMu.Lock() - defer t.failedMu.Unlock() - // Return a copy to prevent external modification - copiedMap := make(map[string]common.RetryObject, len(t.FailedMap)) - for k, v := range t.FailedMap { - copiedMap[k] = v - } - return copiedMap -} - -func (t *TeeLogger) DeleteFromFailedLog(path string) { - t.failedMu.Lock() - defer t.failedMu.Unlock() - delete(t.FailedMap, path) -} - -// --- Internal Utility Methods --- - -// write handles writing the string to all configured writers. -func (t *TeeLogger) write(s string) { - t.mu.RLock() - defer t.mu.RUnlock() - for _, w := range t.writers { - _, _ = fmt.Fprint(w, s) - } -} - -func (t *TeeLogger) GetSucceededCount() int { - return len(t.succeededMap) -} - -func (t *TeeLogger) writeFailedSync(e common.RetryObject) { - t.failedMu.Lock() - defer t.failedMu.Unlock() - - // Store the FileMetadata part in the map - t.FailedMap[e.FilePath] = e - - data, _ := json.MarshalIndent(t.FailedMap, "", " ") - os.WriteFile(t.failedPath, data, 0644) -} - -func (t *TeeLogger) writeSucceededSync(path, guid string) { - t.succeededMu.Lock() - defer t.succeededMu.Unlock() - t.succeededMap[path] = guid - data, _ := json.MarshalIndent(t.succeededMap, "", " ") - os.WriteFile(t.succeededPath, data, 0644) -} - -// --- Tracking Methods (Part of Logger Interface) --- - -func (t *TeeLogger) Failed(filePath, filename string, metadata common.FileMetadata, guid string, retryCount int, multipart bool) { - if t.failedPath != "" { - t.writeFailedSync(common.RetryObject{ - FilePath: filePath, - Filename: filename, - FileMetadata: metadata, - GUID: guid, - RetryCount: retryCount, - Multipart: multipart, - }) - } -} - -func (t *TeeLogger) Succeeded(filePath, guid string) { - // Use t.succeededPath instead of checking the old global succeededPath - if t.succeededPath != "" { - t.writeSucceededSync(filePath, guid) - } -} diff --git a/client/mocks/mock_configure.go b/client/mocks/mock_configure.go deleted file mode 100644 index 697c3da..0000000 --- a/client/mocks/mock_configure.go +++ /dev/null @@ -1,145 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/calypr/data-client/client/jwt (interfaces: ConfigureInterface) -// -// Generated by this command: -// -// mockgen -destination=../mocks/mock_configure.go -package=mocks github.com/calypr/data-client/client/jwt ConfigureInterface -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - url "net/url" - reflect "reflect" - - jwt "github.com/calypr/data-client/client/jwt" - gomock "go.uber.org/mock/gomock" -) - -// MockConfigureInterface is a mock of ConfigureInterface interface. -type MockConfigureInterface struct { - ctrl *gomock.Controller - recorder *MockConfigureInterfaceMockRecorder - isgomock struct{} -} - -// MockConfigureInterfaceMockRecorder is the mock recorder for MockConfigureInterface. -type MockConfigureInterfaceMockRecorder struct { - mock *MockConfigureInterface -} - -// NewMockConfigureInterface creates a new mock instance. -func NewMockConfigureInterface(ctrl *gomock.Controller) *MockConfigureInterface { - mock := &MockConfigureInterface{ctrl: ctrl} - mock.recorder = &MockConfigureInterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockConfigureInterface) EXPECT() *MockConfigureInterfaceMockRecorder { - return m.recorder -} - -// GetConfigPath mocks base method. -func (m *MockConfigureInterface) GetConfigPath() (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetConfigPath") - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetConfigPath indicates an expected call of GetConfigPath. -func (mr *MockConfigureInterfaceMockRecorder) GetConfigPath() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetConfigPath", reflect.TypeOf((*MockConfigureInterface)(nil).GetConfigPath)) -} - -// IsValidCredential mocks base method. -func (m *MockConfigureInterface) IsValidCredential(arg0 jwt.Credential) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsValidCredential", arg0) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// IsValidCredential indicates an expected call of IsValidCredential. -func (mr *MockConfigureInterfaceMockRecorder) IsValidCredential(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsValidCredential", reflect.TypeOf((*MockConfigureInterface)(nil).IsValidCredential), arg0) -} - -// ParseConfig mocks base method. -func (m *MockConfigureInterface) ParseConfig(profile string) (jwt.Credential, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ParseConfig", profile) - ret0, _ := ret[0].(jwt.Credential) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ParseConfig indicates an expected call of ParseConfig. -func (mr *MockConfigureInterfaceMockRecorder) ParseConfig(profile any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParseConfig", reflect.TypeOf((*MockConfigureInterface)(nil).ParseConfig), profile) -} - -// ParseKeyValue mocks base method. -func (m *MockConfigureInterface) ParseKeyValue(str, expr string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ParseKeyValue", str, expr) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ParseKeyValue indicates an expected call of ParseKeyValue. -func (mr *MockConfigureInterfaceMockRecorder) ParseKeyValue(str, expr any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParseKeyValue", reflect.TypeOf((*MockConfigureInterface)(nil).ParseKeyValue), str, expr) -} - -// ReadFile mocks base method. -func (m *MockConfigureInterface) ReadFile(arg0, arg1 string) string { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ReadFile", arg0, arg1) - ret0, _ := ret[0].(string) - return ret0 -} - -// ReadFile indicates an expected call of ReadFile. -func (mr *MockConfigureInterfaceMockRecorder) ReadFile(arg0, arg1 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReadFile", reflect.TypeOf((*MockConfigureInterface)(nil).ReadFile), arg0, arg1) -} - -// UpdateConfigFile mocks base method. -func (m *MockConfigureInterface) UpdateConfigFile(arg0 jwt.Credential) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateConfigFile", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpdateConfigFile indicates an expected call of UpdateConfigFile. -func (mr *MockConfigureInterfaceMockRecorder) UpdateConfigFile(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateConfigFile", reflect.TypeOf((*MockConfigureInterface)(nil).UpdateConfigFile), arg0) -} - -// ValidateUrl mocks base method. -func (m *MockConfigureInterface) ValidateUrl(arg0 string) (*url.URL, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ValidateUrl", arg0) - ret0, _ := ret[0].(*url.URL) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ValidateUrl indicates an expected call of ValidateUrl. -func (mr *MockConfigureInterfaceMockRecorder) ValidateUrl(arg0 any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ValidateUrl", reflect.TypeOf((*MockConfigureInterface)(nil).ValidateUrl), arg0) -} diff --git a/client/mocks/mock_functions.go b/client/mocks/mock_functions.go deleted file mode 100644 index 6c48765..0000000 --- a/client/mocks/mock_functions.go +++ /dev/null @@ -1,135 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/calypr/data-client/client/jwt (interfaces: FunctionInterface) -// -// Generated by this command: -// -// mockgen -destination=../mocks/mock_functions.go -package=mocks github.com/calypr/data-client/client/jwt FunctionInterface -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - http "net/http" - url "net/url" - reflect "reflect" - - jwt "github.com/calypr/data-client/client/jwt" - gomock "go.uber.org/mock/gomock" -) - -// MockFunctionInterface is a mock of FunctionInterface interface. -type MockFunctionInterface struct { - ctrl *gomock.Controller - recorder *MockFunctionInterfaceMockRecorder - isgomock struct{} -} - -// MockFunctionInterfaceMockRecorder is the mock recorder for MockFunctionInterface. -type MockFunctionInterfaceMockRecorder struct { - mock *MockFunctionInterface -} - -// NewMockFunctionInterface creates a new mock instance. -func NewMockFunctionInterface(ctrl *gomock.Controller) *MockFunctionInterface { - mock := &MockFunctionInterface{ctrl: ctrl} - mock.recorder = &MockFunctionInterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockFunctionInterface) EXPECT() *MockFunctionInterfaceMockRecorder { - return m.recorder -} - -// CheckForShepherdAPI mocks base method. -func (m *MockFunctionInterface) CheckForShepherdAPI(profileConfig *jwt.Credential) (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckForShepherdAPI", profileConfig) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CheckForShepherdAPI indicates an expected call of CheckForShepherdAPI. -func (mr *MockFunctionInterfaceMockRecorder) CheckForShepherdAPI(profileConfig any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckForShepherdAPI", reflect.TypeOf((*MockFunctionInterface)(nil).CheckForShepherdAPI), profileConfig) -} - -// CheckPrivileges mocks base method. -func (m *MockFunctionInterface) CheckPrivileges(profileConfig *jwt.Credential) (string, map[string]any, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckPrivileges", profileConfig) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(map[string]any) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// CheckPrivileges indicates an expected call of CheckPrivileges. -func (mr *MockFunctionInterfaceMockRecorder) CheckPrivileges(profileConfig any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckPrivileges", reflect.TypeOf((*MockFunctionInterface)(nil).CheckPrivileges), profileConfig) -} - -// DoRequestWithSignedHeader mocks base method. -func (m *MockFunctionInterface) DoRequestWithSignedHeader(profileConfig *jwt.Credential, endpointPostPrefix, contentType string, bodyBytes []byte) (jwt.JsonMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DoRequestWithSignedHeader", profileConfig, endpointPostPrefix, contentType, bodyBytes) - ret0, _ := ret[0].(jwt.JsonMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DoRequestWithSignedHeader indicates an expected call of DoRequestWithSignedHeader. -func (mr *MockFunctionInterfaceMockRecorder) DoRequestWithSignedHeader(profileConfig, endpointPostPrefix, contentType, bodyBytes any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DoRequestWithSignedHeader", reflect.TypeOf((*MockFunctionInterface)(nil).DoRequestWithSignedHeader), profileConfig, endpointPostPrefix, contentType, bodyBytes) -} - -// GetHost mocks base method. -func (m *MockFunctionInterface) GetHost(profileConfig *jwt.Credential) (*url.URL, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetHost", profileConfig) - ret0, _ := ret[0].(*url.URL) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetHost indicates an expected call of GetHost. -func (mr *MockFunctionInterfaceMockRecorder) GetHost(profileConfig any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHost", reflect.TypeOf((*MockFunctionInterface)(nil).GetHost), profileConfig) -} - -// GetResponse mocks base method. -func (m *MockFunctionInterface) GetResponse(profileConfig *jwt.Credential, endpointPostPrefix, method, contentType string, bodyBytes []byte) (string, *http.Response, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetResponse", profileConfig, endpointPostPrefix, method, contentType, bodyBytes) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(*http.Response) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// GetResponse indicates an expected call of GetResponse. -func (mr *MockFunctionInterfaceMockRecorder) GetResponse(profileConfig, endpointPostPrefix, method, contentType, bodyBytes any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResponse", reflect.TypeOf((*MockFunctionInterface)(nil).GetResponse), profileConfig, endpointPostPrefix, method, contentType, bodyBytes) -} - -// ParseFenceURLResponse mocks base method. -func (m *MockFunctionInterface) ParseFenceURLResponse(resp *http.Response) (jwt.JsonMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ParseFenceURLResponse", resp) - ret0, _ := ret[0].(jwt.JsonMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// ParseFenceURLResponse indicates an expected call of ParseFenceURLResponse. -func (mr *MockFunctionInterfaceMockRecorder) ParseFenceURLResponse(resp any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParseFenceURLResponse", reflect.TypeOf((*MockFunctionInterface)(nil).ParseFenceURLResponse), resp) -} diff --git a/client/mocks/mock_gen3interface.go b/client/mocks/mock_gen3interface.go deleted file mode 100644 index 44dd849..0000000 --- a/client/mocks/mock_gen3interface.go +++ /dev/null @@ -1,180 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/calypr/data-client/client/gen3Client (interfaces: Gen3Interface) -// -// Generated by this command: -// -// mockgen -destination=../mocks/mock_gen3interface.go -package=mocks github.com/calypr/data-client/client/gen3Client Gen3Interface -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - bytes "bytes" - http "net/http" - url "net/url" - reflect "reflect" - - jwt "github.com/calypr/data-client/client/jwt" - logs "github.com/calypr/data-client/client/logs" - gomock "go.uber.org/mock/gomock" -) - -// MockGen3Interface is a mock of Gen3Interface interface. -type MockGen3Interface struct { - ctrl *gomock.Controller - recorder *MockGen3InterfaceMockRecorder - isgomock struct{} -} - -// MockGen3InterfaceMockRecorder is the mock recorder for MockGen3Interface. -type MockGen3InterfaceMockRecorder struct { - mock *MockGen3Interface -} - -// NewMockGen3Interface creates a new mock instance. -func NewMockGen3Interface(ctrl *gomock.Controller) *MockGen3Interface { - mock := &MockGen3Interface{ctrl: ctrl} - mock.recorder = &MockGen3InterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockGen3Interface) EXPECT() *MockGen3InterfaceMockRecorder { - return m.recorder -} - -// CheckForShepherdAPI mocks base method. -func (m *MockGen3Interface) CheckForShepherdAPI() (bool, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckForShepherdAPI") - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// CheckForShepherdAPI indicates an expected call of CheckForShepherdAPI. -func (mr *MockGen3InterfaceMockRecorder) CheckForShepherdAPI() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckForShepherdAPI", reflect.TypeOf((*MockGen3Interface)(nil).CheckForShepherdAPI)) -} - -// CheckPrivileges mocks base method. -func (m *MockGen3Interface) CheckPrivileges() (string, map[string]any, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CheckPrivileges") - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(map[string]any) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// CheckPrivileges indicates an expected call of CheckPrivileges. -func (mr *MockGen3InterfaceMockRecorder) CheckPrivileges() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckPrivileges", reflect.TypeOf((*MockGen3Interface)(nil).CheckPrivileges)) -} - -// DeleteRecord mocks base method. -func (m *MockGen3Interface) DeleteRecord(guid string) (string, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteRecord", guid) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DeleteRecord indicates an expected call of DeleteRecord. -func (mr *MockGen3InterfaceMockRecorder) DeleteRecord(guid any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRecord", reflect.TypeOf((*MockGen3Interface)(nil).DeleteRecord), guid) -} - -// DoRequestWithSignedHeader mocks base method. -func (m *MockGen3Interface) DoRequestWithSignedHeader(endpointPostPrefix, contentType string, bodyBytes []byte) (jwt.JsonMessage, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DoRequestWithSignedHeader", endpointPostPrefix, contentType, bodyBytes) - ret0, _ := ret[0].(jwt.JsonMessage) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DoRequestWithSignedHeader indicates an expected call of DoRequestWithSignedHeader. -func (mr *MockGen3InterfaceMockRecorder) DoRequestWithSignedHeader(endpointPostPrefix, contentType, bodyBytes any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DoRequestWithSignedHeader", reflect.TypeOf((*MockGen3Interface)(nil).DoRequestWithSignedHeader), endpointPostPrefix, contentType, bodyBytes) -} - -// GetCredential mocks base method. -func (m *MockGen3Interface) GetCredential() *jwt.Credential { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetCredential") - ret0, _ := ret[0].(*jwt.Credential) - return ret0 -} - -// GetCredential indicates an expected call of GetCredential. -func (mr *MockGen3InterfaceMockRecorder) GetCredential() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetCredential", reflect.TypeOf((*MockGen3Interface)(nil).GetCredential)) -} - -// GetHost mocks base method. -func (m *MockGen3Interface) GetHost() (*url.URL, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetHost") - ret0, _ := ret[0].(*url.URL) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetHost indicates an expected call of GetHost. -func (mr *MockGen3InterfaceMockRecorder) GetHost() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetHost", reflect.TypeOf((*MockGen3Interface)(nil).GetHost)) -} - -// GetResponse mocks base method. -func (m *MockGen3Interface) GetResponse(endpointPostPrefix, method, contentType string, bodyBytes []byte) (string, *http.Response, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetResponse", endpointPostPrefix, method, contentType, bodyBytes) - ret0, _ := ret[0].(string) - ret1, _ := ret[1].(*http.Response) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// GetResponse indicates an expected call of GetResponse. -func (mr *MockGen3InterfaceMockRecorder) GetResponse(endpointPostPrefix, method, contentType, bodyBytes any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResponse", reflect.TypeOf((*MockGen3Interface)(nil).GetResponse), endpointPostPrefix, method, contentType, bodyBytes) -} - -// Logger mocks base method. -func (m *MockGen3Interface) Logger() *logs.TeeLogger { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Logger") - ret0, _ := ret[0].(*logs.TeeLogger) - return ret0 -} - -// Logger indicates an expected call of Logger. -func (mr *MockGen3InterfaceMockRecorder) Logger() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Logger", reflect.TypeOf((*MockGen3Interface)(nil).Logger)) -} - -// MakeARequest mocks base method. -func (m *MockGen3Interface) MakeARequest(method, apiEndpoint, accessToken, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MakeARequest", method, apiEndpoint, accessToken, contentType, headers, body, noTimeout) - ret0, _ := ret[0].(*http.Response) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// MakeARequest indicates an expected call of MakeARequest. -func (mr *MockGen3InterfaceMockRecorder) MakeARequest(method, apiEndpoint, accessToken, contentType, headers, body, noTimeout any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakeARequest", reflect.TypeOf((*MockGen3Interface)(nil).MakeARequest), method, apiEndpoint, accessToken, contentType, headers, body, noTimeout) -} diff --git a/client/mocks/mock_request.go b/client/mocks/mock_request.go deleted file mode 100644 index 74f87de..0000000 --- a/client/mocks/mock_request.go +++ /dev/null @@ -1,87 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/calypr/data-client/client/jwt (interfaces: RequestInterface) -// -// Generated by this command: -// -// mockgen -destination=../mocks/mock_request.go -package=mocks github.com/calypr/data-client/client/jwt RequestInterface -// - -// Package mocks is a generated GoMock package. -package mocks - -import ( - bytes "bytes" - http "net/http" - reflect "reflect" - - jwt "github.com/calypr/data-client/client/jwt" - logs "github.com/calypr/data-client/client/logs" - gomock "go.uber.org/mock/gomock" -) - -// MockRequestInterface is a mock of RequestInterface interface. -type MockRequestInterface struct { - ctrl *gomock.Controller - recorder *MockRequestInterfaceMockRecorder - isgomock struct{} -} - -// MockRequestInterfaceMockRecorder is the mock recorder for MockRequestInterface. -type MockRequestInterfaceMockRecorder struct { - mock *MockRequestInterface -} - -// NewMockRequestInterface creates a new mock instance. -func NewMockRequestInterface(ctrl *gomock.Controller) *MockRequestInterface { - mock := &MockRequestInterface{ctrl: ctrl} - mock.recorder = &MockRequestInterfaceMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockRequestInterface) EXPECT() *MockRequestInterfaceMockRecorder { - return m.recorder -} - -// Logger mocks base method. -func (m *MockRequestInterface) Logger() logs.Logger { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Logger") - ret0, _ := ret[0].(logs.Logger) - return ret0 -} - -// Logger indicates an expected call of Logger. -func (mr *MockRequestInterfaceMockRecorder) Logger() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Logger", reflect.TypeOf((*MockRequestInterface)(nil).Logger)) -} - -// MakeARequest mocks base method. -func (m *MockRequestInterface) MakeARequest(method, apiEndpoint, accessToken, contentType string, headers map[string]string, body *bytes.Buffer, noTimeout bool) (*http.Response, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "MakeARequest", method, apiEndpoint, accessToken, contentType, headers, body, noTimeout) - ret0, _ := ret[0].(*http.Response) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// MakeARequest indicates an expected call of MakeARequest. -func (mr *MockRequestInterfaceMockRecorder) MakeARequest(method, apiEndpoint, accessToken, contentType, headers, body, noTimeout any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakeARequest", reflect.TypeOf((*MockRequestInterface)(nil).MakeARequest), method, apiEndpoint, accessToken, contentType, headers, body, noTimeout) -} - -// RequestNewAccessToken mocks base method. -func (m *MockRequestInterface) RequestNewAccessToken(accessTokenEndpoint string, profileConfig *jwt.Credential) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RequestNewAccessToken", accessTokenEndpoint, profileConfig) - ret0, _ := ret[0].(error) - return ret0 -} - -// RequestNewAccessToken indicates an expected call of RequestNewAccessToken. -func (mr *MockRequestInterfaceMockRecorder) RequestNewAccessToken(accessTokenEndpoint, profileConfig any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequestNewAccessToken", reflect.TypeOf((*MockRequestInterface)(nil).RequestNewAccessToken), accessTokenEndpoint, profileConfig) -} diff --git a/cmd/auth.go b/cmd/auth.go new file mode 100644 index 0000000..c9de759 --- /dev/null +++ b/cmd/auth.go @@ -0,0 +1,203 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "io" + "sort" + "strings" + + "github.com/calypr/data-client/g3client" + "github.com/calypr/data-client/logs" + "github.com/spf13/cobra" +) + +const authSummaryResourceLimit = 10 + +func init() { + var profile string + var showAll bool + var jsonOutput bool + var authCmd = &cobra.Command{ + Use: "auth", + Short: "Return resource access privileges from profile", + Long: `Gets resource access privileges for specified profile.`, + Example: `./data-client auth --profile= +./data-client auth --profile= --all +./data-client auth --profile= --json`, + RunE: func(cmd *cobra.Command, args []string) error { + // don't initialize transmission logs for non-uploading related commands + + logger, logCloser := logs.New(profile, logs.WithNoConsole()) + defer logCloser() + + g3i, err := g3client.NewGen3Interface( + profile, logger, + g3client.WithClients(g3client.FenceClient), + ) + if err != nil { + return fmt.Errorf("new Gen3 interface: %w", err) + } + + resourceAccess, err := g3i.FenceClient().CheckPrivileges(context.Background()) + if err != nil { + return fmt.Errorf("authentication: %w", err) + } + + if jsonOutput { + encoder := json.NewEncoder(cmd.OutOrStdout()) + encoder.SetIndent("", " ") + return encoder.Encode(resourceAccess) + } + + writeAuthSummary(cmd.OutOrStdout(), g3i.Credentials().Current().APIEndpoint, resourceAccess, showAll) + return nil + }, + } + + authCmd.Flags().StringVar(&profile, "profile", "", "Specify the profile to check your access privileges") + authCmd.Flags().BoolVar(&showAll, "all", false, "Show every resource in each permission group") + authCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output raw resource access JSON") + authCmd.MarkFlagRequired("profile") // nolint: errcheck + RootCmd.AddCommand(authCmd) +} + +func writeAuthSummary(w io.Writer, endpoint string, resourceAccess map[string]any, showAll bool) { + if len(resourceAccess) == 0 { + fmt.Fprintf(w, "No resource access found for %s\n", endpoint) + return + } + + groups := make(map[string][]string) + for resource, permissions := range resourceAccess { + signature := authPermissionSignature(permissions) + groups[signature] = append(groups[signature], resource) + } + + type accessGroup struct { + signature string + resources []string + } + + orderedGroups := make([]accessGroup, 0, len(groups)) + for signature, resources := range groups { + sort.Strings(resources) + orderedGroups = append(orderedGroups, accessGroup{ + signature: signature, + resources: resources, + }) + } + + sort.Slice(orderedGroups, func(i, j int) bool { + if len(orderedGroups[i].resources) != len(orderedGroups[j].resources) { + return len(orderedGroups[i].resources) > len(orderedGroups[j].resources) + } + return orderedGroups[i].signature < orderedGroups[j].signature + }) + + fmt.Fprintf(w, "Access for %s\n", endpoint) + fmt.Fprintf(w, "%d resources in %d permission groups\n\n", len(resourceAccess), len(orderedGroups)) + + for _, group := range orderedGroups { + fmt.Fprintf(w, "%d %s: %s\n", len(group.resources), pluralize("resource", len(group.resources)), group.signature) + + limit := len(group.resources) + if !showAll && limit > authSummaryResourceLimit { + limit = authSummaryResourceLimit + } + + for _, resource := range group.resources[:limit] { + fmt.Fprintf(w, " %s\n", resource) + } + if !showAll && len(group.resources) > limit { + fmt.Fprintf(w, " ... %d more (use --all to show every resource)\n", len(group.resources)-limit) + } + fmt.Fprintln(w) + } +} + +func pluralize(word string, count int) string { + if count == 1 { + return word + } + return word + "s" +} + +func authPermissionSignature(value any) string { + permissions, ok := value.([]any) + if !ok { + return compactJSON(value) + } + if len(permissions) == 0 { + return "no permissions" + } + + if _, ok := permissions[0].(string); ok { + access := make([]string, 0, len(permissions)) + for _, permission := range permissions { + access = append(access, fmt.Sprint(permission)) + } + sort.Strings(access) + return strings.Join(access, ", ") + } + + serviceMethods := make(map[string]map[string]struct{}) + unknown := make([]string, 0) + + for _, permission := range permissions { + method, service, ok := authPermissionFields(permission) + if !ok { + unknown = append(unknown, compactJSON(permission)) + continue + } + + if serviceMethods[service] == nil { + serviceMethods[service] = make(map[string]struct{}) + } + serviceMethods[service][method] = struct{}{} + } + + parts := make([]string, 0, len(serviceMethods)+len(unknown)) + services := make([]string, 0, len(serviceMethods)) + for service := range serviceMethods { + services = append(services, service) + } + sort.Strings(services) + + for _, service := range services { + methods := make([]string, 0, len(serviceMethods[service])) + for method := range serviceMethods[service] { + methods = append(methods, method) + } + sort.Strings(methods) + parts = append(parts, fmt.Sprintf("%s: %s", service, strings.Join(methods, ", "))) + } + + sort.Strings(unknown) + parts = append(parts, unknown...) + return strings.Join(parts, "; ") +} + +func authPermissionFields(value any) (method string, service string, ok bool) { + switch permission := value.(type) { + case map[string]any: + method, methodOK := permission["method"].(string) + service, serviceOK := permission["service"].(string) + return method, service, methodOK && serviceOK + case map[string]string: + method, methodOK := permission["method"] + service, serviceOK := permission["service"] + return method, service, methodOK && serviceOK + default: + return "", "", false + } +} + +func compactJSON(value any) string { + data, err := json.Marshal(value) + if err != nil { + return fmt.Sprint(value) + } + return string(data) +} diff --git a/cmd/auth_test.go b/cmd/auth_test.go new file mode 100644 index 0000000..857f74b --- /dev/null +++ b/cmd/auth_test.go @@ -0,0 +1,81 @@ +package cmd + +import ( + "bytes" + "strings" + "testing" +) + +func TestAuthPermissionSignatureGroupsArboristPermissions(t *testing.T) { + signature := authPermissionSignature([]any{ + map[string]any{"method": "read", "service": "*"}, + map[string]any{"method": "create", "service": "*"}, + map[string]any{"method": "*", "service": "indexd"}, + map[string]any{"method": "read", "service": "requestor"}, + }) + + want := "*: create, read; indexd: *; requestor: read" + if signature != want { + t.Fatalf("signature = %q, want %q", signature, want) + } +} + +func TestAuthPermissionSignatureGroupsFenceProjectAccess(t *testing.T) { + signature := authPermissionSignature([]any{"write", "read", "read"}) + + want := "read, read, write" + if signature != want { + t.Fatalf("signature = %q, want %q", signature, want) + } +} + +func TestWriteAuthSummaryCondensesRepeatedPermissionSets(t *testing.T) { + resourceAccess := map[string]any{ + "/programs/a/projects/one": []any{ + map[string]any{"method": "read", "service": "*"}, + map[string]any{"method": "create", "service": "*"}, + }, + "/programs/a/projects/two": []any{ + map[string]any{"method": "create", "service": "*"}, + map[string]any{"method": "read", "service": "*"}, + }, + "/data_file": []any{ + map[string]any{"method": "*", "service": "indexd"}, + }, + } + + var out bytes.Buffer + writeAuthSummary(&out, "https://example.org", resourceAccess, false) + got := out.String() + + for _, want := range []string{ + "Access for https://example.org", + "3 resources in 2 permission groups", + "2 resources: *: create, read", + " /programs/a/projects/one", + " /programs/a/projects/two", + "1 resource: indexd: *", + " /data_file", + } { + if !strings.Contains(got, want) { + t.Fatalf("summary missing %q:\n%s", want, got) + } + } +} + +func TestWriteAuthSummaryCapsLongGroups(t *testing.T) { + resourceAccess := make(map[string]any) + for i := 0; i < authSummaryResourceLimit+2; i++ { + resourceAccess[string(rune('a'+i))] = []any{ + map[string]any{"method": "read", "service": "*"}, + } + } + + var out bytes.Buffer + writeAuthSummary(&out, "https://example.org", resourceAccess, false) + got := out.String() + + if !strings.Contains(got, "... 2 more (use --all to show every resource)") { + t.Fatalf("summary did not cap long group:\n%s", got) + } +} diff --git a/cmd/collaborator.go b/cmd/collaborator.go new file mode 100644 index 0000000..545affe --- /dev/null +++ b/cmd/collaborator.go @@ -0,0 +1,324 @@ +package cmd + +import ( + "fmt" + "os" + + "regexp" + "strings" + + "github.com/calypr/data-client/g3client" + "github.com/calypr/data-client/logs" + "github.com/calypr/data-client/requestor" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) + +var collaboratorsCmd = &cobra.Command{ + Use: "collaborators", + Short: "Manage collaborators and access requests", +} + +var emailRegex = regexp.MustCompile(`^[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,}$`) + +func printRequest(r requestor.Request) { + b, err := yaml.Marshal(r) + if err != nil { + fmt.Printf("ID: %s (Error formatting details: %v)\n", r.RequestID, err) + return + } + fmt.Println(string(b)) +} + +func getRequestorClient(localProfile string) (requestor.RequestorInterface, func()) { + if localProfile == "" { + fmt.Println("Error: profile is required.") + os.Exit(1) + } + + // Initialize logger + logger, logCloser := logs.New(localProfile) + + // Initialize base Gen3 interface and build requestor client from it. + g3i, err := g3client.NewGen3Interface(localProfile, logger) + if err != nil { + fmt.Printf("Error accessing Gen3: %v\n", err) + logCloser() + os.Exit(1) + } + + return requestor.NewRequestorClient(g3i, g3i.Credentials().Current()), logCloser +} + +var collaboratorListCmd = &cobra.Command{ + Use: "ls [profile]", + Short: "List requests", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + mine, _ := cmd.Flags().GetBool("mine") + active, _ := cmd.Flags().GetBool("active") + username, _ := cmd.Flags().GetString("username") + + client, closer := getRequestorClient(p) + defer closer() + + requests, err := client.ListRequests(cmd.Context(), mine, active, username) + if err != nil { + fmt.Printf("Error listing requests: %v\n", err) + os.Exit(1) + } + + for _, r := range requests { + printRequest(r) + } + }, +} + +var collaboratorPendingCmd = &cobra.Command{ + Use: "pending [profile]", + Short: "List pending requests", + Args: cobra.ExactArgs(1), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + client, closer := getRequestorClient(p) + defer closer() + + // Fetch all requests + requests, err := client.ListRequests(cmd.Context(), false, false, "") + if err != nil { + fmt.Printf("Error listing requests: %v\n", err) + os.Exit(1) + } + + fmt.Println("Pending requests:") + for _, r := range requests { + if r.Status != "SIGNED" { + printRequest(r) + } + } + }, +} + +var collaboratorAddUserCmd = &cobra.Command{ + Use: "add [profile] [email] [program] [project]", + Short: "Add a user to a project", + Args: cobra.ExactArgs(4), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + username := args[1] + program := args[2] + project := args[3] + projectID := fmt.Sprintf("%s-%s", program, project) + + if !emailRegex.MatchString(strings.ToLower(username)) { + fmt.Printf("Error: invalid email address '%s'\n", username) + os.Exit(1) + } + + write, _ := cmd.Flags().GetBool("write") + guppy, _ := cmd.Flags().GetBool("guppy") + approve, _ := cmd.Flags().GetBool("approve") + + client, closer := getRequestorClient(p) + defer closer() + + reqs, err := client.AddUser(cmd.Context(), projectID, username, write, guppy) + if err != nil { + fmt.Printf("Error adding user: %v\n", err) + os.Exit(1) + } + + if approve { + fmt.Println("\nAuto-approving requests...") + for _, r := range reqs { + updatedReq, err := client.UpdateRequest(cmd.Context(), r.RequestID, "SIGNED") + if err != nil { + fmt.Printf("Error approving request %s: %v\n", r.RequestID, err) + } else { + fmt.Printf("Approved request %s:\n", updatedReq.RequestID) + printRequest(*updatedReq) + } + } + } else { + fmt.Println("Created requests:") + for _, r := range reqs { + printRequest(r) + } + fmt.Printf("\nAn authorized user must approve these requests to add %s to %s\n", username, projectID) + } + }, +} + +var collaboratorBulkAddUserCmd = &cobra.Command{ + Use: "bulk-add [profile] [email] [resource_paths...]", + Short: "Add a user to multiple project resources", + Long: "Add a user to multiple project resources from a comma-delimited list. Resource paths may be /programs//projects/ or organization/project.", + Args: cobra.MinimumNArgs(3), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + username := args[1] + resourceList := strings.Join(args[2:], " ") + + if !emailRegex.MatchString(strings.ToLower(username)) { + fmt.Printf("Error: invalid email address '%s'\n", username) + os.Exit(1) + } + + resources, err := requestor.ParseProjectResources(resourceList) + if err != nil { + fmt.Printf("Error parsing resource paths: %v\n", err) + os.Exit(1) + } + + write, _ := cmd.Flags().GetBool("write") + guppy, _ := cmd.Flags().GetBool("guppy") + approve, _ := cmd.Flags().GetBool("approve") + + client, closer := getRequestorClient(p) + defer closer() + + fmt.Printf("Creating collaborator requests for %d project resources...\n", len(resources)) + reqs, err := client.AddUserToResources(cmd.Context(), resources, username, write, guppy) + if err != nil { + fmt.Printf("Error adding user to resources: %v\n", err) + os.Exit(1) + } + + if approve { + fmt.Println("\nAuto-approving requests...") + for _, r := range reqs { + updatedReq, err := client.UpdateRequest(cmd.Context(), r.RequestID, "SIGNED") + if err != nil { + fmt.Printf("Error approving request %s: %v\n", r.RequestID, err) + } else { + fmt.Printf("Approved request %s:\n", updatedReq.RequestID) + printRequest(*updatedReq) + } + } + } else { + fmt.Println("Created requests:") + for _, r := range reqs { + printRequest(r) + } + fmt.Printf("\nAn authorized user must approve these requests to add %s to %d project resources\n", username, len(resources)) + } + }, +} + +var collaboratorRemoveUserCmd = &cobra.Command{ + Use: "rm [profile] [email] [program] [project]", + Short: "Remove a user from a project", + Args: cobra.ExactArgs(4), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + username := args[1] + program := args[2] + project := args[3] + projectID := fmt.Sprintf("%s-%s", program, project) + + if !emailRegex.MatchString(strings.ToLower(username)) { + fmt.Printf("Error: invalid email address '%s'\n", username) + os.Exit(1) + } + + approve, _ := cmd.Flags().GetBool("approve") + + client, closer := getRequestorClient(p) + defer closer() + + reqs, err := client.RemoveUser(cmd.Context(), projectID, username) + if err != nil { + fmt.Printf("Error removing user: %v\n", err) + os.Exit(1) + } + + if approve { + fmt.Println("\nAuto-approving revoke requests...") + for _, r := range reqs { + updatedReq, err := client.UpdateRequest(cmd.Context(), r.RequestID, "SIGNED") + if err != nil { + fmt.Printf("Error approving request %s: %v\n", r.RequestID, err) + } else { + fmt.Printf("Approved request %s:\n", updatedReq.RequestID) + printRequest(*updatedReq) + } + } + } else { + fmt.Println("Created revoke requests:") + for _, r := range reqs { + printRequest(r) + } + } + }, +} + +var collaboratorApproveCmd = &cobra.Command{ + Use: "approve [profile] [request_id]", + Short: "Approve a request (sign it)", + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + requestID := args[1] + + client, closer := getRequestorClient(p) + defer closer() + + req, err := client.UpdateRequest(cmd.Context(), requestID, "SIGNED") + if err != nil { + fmt.Printf("Error approving request: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Approved request %s\n", req.RequestID) + printRequest(*req) + }, +} + +var collaboratorUpdateCmd = &cobra.Command{ + Use: "update [profile] [request_id] [status]", + Short: "Update a request status", + Hidden: true, + Args: cobra.ExactArgs(3), + Run: func(cmd *cobra.Command, args []string) { + p := args[0] + requestID := args[1] + status := args[2] + + client, closer := getRequestorClient(p) + defer closer() + + req, err := client.UpdateRequest(cmd.Context(), requestID, status) + if err != nil { + fmt.Printf("Error updating request: %v\n", err) + os.Exit(1) + } + + fmt.Printf("Updated request %s to status %s\n", req.RequestID, req.Status) + }, +} + +func init() { + RootCmd.AddCommand(collaboratorsCmd) + collaboratorsCmd.AddCommand(collaboratorListCmd) + collaboratorsCmd.AddCommand(collaboratorPendingCmd) + collaboratorsCmd.AddCommand(collaboratorAddUserCmd) + collaboratorsCmd.AddCommand(collaboratorBulkAddUserCmd) + collaboratorsCmd.AddCommand(collaboratorRemoveUserCmd) + collaboratorsCmd.AddCommand(collaboratorApproveCmd) + collaboratorsCmd.AddCommand(collaboratorUpdateCmd) + + collaboratorListCmd.Flags().Bool("mine", false, "List my requests") + collaboratorListCmd.Flags().Bool("active", false, "List only active requests") + collaboratorListCmd.Flags().String("username", "", "List requests for user") + + collaboratorAddUserCmd.Flags().BoolP("write", "w", false, "Grant write access") + collaboratorAddUserCmd.Flags().BoolP("guppy", "g", false, "Grant guppy admin access") + collaboratorAddUserCmd.Flags().BoolP("approve", "a", false, "Automatically approve the requests") + + collaboratorBulkAddUserCmd.Flags().BoolP("write", "w", false, "Grant write access") + collaboratorBulkAddUserCmd.Flags().BoolP("guppy", "g", false, "Grant guppy admin access") + collaboratorBulkAddUserCmd.Flags().BoolP("approve", "a", false, "Automatically approve the requests") + + collaboratorRemoveUserCmd.Flags().BoolP("approve", "a", false, "Automatically approve the revoke requests") +} diff --git a/client/g3cmd/configure.go b/cmd/configure.go similarity index 73% rename from client/g3cmd/configure.go rename to cmd/configure.go index d9d97e2..0b2165f 100644 --- a/client/g3cmd/configure.go +++ b/cmd/configure.go @@ -1,14 +1,27 @@ -package g3cmd +package cmd import ( + "context" "fmt" - "github.com/calypr/data-client/client/common" - "github.com/calypr/data-client/client/jwt" - "github.com/calypr/data-client/client/logs" + "github.com/calypr/data-client/conf" + "github.com/calypr/data-client/g3client" + "github.com/calypr/data-client/logs" "github.com/spf13/cobra" ) +func mergeImportedCredential(target *conf.Credential, imported *conf.Credential) { + if target == nil || imported == nil { + return + } + target.KeyID = imported.KeyID + target.APIKey = imported.APIKey + if target.APIEndpoint == "" && imported.APIEndpoint != "" { + target.APIEndpoint = imported.APIEndpoint + } + target.AccessToken = "" +} + func init() { var profile string var credFile string @@ -24,7 +37,7 @@ func init() { Example: `./data-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org`, Run: func(cmd *cobra.Command, args []string) { // don't initialize transmission logs for non-uploading related commands - cred := &jwt.Credential{ + cred := &conf.Credential{ Profile: profile, APIEndpoint: apiEndpoint, AccessToken: fenceToken, @@ -34,21 +47,17 @@ func init() { logger, logCloser := logs.New(profile, logs.WithConsole()) defer logCloser() - conf := jwt.Configure{Logs: logger} - + configure := conf.NewConfigure(logger.Logger) if credFile != "" { - readCred, err := conf.ReadCredentials(credFile, "") + readCred, err := configure.Import(credFile, "") if err != nil { logger.Fatal(err) // or return proper error } - cred.KeyId = readCred.KeyId - cred.APIKey = readCred.APIKey - if readCred.APIEndpoint != "" { - cred.APIEndpoint = readCred.APIEndpoint - } - cred.AccessToken = "" + mergeImportedCredential(cred, readCred) } - err := jwt.UpdateConfig(logger, cred) + + g3i := g3client.NewGen3InterfaceFromCredential(cred, logger, g3client.WithClients()) + err := g3i.Credentials().Export(context.Background(), cred) if err != nil { logger.Println(err.Error()) } @@ -63,7 +72,7 @@ func init() { configureCmd.Flags().StringVar(&fenceToken, "fenceToken", "", "Specify the fence token to use as a substitute for credential file") configureCmd.Flags().StringVar(&apiEndpoint, "apiendpoint", "", "Specify the API endpoint of the data commons") configureCmd.MarkFlagRequired("apiendpoint") //nolint:errcheck - configureCmd.Flags().StringVar(&useShepherd, "use-shepherd", "", fmt.Sprintf("Enables or disables support for the Shepherd API. If enabled, gen3client will use the Shepherd API if available. (Default: %v)", common.DefaultUseShepherd)) - configureCmd.Flags().StringVar(&minShepherdVersion, "min-shepherd-version", "", fmt.Sprintf("Specify the minimum version of Shepherd that the gen3client will use if Shepherd is enabled. (Default: %v)", common.DefaultMinShepherdVersion)) + configureCmd.Flags().StringVar(&useShepherd, "use-shepherd", "", fmt.Sprintf("Enables or disables support for the Shepherd API. If enabled, gen3client will use the Shepherd API if available. (Default: %v)", false)) + configureCmd.Flags().StringVar(&minShepherdVersion, "min-shepherd-version", "", fmt.Sprintf("Specify the minimum version of Shepherd that the gen3client will use if Shepherd is enabled. (Default: %v)", "2.0.0")) RootCmd.AddCommand(configureCmd) } diff --git a/cmd/configure_test.go b/cmd/configure_test.go new file mode 100644 index 0000000..a8ce1dd --- /dev/null +++ b/cmd/configure_test.go @@ -0,0 +1,46 @@ +package cmd + +import ( + "testing" + + "github.com/calypr/data-client/conf" +) + +func TestMergeImportedCredentialPreservesExplicitAPIEndpoint(t *testing.T) { + target := &conf.Credential{ + Profile: "dev", + APIEndpoint: "https://explicit.example.org", + AccessToken: "stale-token", + } + imported := &conf.Credential{ + KeyID: "key-id", + APIKey: "api-key", + APIEndpoint: "https://embedded.example.org", + } + + mergeImportedCredential(target, imported) + + if target.KeyID != "key-id" { + t.Fatalf("KeyID = %q, want %q", target.KeyID, "key-id") + } + if target.APIKey != "api-key" { + t.Fatalf("APIKey = %q, want %q", target.APIKey, "api-key") + } + if target.APIEndpoint != "https://explicit.example.org" { + t.Fatalf("APIEndpoint = %q, want explicit endpoint", target.APIEndpoint) + } + if target.AccessToken != "" { + t.Fatalf("AccessToken = %q, want empty", target.AccessToken) + } +} + +func TestMergeImportedCredentialUsesImportedAPIEndpointWhenMissing(t *testing.T) { + target := &conf.Credential{Profile: "dev"} + imported := &conf.Credential{APIEndpoint: "https://embedded.example.org"} + + mergeImportedCredential(target, imported) + + if target.APIEndpoint != "https://embedded.example.org" { + t.Fatalf("APIEndpoint = %q, want %q", target.APIEndpoint, "https://embedded.example.org") + } +} diff --git a/cmd/download-multipart.go b/cmd/download-multipart.go new file mode 100644 index 0000000..3718720 --- /dev/null +++ b/cmd/download-multipart.go @@ -0,0 +1,261 @@ +package cmd + +/* +// DownloadSignedURL downloads a file from a signed URL with: +// - Resumable single-stream download (if partial file exists) +// - Concurrent multipart download for large files (>1GB) +// - Retries via go-retryablehttp +// - Progress bar support via mpb +func DownloadSignedURL(signedURL, dstPath string) error { + // Setup retryable client + retryClient := retryablehttp.NewClient() + retryClient.RetryMax = 10 + retryClient.RetryWaitMin = 1 * time.Second + retryClient.RetryWaitMax = 30 * time.Second + retryClient.Logger = nil // silent + client := retryClient.StandardClient() + client.Timeout = 0 // no timeout for large downloads + + // HEAD to get size and Accept-Ranges support + headResp, err := client.Head(signedURL) + if err != nil { + return fmt.Errorf("HEAD request failed: %w", err) + } + defer headResp.Body.Close() + + if headResp.StatusCode != http.StatusOK { + return fmt.Errorf("HEAD failed: %s", headResp.Status) + } + + contentLength := headResp.ContentLength + if contentLength <= 0 { + return fmt.Errorf("invalid Content-Length") + } + + acceptRanges := headResp.Header.Get("Accept-Ranges") == "bytes" + if !acceptRanges { + return fmt.Errorf("server does not support range requests") + } + + // Ensure directory exists + if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { + return fmt.Errorf("mkdir failed: %w", err) + } + + // Check if partial file exists + stat, _ := os.Stat(dstPath) + existingSize := int64(0) + if stat != nil { + existingSize = stat.Size() + } + + // If we have a partial file, resume with single stream (safer and simpler) + if existingSize > 0 && existingSize < contentLength { + return downloadResumableSingle(signedURL, dstPath, contentLength, existingSize, client) + } + + // For complete downloads: use multipart if file is large enough + if contentLength >= 5*1024*1024*1024 { + return downloadConcurrentMultipart(signedURL, dstPath, contentLength, client) + } + + // Otherwise: simple single-stream download + return downloadResumableSingle(signedURL, dstPath, contentLength, 0, client) +} + +// downloadResumableSingle handles single-stream (possibly resumed) download +func downloadResumableSingle(signedURL, dstPath string, totalSize, startByte int64, client *http.Client) error { + req, err := http.NewRequest("GET", signedURL, nil) + if err != nil { + return err + } + if startByte > 0 { + req.Header.Set("Range", fmt.Sprintf("bytes=%d-", startByte)) + } + + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("GET failed: %w", err) + } + defer resp.Body.Close() + + if startByte > 0 && resp.StatusCode != http.StatusPartialContent { + return fmt.Errorf("expected 206 Partial Content, got %d", resp.StatusCode) + } + if startByte == 0 && resp.StatusCode != http.StatusOK { + return fmt.Errorf("expected 200 OK, got %d", resp.StatusCode) + } + + file, err := os.OpenFile(dstPath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer file.Close() + + if startByte > 0 { + if _, err := file.Seek(startByte, io.SeekStart); err != nil { + return err + } + } else { + if err := file.Truncate(0); err != nil { + return err + } + } + + var writer io.Writer = file + if progress != nil { + bar := progress.AddBar(totalSize, + mpb.PrependDecorators( + decor.Name(filepath.Base(dstPath)+" "), + decor.CountersKibiByte("% .1f / % .1f"), + ), + mpb.AppendDecorators( + decor.Percentage(), + decor.AverageSpeed(decor.SizeB1024(0), "% .1f"), + ), + ) + if startByte > 0 { + bar.SetCurrent(startByte) + } + writer = bar.ProxyWriter(file) + } + + _, err = io.Copy(writer, resp.Body) + return err +} + +// downloadConcurrentMultipart downloads in parallel chunks +func downloadConcurrentMultipart(signedURL, dstPath string, totalSize int64, client *http.Client) error { + numChunks := int((totalSize + chunkSize - 1) / chunkSize) + if numChunks < defaultConcurrency { + numChunks = defaultConcurrency + } + chunkSizeActual := (totalSize + int64(numChunks) - 1) / int64(numChunks) + + // Pre-allocate file + file, err := os.Create(dstPath) + if err != nil { + return err + } + if err := file.Truncate(totalSize); err != nil { + file.Close() + return err + } + file.Close() + + var wg sync.WaitGroup + var mu sync.Mutex + var downloadErr error + + // Shared progress bar for total + var totalBar *mpb.Bar + if progress != nil { + totalBar = progress.AddBar(totalSize, + mpb.PrependDecorators( + decor.Name(filepath.Base(dstPath)+" (multipart) "), + decor.CountersKibiByte("% .1f / % .1f"), + ), + mpb.AppendDecorators( + decor.Percentage(), + decor.AverageSpeed(decor.SizeB1024(0), "% .1f"), + ), + ) + } + + concurrency := defaultConcurrency + sem := make(chan struct{}, concurrency) + + for i := 0; i < int(numChunks); i++ { + start := int64(i) * chunkSizeActual + end := start + chunkSizeActual - 1 + if end >= totalSize { + end = totalSize - 1 + } + if start > end { + break + } + + wg.Add(1) + sem <- struct{}{} + + go func(start, end int64, chunkIdx int) { + defer wg.Done() + defer func() { <-sem }() + + req, _ := http.NewRequest("GET", signedURL, nil) + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + + resp, err := client.Do(req) + if err != nil { + mu.Lock() + if downloadErr == nil { + downloadErr = fmt.Errorf("chunk %d failed: %w", chunkIdx, err) + } + mu.Unlock() + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusPartialContent { + mu.Lock() + if downloadErr == nil { + downloadErr = fmt.Errorf("chunk %d expected 206, got %d", chunkIdx, resp.StatusCode) + } + mu.Unlock() + return + } + + file, err := os.OpenFile(dstPath, os.O_WRONLY, 0644) + if err != nil { + mu.Lock() + downloadErr = err + mu.Unlock() + return + } + file.Seek(start, io.SeekStart) + writer := io.Writer(file) + + var chunkWriter io.Writer = writer + if progress != nil { + chunkBar := progress.AddBar(end-start+1, + mpb.BarRemoveOnComplete(), + mpb.PrependDecorators(decor.Name(fmt.Sprintf("chunk %d ", chunkIdx))), + ) + chunkWriter = chunkBar.ProxyWriter(writer) + defer file.Close() + } + + if _, err := io.Copy(chunkWriter, resp.Body); err != nil { + mu.Lock() + if downloadErr == nil { + downloadErr = fmt.Errorf("chunk %d copy failed: %w", chunkIdx, err) + } + mu.Unlock() + } else { + if totalBar != nil { + totalBar.IncrBy(int(end - start + 1)) + } + } + if progress == nil { + file.Close() + } + }(start, end, i) + } + + wg.Wait() + + if downloadErr != nil { + if totalBar != nil { + totalBar.Abort(true) + } + return downloadErr + } + + if totalBar != nil { + totalBar.SetCurrent(totalSize) + } + + return nil +} + +*/ diff --git a/cmd/download-multiple.go b/cmd/download-multiple.go new file mode 100644 index 0000000..bd17e80 --- /dev/null +++ b/cmd/download-multiple.go @@ -0,0 +1,84 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + + "github.com/calypr/data-client/g3client" + "github.com/calypr/data-client/logs" + sydownload "github.com/calypr/syfon/client/transfer/download" + "github.com/spf13/cobra" +) + +func init() { + var manifestPath string + var downloadPath string + var numParallel int + var profile string + + var downloadMultipleCmd = &cobra.Command{ + Use: "download-multiple", + Short: "Download multiple of files from a specified manifest", + Long: `Get presigned URLs for multiple of files specified in a manifest file and then download all of them.`, + Example: `./data-client download-multiple --profile --manifest --download-path `, + Run: func(cmd *cobra.Command, args []string) { + logger, logCloser := logs.New(profile, logs.WithConsole(), logs.WithFailedLog(), logs.WithScoreboard(), logs.WithSucceededLog()) + defer logCloser() + + g3i, err := g3client.NewGen3Interface(profile, logger) + if err != nil { + log.Fatalf("Failed to parse config on profile %s, %v", profile, err) + } + + guids, err := loadManifestGuids(manifestPath) + if err != nil { + log.Fatalf("Failed to read manifest %s: %v", manifestPath, err) + } + + syfon := g3i.SyfonClient() + if syfon == nil { + logger.Fatal("failed to initialize syfon client") + } + err = sydownload.DownloadMultiple(context.Background(), syfon.DRS(), syfon.Data(), guids, downloadPath, numParallel, false) + if err != nil { + logger.Fatal(err.Error()) + } + }, + } + + downloadMultipleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") + downloadMultipleCmd.MarkFlagRequired("profile") //nolint:errcheck + downloadMultipleCmd.Flags().StringVar(&manifestPath, "manifest", "", "The manifest file to read from. A valid manifest can be acquired by using the \"Download Manifest\" button in Data Explorer from a data common's portal") + downloadMultipleCmd.MarkFlagRequired("manifest") //nolint:errcheck + downloadMultipleCmd.Flags().StringVar(&downloadPath, "download-path", ".", "The directory in which to store the downloaded files") + downloadMultipleCmd.Flags().IntVar(&numParallel, "numparallel", 1, "Number of downloads to run in parallel") + RootCmd.AddCommand(downloadMultipleCmd) +} + +func loadManifestGuids(path string) ([]string, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var entries []struct { + GUID string `json:"guid"` + } + if err := json.Unmarshal(data, &entries); err != nil { + return nil, fmt.Errorf("parse manifest: %w", err) + } + + guids := make([]string, 0, len(entries)) + for _, entry := range entries { + if entry.GUID != "" { + guids = append(guids, entry.GUID) + } + } + if len(guids) == 0 { + return nil, fmt.Errorf("manifest %s did not contain any GUIDs", path) + } + return guids, nil +} diff --git a/cmd/download-single.go b/cmd/download-single.go new file mode 100644 index 0000000..5e1227c --- /dev/null +++ b/cmd/download-single.go @@ -0,0 +1,49 @@ +package cmd + +import ( + "context" + "log" + + "github.com/calypr/data-client/g3client" + "github.com/calypr/data-client/logs" + sydownload "github.com/calypr/syfon/client/transfer/download" + "github.com/spf13/cobra" +) + +func init() { + var guid string + var downloadPath string + var profile string + + var downloadSingleCmd = &cobra.Command{ + Use: "download-single", + Short: "Download a single file from a GUID", + Long: `Gets a presigned URL for a file from a GUID and then downloads the specified file.`, + Example: `./data-client download-single --profile= --guid=206dfaa6-bcf1-4bc9-b2d0-77179f0f48fc`, + Run: func(cmd *cobra.Command, args []string) { + logger, logCloser := logs.New(profile, logs.WithConsole(), logs.WithFailedLog(), logs.WithSucceededLog(), logs.WithScoreboard()) + defer logCloser() + + g3I, err := g3client.NewGen3Interface(profile, logger) + if err != nil { + log.Fatalf("Failed to parse config on profile %s, %v", profile, err) + } + + syfon := g3I.SyfonClient() + if syfon == nil { + logger.Fatal("failed to initialize syfon client") + } + err = sydownload.DownloadSingleWithProgress(context.Background(), syfon.DRS(), syfon.Data(), guid, downloadPath, "original") + if err != nil { + logger.Println(err.Error()) + } + }, + } + + downloadSingleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") + downloadSingleCmd.MarkFlagRequired("profile") //nolint:errcheck + downloadSingleCmd.Flags().StringVar(&guid, "guid", "", "Specify the guid for the data you would like to work with") + downloadSingleCmd.MarkFlagRequired("guid") //nolint:errcheck + downloadSingleCmd.Flags().StringVar(&downloadPath, "download-path", ".", "The directory in which to store the downloaded files") + RootCmd.AddCommand(downloadSingleCmd) +} diff --git a/cmd/gitversion.go b/cmd/gitversion.go new file mode 100644 index 0000000..ce96e41 --- /dev/null +++ b/cmd/gitversion.go @@ -0,0 +1,6 @@ +package cmd + +var ( + gitcommit = "N/A" + gitversion = "2026.2" +) diff --git a/cmd/retry-upload.go b/cmd/retry-upload.go new file mode 100644 index 0000000..01d47d3 --- /dev/null +++ b/cmd/retry-upload.go @@ -0,0 +1,58 @@ +package cmd + +import ( + "context" + + "github.com/calypr/data-client/g3client" + "github.com/calypr/data-client/logs" + syclient "github.com/calypr/syfon/client" + sycommon "github.com/calypr/syfon/client/common" + + "github.com/spf13/cobra" +) + +func init() { + var failedLogPath, profile string + + var retryUploadCmd = &cobra.Command{ + Use: "retry-upload", + Short: "Retry failed uploads from a failed_log.json", + Long: `Re-uploads files listed in a failed log using exponential backoff and progress bars.`, + Example: `./data-client retry-upload --profile=myprofile --failed-log-path=/path/to/failed_log.json`, + Run: func(cmd *cobra.Command, args []string) { + Logger, closer := logs.New(profile, + logs.WithConsole(), + logs.WithMessageFile(), + logs.WithFailedLog(), + logs.WithSucceededLog(), + ) + defer closer() + + g3, err := g3client.NewGen3Interface(profile, Logger) + if err != nil { + Logger.Fatalf("Failed to initialize client: %v", err) + } + if _, err := sycommon.LoadFailedLog(failedLogPath); err != nil { + Logger.Fatalf("Cannot read failed log: %v", err) + } + syfon := g3.SyfonClient() + if syfon == nil { + Logger.Fatal("failed to initialize syfon client") + } + err = syclient.Upload(context.Background(), syfon.Data(), "", syclient.UploadOptions{ + RetryFailedLogPath: failedLogPath, + }) + if err != nil { + Logger.Fatalf("Retry upload failed: %v", err) + } + }, + } + + retryUploadCmd.Flags().StringVar(&profile, "profile", "", "Profile to use") + retryUploadCmd.MarkFlagRequired("profile") + + retryUploadCmd.Flags().StringVar(&failedLogPath, "failed-log-path", "", "Path to failed_log.json") + retryUploadCmd.MarkFlagRequired("failed-log-path") + + RootCmd.AddCommand(retryUploadCmd) +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..2a676e4 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,33 @@ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +var profile string +var backendType string + +// RootCmd represents the base command when called without any subcommands +var RootCmd = &cobra.Command{ + Use: "data-client", + Short: "Use the data-client to interact with a Gen3 Data Commons", + Long: "Gen3 Client for downloading, uploading and submitting data to data commons.\ndata-client version: " + gitversion + ", commit: " + gitcommit, + Version: gitversion, +} + +// Execute adds all child commands to the root command sets flags appropriately +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + if err := RootCmd.Execute(); err != nil { + os.Stderr.WriteString("Error: " + err.Error() + "\n") + os.Exit(1) + } +} + +func init() { + RootCmd.PersistentFlags().StringVar(&profile, "profile", "", "Specify profile to use") + RootCmd.PersistentFlags().StringVar(&backendType, "backend", "gen3", "Specify backend to use (gen3 or drs)") + _ = RootCmd.MarkFlagRequired("profile") +} diff --git a/cmd/upload-multipart.go b/cmd/upload-multipart.go new file mode 100644 index 0000000..47058fa --- /dev/null +++ b/cmd/upload-multipart.go @@ -0,0 +1,69 @@ +package cmd + +import ( + "context" + + "github.com/calypr/data-client/g3client" + "github.com/calypr/data-client/logs" + syclient "github.com/calypr/syfon/client" + "github.com/spf13/cobra" +) + +func init() { + var ( + profile string + filePath string + guid string + bucketName string + ) + + var uploadMultipartCmd = &cobra.Command{ + Use: "upload-multipart", + Short: "Upload a single file using managed multipart upload", + Long: `Uploads a file to object storage using managed multipart upload +(init -> presigned part URLs -> complete).`, + Example: `./data-client upload-multipart --profile=myprofile --file-path=./large.bam +./data-client upload-multipart --profile=myprofile --file-path=./data.bam --guid=existing-guid`, + Run: func(cmd *cobra.Command, args []string) { + // Initialize logger + logger, logCloser := logs.New(profile, logs.WithConsole()) + defer logCloser() + + logger, closer := logs.New(profile, logs.WithSucceededLog(), logs.WithFailedLog(), logs.WithScoreboard()) + defer closer() + + g3, err := g3client.NewGen3Interface( + profile, + logger, + ) + + if err != nil { + logger.Fatalf("failed to initialize Gen3 interface: %v", err) + } + + syfon := g3.SyfonClient() + if syfon == nil { + logger.Fatal("failed to initialize syfon client") + } + err = syclient.Upload(context.Background(), syfon.Data(), filePath, syclient.UploadOptions{ + Bucket: bucketName, + GUID: guid, + ShowProgress: true, + ForceMultipart: true, + }) + if err != nil { + logger.Fatal(err) + } + }, + } + + uploadMultipartCmd.Flags().StringVar(&profile, "profile", "", "Specify the profile to use for upload") + uploadMultipartCmd.Flags().StringVar(&filePath, "file-path", "", "Path to the file to upload") + uploadMultipartCmd.Flags().StringVar(&guid, "guid", "", "Optional existing GUID (otherwise generated)") + uploadMultipartCmd.Flags().StringVar(&bucketName, "bucket", "", "Target bucket (defaults to configured DATA_UPLOAD_BUCKET)") + + _ = uploadMultipartCmd.MarkFlagRequired("profile") + _ = uploadMultipartCmd.MarkFlagRequired("file-path") + + RootCmd.AddCommand(uploadMultipartCmd) +} diff --git a/cmd/upload-multiple.go b/cmd/upload-multiple.go new file mode 100644 index 0000000..b6a26f0 --- /dev/null +++ b/cmd/upload-multiple.go @@ -0,0 +1,81 @@ +package cmd + +// Deprecated: Use "upload" instead for new uploads (without pre-existing GUIDs). +import ( + "context" + "fmt" + + "github.com/calypr/data-client/g3client" + "github.com/calypr/data-client/logs" + syclient "github.com/calypr/syfon/client" + "github.com/spf13/cobra" +) + +func init() { + var bucketName string + var manifestPath string + var uploadPath string + var batch bool + var numParallel int + var includeSubDirName bool + + uploadMultipleCmd := &cobra.Command{ + Use: "upload-multiple", + Short: "Upload multiple files from a specified manifest (uses pre-existing GUIDs)", + Long: `Get presigned URLs for multiple files specified in a manifest file and then upload all of them. +This command is for uploading to existing GUIDs (e.g., from a downloaded manifest). +For new uploads (new GUIDs generated), use "data-client upload" instead. + +Options to run multipart uploads for large files and parallel batch uploading are available.`, + Example: `./data-client upload-multiple --profile= --manifest= --upload-path= --bucket= --batch`, + Run: func(cmd *cobra.Command, args []string) { + // Warning message + fmt.Printf("Notice: this command uploads to pre-existing GUIDs from a manifest.\nIf you want to upload new files (new GUIDs generated automatically), use \"./data-client upload\" instead.\n\n") + + ctx := context.Background() + logger, closer := logs.New(profile, logs.WithSucceededLog(), logs.WithFailedLog(), logs.WithScoreboard()) + defer closer() + + g3i, err := g3client.NewGen3Interface(profile, logger) + if err != nil { + logger.Fatalf("Failed to parse config on profile %s: %v", profile, err) + } + + syfon := g3i.SyfonClient() + if syfon == nil { + logger.Fatal("failed to initialize syfon client") + } + err = syclient.Upload(ctx, syfon.Data(), uploadPath, syclient.UploadOptions{ + Bucket: bucketName, + IncludeSubDirName: includeSubDirName, + Batch: batch, + NumParallel: numParallel, + ManifestPath: manifestPath, + ShowProgress: true, + }) + if err != nil { + logger.Println("Upload failed:", err) + } + logger.Scoreboard().PrintSB() + }, + } + + // Flags + uploadMultipleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") + uploadMultipleCmd.MarkFlagRequired("profile") + + uploadMultipleCmd.Flags().StringVar(&manifestPath, "manifest", "", "Path to the manifest JSON file") + uploadMultipleCmd.MarkFlagRequired("manifest") + + uploadMultipleCmd.Flags().StringVar(&uploadPath, "upload-path", "", "Directory containing the files to upload") + uploadMultipleCmd.MarkFlagRequired("upload-path") + + uploadMultipleCmd.Flags().BoolVar(&batch, "batch", true, "Upload single-part files in parallel") + uploadMultipleCmd.Flags().IntVar(&numParallel, "numparallel", 4, "Number of parallel uploads") + + uploadMultipleCmd.Flags().StringVar(&bucketName, "bucket", "", "Target bucket (defaults to configured DATA_UPLOAD_BUCKET)") + + uploadMultipleCmd.Flags().BoolVar(&includeSubDirName, "include-subdirname", true, "Include subdirectory names in object key") + + RootCmd.AddCommand(uploadMultipleCmd) +} diff --git a/cmd/upload-single.go b/cmd/upload-single.go new file mode 100644 index 0000000..69aa56a --- /dev/null +++ b/cmd/upload-single.go @@ -0,0 +1,54 @@ +package cmd + +// Deprecated: Use upload instead. +import ( + "context" + "log" + + "github.com/calypr/data-client/g3client" + "github.com/calypr/data-client/logs" + syclient "github.com/calypr/syfon/client" + "github.com/spf13/cobra" +) + +func init() { + var guid string + var filePath string + var bucketName string + + var uploadSingleCmd = &cobra.Command{ + Use: "upload-single", + Short: "Upload a single file to a GUID", + Long: `Gets a presigned URL for which to upload a file associated with a GUID and then uploads the specified file.`, + Example: `./data-client upload-single --profile= --guid=f6923cf3-xxxx-xxxx-xxxx-14ab3f84f9d6 --file=`, + Run: func(cmd *cobra.Command, args []string) { + logger, closer := logs.New(profile, logs.WithSucceededLog(), logs.WithFailedLog(), logs.WithScoreboard(), logs.WithConsole()) + defer closer() + + g3i, err := g3client.NewGen3Interface(profile, logger) + if err != nil { + log.Fatalf("Failed to parse config on profile %s: %v", profile, err) + } + syfon := g3i.SyfonClient() + if syfon == nil { + log.Fatal("failed to initialize syfon client") + } + err = syclient.Upload(context.Background(), syfon.Data(), filePath, syclient.UploadOptions{ + Bucket: bucketName, + GUID: guid, + ShowProgress: true, + }) + if err != nil { + log.Fatalln(err.Error()) + } + }, + } + uploadSingleCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") + uploadSingleCmd.MarkFlagRequired("profile") //nolint:errcheck + uploadSingleCmd.Flags().StringVar(&guid, "guid", "", "Specify the guid for the data you would like to work with") + uploadSingleCmd.MarkFlagRequired("guid") //nolint:errcheck + uploadSingleCmd.Flags().StringVar(&filePath, "file", "", "Specify file to upload to with --file=~/path/to/file") + uploadSingleCmd.MarkFlagRequired("file") //nolint:errcheck + uploadSingleCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") + RootCmd.AddCommand(uploadSingleCmd) +} diff --git a/cmd/upload.go b/cmd/upload.go new file mode 100644 index 0000000..41549f1 --- /dev/null +++ b/cmd/upload.go @@ -0,0 +1,83 @@ +package cmd + +import ( + "context" + "log" + + "github.com/calypr/data-client/g3client" + "github.com/calypr/data-client/logs" + syclient "github.com/calypr/syfon/client" + "github.com/spf13/cobra" +) + +func init() { + var bucketName string + var includeSubDirName bool + var uploadPath string + var batch bool + var numParallel int + var hasMetadata bool + var uploadCmd = &cobra.Command{ + Use: "upload", + Short: "Upload file(s) to object storage.", + Long: `Gets a presigned URL for each file and then uploads the specified file(s).`, + Example: "For uploading a single file:\n./data-client upload --profile= --upload-path=\n" + + "For uploading all files within an folder:\n./data-client upload --profile= --upload-path=\n" + + "Can also support regex such as:\n./data-client upload --profile= --upload-path=\n" + + "Or:\n./data-client upload --profile= --upload-path=\n" + + "This command can also upload file metadata using the --metadata flag. If the --metadata flag is passed, the data-client will look for a file called [filename]_metadata.json in the same folder, which contains the metadata to upload.\n" + + "For example, if uploading the file `folder/my_file.bam`, the data-client will look for a metadata file at `folder/my_file_metadata.json`.\n" + + "For the format of the metadata files, see the README.", + Run: func(cmd *cobra.Command, args []string) { + + ctx := context.Background() + rootLogger, logCloser := logs.New(profile, logs.WithSucceededLog(), logs.WithScoreboard(), logs.WithFailedLog()) + defer logCloser() + // Instantiate interface to Gen3 + g3i, err := g3client.NewGen3Interface(profile, rootLogger) + if err != nil { + log.Fatalf("Failed to parse config on profile %s, %v", profile, err) + } + logger := g3i.Logger() + if hasMetadata { + hasShepherd, err := g3i.FenceClient().CheckForShepherdAPI(ctx) + if err != nil { + logger.Printf("WARNING: Error when checking for Shepherd API: %v", err) + } else { + if !hasShepherd { + logger.Fatalf("ERROR: Metadata upload (`--metadata`) is not supported in the environment you are uploading to. Double check that you are uploading to the right profile.") + } + } + } + + syfon := g3i.SyfonClient() + if syfon == nil { + logger.Fatalf("failed to initialize syfon client") + } + + err = syclient.Upload(ctx, syfon.Data(), uploadPath, syclient.UploadOptions{ + Bucket: bucketName, + IncludeSubDirName: includeSubDirName, + HasMetadata: hasMetadata, + Batch: batch, + NumParallel: numParallel, + }) + if err != nil { + logger.Error("Upload failed", "error", err) + return + } + g3i.Logger().Scoreboard().PrintSB() + }, + } + + uploadCmd.Flags().StringVar(&profile, "profile", "", "Specify profile to use") + uploadCmd.MarkFlagRequired("profile") //nolint:errcheck + uploadCmd.Flags().StringVar(&uploadPath, "upload-path", "", "The directory or file in which contains file(s) to be uploaded") + uploadCmd.MarkFlagRequired("upload-path") //nolint:errcheck + uploadCmd.Flags().BoolVar(&batch, "batch", false, "Upload in parallel") + uploadCmd.Flags().IntVar(&numParallel, "numparallel", 3, "Number of uploads to run in parallel") + uploadCmd.Flags().BoolVar(&includeSubDirName, "include-subdirname", true, "Include subdirectory names in file name") + uploadCmd.Flags().BoolVar(&hasMetadata, "metadata", false, "Search for and upload file metadata alongside the file") + uploadCmd.Flags().StringVar(&bucketName, "bucket", "", "The bucket to which files will be uploaded. If not provided, defaults to Gen3's configured DATA_UPLOAD_BUCKET.") + RootCmd.AddCommand(uploadCmd) +} diff --git a/conf/config.go b/conf/config.go new file mode 100644 index 0000000..f198f0c --- /dev/null +++ b/conf/config.go @@ -0,0 +1,257 @@ +package conf + +//go:generate go run go.uber.org/mock/mockgen@v0.6.0 -destination=../mocks/mock_configure.go -package=mocks github.com/calypr/data-client/conf ManagerInterface + +import ( + "encoding/json" + "errors" + "fmt" + "log/slog" + "os" + "path" + "strings" + + sycommon "github.com/calypr/syfon/client/common" + syconf "github.com/calypr/syfon/client/config" + "gopkg.in/ini.v1" +) + +var ErrProfileNotFound = errors.New("profile not found in config file") + +type Credential = syconf.Credential + +type Manager struct { + Logger *slog.Logger +} + +func NewConfigure(logs *slog.Logger) ManagerInterface { + return &Manager{ + Logger: logs, + } +} + +type ManagerInterface interface { + // Loads credential from ~/.gen3/ credential file + Import(filePath, fenceToken string) (*Credential, error) + + // Loads credential from ~/.gen3/config.ini + Load(profile string) (*Credential, error) + Save(cred *Credential) error + + EnsureExists() error + IsCredentialValid(*Credential) (bool, error) + IsTokenValid(string) (bool, error) +} + +func (man *Manager) configPath() (string, error) { + homeDir, err := os.UserHomeDir() + if err != nil { + return "", err + } + configPath := path.Join( + homeDir + + sycommon.PathSeparator + + ".gen3" + + sycommon.PathSeparator + + "gen3_client_config.ini", + ) + return configPath, nil +} + +func (man *Manager) Load(profile string) (*Credential, error) { + /* + Looking profile in config file. The config file is a text file located at ~/.gen3 directory. It can + contain more than 1 profile. If there is no profile found, the user is asked to run a command to + create the profile + + The format of config file is described as following + + [profile1] + key_id=key_id_example_1 + api_key=api_key_example_1 + access_token=access_token_example_1 + api_endpoint=http://localhost:8000 + use_shepherd=true + min_shepherd_version=2.0.0 + + [profile2] + key_id=key_id_example_2 + api_key=api_key_example_2 + access_token=access_token_example_2 + api_endpoint=http://localhost:8000 + use_shepherd=false + min_shepherd_version= + + Args: + profile: the specific profile in config file + Returns: + An instance of Credential + */ + + homeDir, err := os.UserHomeDir() + if err != nil { + errs := fmt.Errorf("Error occurred when getting home directory: %s", err.Error()) + man.Logger.Error(errs.Error()) + return nil, errs + } + configPath := path.Join(homeDir + sycommon.PathSeparator + ".gen3" + sycommon.PathSeparator + "gen3_client_config.ini") + + if _, err := os.Stat(configPath); os.IsNotExist(err) { + return nil, fmt.Errorf("%w Run configure command (with a profile if desired) to set up account credentials \n"+ + "Example: ./data-client configure --profile= --cred= --apiendpoint=https://data.mycommons.org", ErrProfileNotFound) + } + + // If profile not in config file, prompt user to set up config first + cfg, err := ini.Load(configPath) + if err != nil { + errs := fmt.Errorf("Error occurred when reading config file: %s", err.Error()) + return nil, errs + } + sec, err := cfg.GetSection(profile) + if err != nil { + return nil, fmt.Errorf("%w: Need to run \"data-client configure --profile="+profile+" --cred= --apiendpoint=\" first", ErrProfileNotFound) + } + + profileConfig := &Credential{ + Profile: profile, + KeyID: sec.Key("key_id").String(), + APIKey: sec.Key("api_key").String(), + AccessToken: sec.Key("access_token").String(), + APIEndpoint: sec.Key("api_endpoint").String(), + UseShepherd: sec.Key("use_shepherd").String(), + MinShepherdVersion: sec.Key("min_shepherd_version").String(), + Bucket: sec.Key("bucket").String(), + ProjectID: sec.Key("project_id").String(), + } + + if profileConfig.KeyID == "" && profileConfig.APIKey == "" && profileConfig.AccessToken == "" { + errs := fmt.Errorf("key_id, api_key and access_token not found in profile.") + return nil, errs + } + if profileConfig.APIEndpoint == "" { + errs := fmt.Errorf("api_endpoint not found in profile.") + return nil, errs + } + + return profileConfig, nil +} + +func (man *Manager) Save(profileConfig *Credential) error { + /* + Overwrite the config file with new credential + + Args: + profileConfig: Credential object represents config of a profile + configPath: file path to config file + */ + configPath, err := man.configPath() + if err != nil { + errs := fmt.Errorf("error occurred when getting config path: %s", err.Error()) + man.Logger.Error(errs.Error()) + return errs + } + cfg, err := ini.Load(configPath) + if err != nil { + errs := fmt.Errorf("error occurred when loading config file: %s", err.Error()) + man.Logger.Error(errs.Error()) + return errs + } + + section := cfg.Section(profileConfig.Profile) + if profileConfig.KeyID != "" { + section.Key("key_id").SetValue(profileConfig.KeyID) + } + if profileConfig.APIKey != "" { + section.Key("api_key").SetValue(profileConfig.APIKey) + } + if profileConfig.AccessToken != "" { + section.Key("access_token").SetValue(profileConfig.AccessToken) + } + if profileConfig.APIEndpoint != "" { + section.Key("api_endpoint").SetValue(profileConfig.APIEndpoint) + } + + section.Key("use_shepherd").SetValue(profileConfig.UseShepherd) + section.Key("min_shepherd_version").SetValue(profileConfig.MinShepherdVersion) + section.Key("bucket").SetValue(profileConfig.Bucket) + section.Key("project_id").SetValue(profileConfig.ProjectID) + err = cfg.SaveTo(configPath) + if err != nil { + errs := fmt.Errorf("error occurred when saving config file: %s", err.Error()) + man.Logger.Error(errs.Error()) + return fmt.Errorf("error occurred when saving config file: %s", err.Error()) + } + return nil +} + +func (man *Manager) EnsureExists() error { + /* + Make sure the config exists on start up + */ + configPath, err := man.configPath() + if err != nil { + return err + } + + if _, err := os.Stat(path.Dir(configPath)); os.IsNotExist(err) { + osErr := os.Mkdir(path.Join(path.Dir(configPath)), os.FileMode(0777)) + if osErr != nil { + return err + } + _, osErr = os.Create(configPath) + if osErr != nil { + return err + } + } + if _, err := os.Stat(configPath); os.IsNotExist(err) { + _, osErr := os.Create(configPath) + if osErr != nil { + return err + } + } + _, err = ini.Load(configPath) + + return err +} + +func (man *Manager) Import(filePath, fenceToken string) (*Credential, error) { + var cred Credential + + if filePath != "" { + fullPath, err := sycommon.GetAbsolutePath(filePath) + if err != nil { + man.Logger.Error("error parsing credential file path: " + err.Error()) + return nil, err + } + + content, err := os.ReadFile(fullPath) + if err != nil { + if os.IsNotExist(err) { + man.Logger.Error("File not found: " + fullPath) + } else { + man.Logger.Error("error reading file: " + err.Error()) + } + return nil, err + } + + jsonStr := strings.ReplaceAll(string(content), "\n", "") + // Normalize keys from snake_case to CamelCase for unmarshaling + jsonStr = strings.ReplaceAll(jsonStr, "key_id", "KeyID") + jsonStr = strings.ReplaceAll(jsonStr, "api_key", "APIKey") + jsonStr = strings.ReplaceAll(jsonStr, "access_token", "AccessToken") + jsonStr = strings.ReplaceAll(jsonStr, "api_endpoint", "APIEndpoint") + jsonStr = strings.ReplaceAll(jsonStr, "project_id", "ProjectID") + + if err := json.Unmarshal([]byte(jsonStr), &cred); err != nil { + errMsg := fmt.Errorf("cannot parse JSON credential file: %w", err) + man.Logger.Error(errMsg.Error()) + return nil, errMsg + } + } else if fenceToken != "" { + cred.AccessToken = fenceToken + } else { + return nil, errors.New("either credential file or fence token must be provided") + } + + return &cred, nil +} diff --git a/conf/config_test.go b/conf/config_test.go new file mode 100644 index 0000000..24b7ea2 --- /dev/null +++ b/conf/config_test.go @@ -0,0 +1,261 @@ +package conf + +import ( + "log/slog" + "os" + "path" + "testing" +) + +func TestNewConfigure(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := NewConfigure(logger) + + if manager == nil { + t.Fatal("Expected non-nil manager") + } + + // Type assertion to verify it's a *Manager + if _, ok := manager.(*Manager); !ok { + t.Error("Expected manager to be of type *Manager") + } +} + +func TestConfigPath(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + configPath, err := manager.configPath() + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if configPath == "" { + t.Error("Expected non-empty config path") + } + + // Verify path contains expected components + if !contains(configPath, ".gen3") { + t.Error("Expected config path to contain .gen3 directory") + } + + if !contains(configPath, "gen3_client_config.ini") { + t.Error("Expected config path to contain gen3_client_config.ini") + } +} + +func TestImport_WithCredentialFile(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + // Create a temporary credential file + tmpDir := t.TempDir() + credFile := path.Join(tmpDir, "cred.json") + + credContent := `{ + "KeyID": "test-key-id", + "APIKey": "test-api-key" + }` + + if err := os.WriteFile(credFile, []byte(credContent), 0644); err != nil { + t.Fatalf("Failed to create test credential file: %v", err) + } + + cred, err := manager.Import(credFile, "") + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if cred == nil { + t.Fatal("Expected non-nil credential") + } + + if cred.KeyID != "test-key-id" { + t.Errorf("Expected KeyID 'test-key-id', got '%s'", cred.KeyID) + } + + if cred.APIKey != "test-api-key" { + t.Errorf("Expected APIKey 'test-api-key', got '%s'", cred.APIKey) + } +} + +func TestImport_WithSnakeCaseCredentialFields(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + tmpDir := t.TempDir() + credFile := path.Join(tmpDir, "cred.json") + + credContent := `{ + "key_id": "test-key-id", + "api_key": "test-api-key", + "access_token": "test-access-token", + "api_endpoint": "https://example.org", + "project_id": "project-1" + }` + + if err := os.WriteFile(credFile, []byte(credContent), 0644); err != nil { + t.Fatalf("Failed to create test credential file: %v", err) + } + + cred, err := manager.Import(credFile, "") + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if cred.KeyID != "test-key-id" { + t.Fatalf("KeyID = %q, want %q", cred.KeyID, "test-key-id") + } + if cred.APIKey != "test-api-key" { + t.Fatalf("APIKey = %q, want %q", cred.APIKey, "test-api-key") + } + if cred.AccessToken != "test-access-token" { + t.Fatalf("AccessToken = %q, want %q", cred.AccessToken, "test-access-token") + } + if cred.APIEndpoint != "https://example.org" { + t.Fatalf("APIEndpoint = %q, want %q", cred.APIEndpoint, "https://example.org") + } + if cred.ProjectID != "project-1" { + t.Fatalf("ProjectID = %q, want %q", cred.ProjectID, "project-1") + } +} + +func TestImport_WithFenceToken(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + token := "test-fence-token-12345" + cred, err := manager.Import("", token) + + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if cred == nil { + t.Fatal("Expected non-nil credential") + } + + if cred.AccessToken != token { + t.Errorf("Expected AccessToken '%s', got '%s'", token, cred.AccessToken) + } +} + +func TestImport_NoCredentialOrToken(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + _, err := manager.Import("", "") + + if err == nil { + t.Fatal("Expected error when neither credential file nor token provided") + } + + if !contains(err.Error(), "either credential file or fence token must be provided") { + t.Errorf("Expected specific error message, got: %v", err) + } +} + +func TestImport_InvalidCredentialFile(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + // Test with non-existent file + _, err := manager.Import("/nonexistent/path/cred.json", "") + + if err == nil { + t.Fatal("Expected error for non-existent file") + } +} + +func TestImport_InvalidJSON(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + // Create a temporary file with invalid JSON + tmpDir := t.TempDir() + credFile := path.Join(tmpDir, "invalid.json") + + invalidJSON := `{invalid json content` + + if err := os.WriteFile(credFile, []byte(invalidJSON), 0644); err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + + _, err := manager.Import(credFile, "") + + if err == nil { + t.Fatal("Expected error for invalid JSON") + } + + if !contains(err.Error(), "cannot parse JSON credential file") { + t.Errorf("Expected JSON parse error, got: %v", err) + } +} + +func TestEnsureExists(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + // This test is tricky because it modifies the user's home directory + // We'll just verify it doesn't panic and returns a reasonable error or nil + err := manager.EnsureExists() + + // We accept either success or a reasonable error + if err != nil { + // Just log the error, don't fail the test + t.Logf("EnsureExists returned error (may be expected): %v", err) + } +} + +func TestLoad_ProfileNotFound(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + // Try to load a profile that doesn't exist + _, err := manager.Load("nonexistent-profile") + + if err == nil { + t.Fatal("Expected error for non-existent profile") + } + + // Should contain profile not found error + if !contains(err.Error(), "profile not found") && !contains(err.Error(), "Need to run") { + t.Logf("Got error (may be expected): %v", err) + } +} + +func TestIsCredentialValid_RejectsAccessTokenMatchingAPIKey(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + manager := &Manager{Logger: logger} + + cred := &Credential{ + APIKey: "same-token", + AccessToken: "same-token", + } + + valid, err := manager.IsCredentialValid(cred) + if err == nil { + t.Fatal("expected error when access token matches api key") + } + if valid { + t.Fatal("expected credential to be invalid") + } + if !contains(err.Error(), "access_token matches api_key") { + t.Fatalf("unexpected error: %v", err) + } +} + +// Helper function +func contains(s, substr string) bool { + return len(s) >= len(substr) && (s == substr || len(substr) == 0 || + (len(s) > 0 && len(substr) > 0 && findSubstring(s, substr))) +} + +func findSubstring(s, substr string) bool { + for i := 0; i <= len(s)-len(substr); i++ { + if s[i:i+len(substr)] == substr { + return true + } + } + return false +} diff --git a/conf/validate.go b/conf/validate.go new file mode 100644 index 0000000..6eb94f8 --- /dev/null +++ b/conf/validate.go @@ -0,0 +1,100 @@ +package conf + +import ( + "errors" + "fmt" + "net/url" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +func ValidateUrl(apiEndpoint string) (*url.URL, error) { + parsedURL, err := url.Parse(apiEndpoint) + if err != nil { + return parsedURL, errors.New("Error occurred when parsing apiendpoint URL: " + err.Error()) + } + if parsedURL.Host == "" { + return parsedURL, errors.New("Invalid endpoint. A valid endpoint looks like: https://www.tests.com") + } + return parsedURL, nil +} + +func (man *Manager) IsTokenValid(tokenStr string) (bool, error) { + if tokenStr == "" { + return false, fmt.Errorf("token is empty") + } + // Parse the token without verifying the signature to access the claims. + token, _, err := new(jwt.Parser).ParseUnverified(tokenStr, jwt.MapClaims{}) + if err != nil { + return false, fmt.Errorf("invalid token format: %v", err) + } + + claims, ok := token.Claims.(jwt.MapClaims) + if !ok { + return false, fmt.Errorf("unable to parse claims from provided token") + } + + exp, ok := claims["exp"].(float64) + if !ok { + return false, fmt.Errorf("'exp' claim not found or is not a number") + } + + iat, ok := claims["iat"].(float64) + if !ok { + // iat is not strictly required for validity in all cases, but we'll keep it for now as per original code + return false, fmt.Errorf("'iat' claim not found or is not a number") + } + + now := time.Now().UTC() + expTime := time.Unix(int64(exp), 0).UTC() + iatTime := time.Unix(int64(iat), 0).UTC() + + if expTime.Before(now) { + return false, fmt.Errorf("token expired at %s (now: %s)", expTime.Format(time.RFC3339), now.Format(time.RFC3339)) + } + if iatTime.After(now) { + return false, fmt.Errorf("token not yet valid: iat %s > now %s", iatTime.Format(time.RFC3339), now.Format(time.RFC3339)) + } + + delta := expTime.Sub(now) + // threshold days set to 10 + if delta > 0 && delta.Hours() < float64(10*24) { + daysUntilExpiration := int(delta.Hours() / 24) + if daysUntilExpiration > 0 && man.Logger != nil { + man.Logger.Warn(fmt.Sprintf("Token will expire in %d days, on %s", daysUntilExpiration, expTime.Format(time.RFC3339))) + } + } + + return true, nil +} + +func (man *Manager) IsCredentialValid(profileConfig *Credential) (bool, error) { + if profileConfig == nil { + return false, fmt.Errorf("profileConfig is nil") + } + + if profileConfig.AccessToken != "" && + profileConfig.APIKey != "" && + profileConfig.AccessToken == profileConfig.APIKey { + return false, fmt.Errorf("access_token matches api_key and appears to be misconfigured") + } + + accessTokenValid, accessErr := man.IsTokenValid(profileConfig.AccessToken) + apiKeyValid, apiErr := man.IsTokenValid(profileConfig.APIKey) + + if !accessTokenValid && !apiKeyValid { + return false, fmt.Errorf("both access_token and api_key are invalid: %v; %v", accessErr, apiErr) + } + + if !accessTokenValid && apiKeyValid { + return false, fmt.Errorf("access_token is invalid but api_key is valid: %v", accessErr) + } + + return true, nil +} + +func (man *Manager) IsValid(profileConfig *Credential) (bool, error) { + // Maintain backward compatibility by checking APIKey as before, but using the new helper + return man.IsTokenValid(profileConfig.APIKey) +} diff --git a/conf/validate_test.go b/conf/validate_test.go new file mode 100644 index 0000000..58d230d --- /dev/null +++ b/conf/validate_test.go @@ -0,0 +1,140 @@ +package conf + +import ( + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +func createTestToken(exp time.Time, iat time.Time) string { + claims := jwt.MapClaims{ + "exp": exp.Unix(), + "iat": iat.Unix(), + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + // We don't need a real signature for ParseUnverified + tokenString, _ := token.SignedString([]byte("secret")) + return tokenString +} + +func TestIsTokenValid(t *testing.T) { + man := &Manager{} + now := time.Now().UTC() + + tests := []struct { + name string + token string + want bool + wantErr bool + }{ + { + name: "Valid Token", + token: createTestToken(now.Add(time.Hour), now.Add(-time.Hour)), + want: true, + wantErr: false, + }, + { + name: "Expired Token", + token: createTestToken(now.Add(-time.Hour), now.Add(-2*time.Hour)), + want: false, + wantErr: true, + }, + { + name: "Not Yet Valid Token", + token: createTestToken(now.Add(2*time.Hour), now.Add(time.Hour)), + want: false, + wantErr: true, + }, + { + name: "Empty Token", + token: "", + want: false, + wantErr: true, + }, + { + name: "Invalid Token Format", + token: "not.a.token", + want: false, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := man.IsTokenValid(tt.token) + if (err != nil) != tt.wantErr { + t.Errorf("IsTokenValid() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("IsTokenValid() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsCredentialValid(t *testing.T) { + man := &Manager{} + now := time.Now().UTC() + validAccessToken := createTestToken(now.Add(time.Hour), now.Add(-time.Hour)) + validAPIKey := createTestToken(now.Add(2*time.Hour), now.Add(-time.Hour)) + expiredToken := createTestToken(now.Add(-time.Hour), now.Add(-2*time.Hour)) + + tests := []struct { + name string + cred *Credential + want bool + wantErr bool + }{ + { + name: "Both Valid", + cred: &Credential{ + AccessToken: validAccessToken, + APIKey: validAPIKey, + }, + want: true, + wantErr: false, + }, + { + name: "AccessToken Invalid, APIKey Valid (Needs Refresh)", + cred: &Credential{ + AccessToken: expiredToken, + APIKey: validAPIKey, + }, + want: false, + wantErr: true, + }, + { + name: "AccessToken Matches APIKey", + cred: &Credential{ + AccessToken: validAccessToken, + APIKey: validAccessToken, + }, + want: false, + wantErr: true, + }, + { + name: "Both Invalid", + cred: &Credential{ + AccessToken: expiredToken, + APIKey: expiredToken, + }, + want: false, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := man.IsCredentialValid(tt.cred) + if (err != nil) != tt.wantErr { + t.Errorf("IsCredentialValid() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("IsCredentialValid() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/credentials/refresh.go b/credentials/refresh.go new file mode 100644 index 0000000..115d7d9 --- /dev/null +++ b/credentials/refresh.go @@ -0,0 +1,45 @@ +package credentials + +import ( + "context" + "fmt" + "log/slog" + "strings" + + "github.com/calypr/data-client/conf" + "github.com/calypr/data-client/fence" + "github.com/calypr/data-client/logs" + "github.com/calypr/data-client/request" +) + +// EnsureValidCredential validates a profile credential and refreshes its +// access token from the API key when the token has expired. +func EnsureValidCredential(ctx context.Context, cred *conf.Credential, baseLogger *slog.Logger) error { + manager := conf.NewConfigure(baseLogger) + logger := logs.NewGen3Logger(baseLogger, "", cred.Profile) + + valid, err := manager.IsCredentialValid(cred) + if valid { + return nil + } + if err == nil { + return fmt.Errorf("invalid credential") + } + + if !strings.Contains(err.Error(), "access_token is invalid but api_key is valid") { + return fmt.Errorf("invalid credential: %v", err) + } + + req := request.NewRequestInterface(logger, cred, manager) + fClient := fence.NewFenceClient(req, cred, baseLogger) + newToken, refreshErr := fClient.NewAccessToken(ctx) + if refreshErr != nil { + return fmt.Errorf("failed to refresh access token: %v (original error: %v)", refreshErr, err) + } + + cred.AccessToken = newToken + if saveErr := manager.Save(cred); saveErr != nil { + logger.Warn(fmt.Sprintf("failed to save refreshed token: %v", saveErr)) + } + return nil +} diff --git a/docs/DEVELOPER_DOCS.md b/docs/DEVELOPER_DOCS.md new file mode 100644 index 0000000..54478a7 --- /dev/null +++ b/docs/DEVELOPER_DOCS.md @@ -0,0 +1,91 @@ +# Dev Docs + +This repo is a heavily updated / refactored version of https://github.com/uc-cdis/cdis-data-client + +The new architecture splits out many of the mega packages into smaller, more digestable pieces. This whole CLI is essentially a Go client library for Gen3's Fence microservice. + +These new packages are: + +├── api +│   ├── gen3.go +│   └── types.go +├── client +│   └── client.go +├── common +│   ├── common.go +│   ├── constants.go +│   ├── isHidden_notwindows.go +│   ├── isHidden_windows.go +│   ├── logHelper.go +│   └── types.go +├── conf +│   ├── config.go +│   └── validate.go +├── download +│   ├── batch.go +│   ├── downloader.go +│   ├── file_info.go +│   ├── types.go +│   ├── url_resolution.go +│   └── utils.go +├── logs +│   ├── factory.go +│   ├── logger.go +│   ├── scoreboard.go +│   └── tee_logger.go +├── mocks +│   ├── mock_configure.go +│   ├── mock_functions.go +│   ├── mock_gen3interface.go +│   └── mock_request.go +├── request +│   ├── auth.go +│   ├── builder.go +│   └── request.go +└── upload + ├── batch.go + ├── multipart.go + ├── request.go + ├── retry.go + ├── singleFile.go + ├── types.go + ├── upload.go + └── utils.go + + +# api + +This is the main Client API for talking to fence. Some of the functions that are currently defined in upload/ and download should probablyl be broken out into this library also. + +# client + +This is a thin wrapper client that wraps the API interface to make the API calls easier to use from other packages. + +# common + +This contains common constants / utility functions that are used in the repo + +# conf + +This is the config package for loading / storing the gen3 credential. Note ~/.gen3/.ini file is where credentials / configurations are stored, +but the raw credential is also stored in ~/.gen3/ under whatever you called it. + +# download + +This is the business logic for all download and download related operations in the depo + +# logs + +This is where the logger is defined + +# mocks + +This contains mocks for testing the data-client + +# request + +This is the lowest level interface for doing requests. It implements some basic retry, and wraps the http round trip with a token if one is provided + +# upload + +This contains the business logic for all upload and upload related operations. diff --git a/docs/optimal-chunk-size.md b/docs/optimal-chunk-size.md new file mode 100644 index 0000000..55cb850 --- /dev/null +++ b/docs/optimal-chunk-size.md @@ -0,0 +1,151 @@ + +# Engineering note — Optimal Chunk Size Calculation for Multipart Uploads + +## OLD: + optimalChunkSize determines the ideal chunk/part size for multipart upload based on file size. + The chunk size (also known as "message size" or "part size") affects upload performance and + must comply with S3 constraints. + + Calculation logic: + - For files ≤ 512 MB: Returns 32 MB chunks for optimal performance + - For files > 512 MB: Calculates fileSize/maxMultipartParts, with minimum of 5 MB + - Enforces minimum of 5 MB (S3 requirement for all parts except the last) + - Rounds up to nearest MB for alignment + + This results in: + - Files ≤ 512 MB: 32 MB chunks + - Files 512 MB - ~49 GB: 5 MB chunks (minimum enforced) + The ~49 GB threshold (10,000 parts × 5 MB) is where files exceed S3's + 10,000 part limit when using the minimum chunk size + - Files > ~49 GB: Dynamically calculated to stay under 10,000 parts + + Examples: + - 100 MB file → 32 MB chunks (4 parts) + - 1 GB file → 5 MB chunks (~205 parts) + - 10 GB file → 5 MB chunks (~2,048 parts) + - 50 GB file → 6 MB chunks (~8,534 parts) + - 100 GB file → 11 MB chunks (~9,310 parts) + - 1 TB file → 105 MB chunks (~9,987 parts) + +## NEW + +OptimalChunkSize determines the ideal chunk/part size for multipart upload based on file size. +The chunk size (also known as "message size" or "part size") affects upload performance and +must comply with S3 constraints. + +Calculation logic: + - For files ≤ 100 MB: Returns the file size itself (single PUT, no multipart) + - For files > 100 MB and ≤ 1 GB: Returns 10 MB chunks + - For files > 1 GB and ≤ 10 GB: Scales linearly between 25 MB and 128 MB + - For files > 10 GB and ≤ 100 GB: Returns 256 MB chunks + - For files > 100 GB: Scales linearly between 512 MB and 1024 MB (capped at 1 TB for ratio purposes) + - All chunk sizes are rounded down to the nearest MB + - Minimum chunk size is 1 MB (for zero or negative input) + +This results in: + - Files ≤ 100 MB: Single PUT upload + - Files 100 MB - 1 GB: 10 MB chunks + - Files 1 GB - 10 GB: 25-128 MB chunks (scaled) + - Files 10 GB - 100 GB: 256 MB chunks + - Files > 100 GB: 512-1024 MB chunks (scaled) + +Examples: + - 100 MB file → 100 MB chunk (1 part, single PUT) + - 500 MB file → 10 MB chunks (50 parts) + - 1 GB file → 10 MB chunks (103 parts) + - 5 GB file → 70 MB chunks (74 parts, scaled) + - 10 GB file → 128 MB chunks (80 parts) + - 50 GB file → 256 MB chunks (200 parts) + - 100 GB file → 256 MB chunks (400 parts) + - 500 GB file → 739 MB chunks (693 parts, scaled) + - 1 TB file → 1024 MB chunks (1024 parts) + +### Testing + + +```bash +go test ./upload -run '^TestOptimalChunkSize$' -v + +``` + +Purpose +- Validate `OptimalChunkSize` behavior and return values (chunk size and number of parts) across thresholds, boundaries and scaled ranges. + +Key behavior to assert +1. Input type and units: sizes are `int64` bytes; tests should use `common.MB` / `common.GB` constants. +2. Parts calculation: `parts = ceil(fileSize / chunk)`; `fileSize == 0` returns `parts == 0`. +3. Scaling: scaled ranges are linear, rounded **down** to the nearest MB and clamped to range. +4. Minimum chunk clamp: result is at least `1 MB`. +5. Boundary semantics: implementation uses `<=` and some ranges start at `X + 1` — include exact, \-1 and \+1 byte checks. + +Parameterized test cases (file size ⇒ expected chunk ⇒ expected parts) +1. `0` bytes + - chunk: `1 MB` (fallback) + - parts: `0` + +2. `1 MB` + - chunk: `1 MB` (<= 100 MB) + - parts: `1` + +3. `100 MB` + - chunk: `100 MB` (<= 100 MB) + - parts: `1` + +4. `100 MB + 1 B` + - chunk: `10 MB` (> 100 MB - <= 1 GB) + - parts: ceil((100 MB + 1 B) / 10 MB) = `11` + +5. `500 MB` + - chunk: `10 MB` + - parts: `50` + +6. `1 GB` (1024 MB) + - chunk: `10 MB` (<= 1 GB) + - parts: ceil(1024 / 10) = `103` + +7. `1 GB + 1 B` + - chunk: `25 MB` (start of 1 GB - 10 GB scaled range) + - parts: ceil((1024 MB + 1 B) / 25 MB) = `41` + +8. `5 GB` (5120 MB) + - chunk: linear between `25 MB` and `128 MB` → ≈ `70 MB` (rounded down) + - parts: ceil(5120 / 70) = `74` + +9. `10 GB` (10240 MB) + - chunk: `128 MB` (end of 1 GB - 10 GB scaled range) + - parts: `80` + +10. `10 GB + 1 B` + - chunk: `256 MB` (> 10 GB - <= 100 GB fixed) + - parts: ceil((10240 MB + 1 B) / 256 MB) = `41` + +11. `50 GB` (51200 MB) + - chunk: `256 MB` + - parts: `200` + +12. `100 GB` (102400 MB) + - chunk: `256 MB` + - parts: `400` + +13. `100 GB + 1 B` + - chunk: `512 MB` (start of > 100 GB scaled range) + - parts: ceil((102400 MB + 1 B) / 512 MB) = `201` + +14. `500 GB` (512000 MB) + - chunk: linear between `512 MB` and `1024 MB` → ≈ `739 MB` (rounded down) + - parts: ceil(512000 / 739) = `693` + +15. `1 TB` (1024 GB = 1,048,576 MB) — note: use project units consistently + - chunk: `1024 MB` (max of scaled range) + - parts: 1,048,576 / 1024 = `1024` + +Test design notes (concise) +1. Use table-driven subtests in `upload/utils_test.go`. Include fields: name, `fileSize int64`, `wantChunk int64`, `wantParts int64`. +2. For scaled cases assert: MB alignment, clamped to min/max, and exact `wantParts`. Use integer arithmetic for parts. +3. Add explicit boundary triples for each threshold: exact, -1 byte, +1 byte. +4. Include negative and zero cases to verify fallback behavior. +5. Keep tests deterministic and fast (no external deps). + +Execution +- Run from repo root: `go test ./upload -v` +- Run single test: `go test ./upload -run '^TestOptimalChunkSize$' -v` \ No newline at end of file diff --git a/fence/client.go b/fence/client.go new file mode 100644 index 0000000..20ae4bc --- /dev/null +++ b/fence/client.go @@ -0,0 +1,659 @@ +package fence + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "log/slog" + + "github.com/calypr/data-client/conf" + "github.com/calypr/data-client/request" + sycommon "github.com/calypr/syfon/client/common" + "github.com/hashicorp/go-version" +) + +// FenceBucketEndpoint is the endpoint postfix for FENCE bucket list +const FenceBucketEndpoint = "/data/buckets" + +const ( + fenceAccessTokenEndpoint = "/user/credentials/api/access_token" + fenceUserEndpoint = "/user/user" + fenceDataEndpoint = "/user/data" + fenceDataUploadEndpoint = fenceDataEndpoint + "/upload" + fenceDataDownloadEndpoint = fenceDataEndpoint + "/download" + fenceDataMultipartInitEndpoint = fenceDataEndpoint + "/multipart/init" + fenceDataMultipartUploadEndpoint = fenceDataEndpoint + "/multipart/upload" + fenceDataMultipartCompleteEndpoint = fenceDataEndpoint + "/multipart/complete" + shepherdEndpoint = "/mds" + shepherdVersionEndpoint = "/mds/version" + defaultUseShepherd = false + defaultMinShepherdVersion = "2.0.0" +) + +//go:generate go run go.uber.org/mock/mockgen@v0.6.0 -destination=../mocks/mock_fence.go -package=mocks github.com/calypr/data-client/fence FenceInterface + +// FenceInterface defines the interface for Fence client +type FenceInterface interface { + request.RequestInterface + + NewAccessToken(ctx context.Context) (string, error) + CheckPrivileges(ctx context.Context) (map[string]any, error) + CheckForShepherdAPI(ctx context.Context) (bool, error) + DeleteRecord(ctx context.Context, guid string) (string, error) + GetDownloadPresignedUrl(ctx context.Context, guid, protocolText string) (string, error) + + UserPing(ctx context.Context) (*PingResp, error) + + // Bucket details + GetBucketDetails(ctx context.Context, bucket string) (*S3Bucket, error) + + // Upload methods + InitUpload(ctx context.Context, filename string, bucket string, guid string) (FenceResponse, error) + GetUploadPresignedUrl(ctx context.Context, guid string, filename string, bucket string) (FenceResponse, error) + + // Multipart methods + InitMultipartUpload(ctx context.Context, filename string, bucket string, guid string) (FenceResponse, error) + GenerateMultipartPresignedURL(ctx context.Context, key string, uploadID string, partNumber int, bucket string) (string, error) + CompleteMultipartUpload(ctx context.Context, key string, uploadID string, parts []MultipartPart, bucket string) error + ParseFenceURLResponse(resp *http.Response) (FenceResponse, error) + + RefreshToken(ctx context.Context) error +} + +// FenceClient implements FenceInterface +// FenceClient implements FenceInterface +type FenceClient struct { + request.RequestInterface + cred *conf.Credential + logger *slog.Logger +} + +// NewFenceClient creates a new FenceClient +func NewFenceClient(req request.RequestInterface, cred *conf.Credential, logger *slog.Logger) FenceInterface { + return &FenceClient{ + RequestInterface: req, + cred: cred, + logger: logger, + } +} + +func (f *FenceClient) NewAccessToken(ctx context.Context) (string, error) { + if f.cred.APIKey == "" { + return "", errors.New("APIKey is required to refresh access token") + } + + payload, err := json.Marshal(map[string]string{"api_key": f.cred.APIKey}) + if err != nil { + return "", err + } + bodyReader := bytes.NewReader(payload) + + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Method: http.MethodPost, + Url: f.cred.APIEndpoint + fenceAccessTokenEndpoint, + Headers: map[string]string{sycommon.HeaderContentType: sycommon.MIMEApplicationJSON}, + Body: bodyReader, + }, + ) + + if err != nil { + return "", fmt.Errorf("error when calling Request.Do: %s", err) + } + + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", errors.New("failed to refresh token, status: " + strconv.Itoa(resp.StatusCode)) + } + + var result struct { + AccessToken string `json:"access_token"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", errors.New("failed to parse token response: " + err.Error()) + } + + return result.AccessToken, nil +} + +func (f *FenceClient) RefreshToken(ctx context.Context) error { + token, err := f.NewAccessToken(ctx) + if err != nil { + return err + } + f.cred.AccessToken = token + return nil +} + +func (f *FenceClient) CheckPrivileges(ctx context.Context) (map[string]any, error) { + resp, err := f.Do(ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + fenceUserEndpoint, + Method: http.MethodGet, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return nil, errors.New("error occurred when getting response from remote: " + err.Error()) + } + defer resp.Body.Close() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var data map[string]any + err = json.Unmarshal(bodyBytes, &data) + if err != nil { + return nil, errors.New("error occurred when unmarshalling response: " + err.Error()) + } + + resourceAccess, ok := data["authz"].(map[string]any) + + // If the `authz` section (Arborist permissions) is empty or missing, try get `project_access` section (Fence permissions) + if len(resourceAccess) == 0 || !ok { + resourceAccess, ok = data["project_access"].(map[string]any) + if !ok { + return nil, errors.New("not possible to read access privileges of user") + } + } + + return resourceAccess, nil +} + +func (f *FenceClient) CheckForShepherdAPI(ctx context.Context) (bool, error) { + // Check if Shepherd is enabled + if f.cred.UseShepherd == "false" { + return false, nil + } + if f.cred.UseShepherd != "true" && defaultUseShepherd == false { + return false, nil + } + // If Shepherd is enabled, make sure that the commons has a compatible version of Shepherd deployed. + // Compare the version returned from the Shepherd version endpoint with the minimum acceptable Shepherd version. + var minShepherdVersion string + if f.cred.MinShepherdVersion == "" { + minShepherdVersion = defaultMinShepherdVersion + } else { + minShepherdVersion = f.cred.MinShepherdVersion + } + + res, err := f.Do(ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + shepherdVersionEndpoint, + Method: http.MethodGet, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return false, errors.New("Error occurred during generating HTTP request: " + err.Error()) + } + defer res.Body.Close() + if res.StatusCode != 200 { + return false, nil + } + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + return false, errors.New("Error occurred when reading HTTP request: " + err.Error()) + } + body, err := strconv.Unquote(string(bodyBytes)) + if err != nil { + return false, fmt.Errorf("Error occurred when parsing version from Shepherd: %v: %v", string(body), err) + } + // Compare the version in the response to the target version + ver, err := version.NewVersion(body) + if err != nil { + return false, fmt.Errorf("Error occurred when parsing version from Shepherd: %v: %v", string(body), err) + } + minVer, err := version.NewVersion(minShepherdVersion) + if err != nil { + return false, fmt.Errorf("Error occurred when parsing minimum acceptable Shepherd version: %v: %v", minShepherdVersion, err) + } + if ver.GreaterThanOrEqual(minVer) { + return true, nil + } + return false, fmt.Errorf("Shepherd is enabled, but %v does not have correct Shepherd version. (Need Shepherd version >=%v, got %v)", f.cred.APIEndpoint, minVer, ver) +} + +func (f *FenceClient) DeleteRecord(ctx context.Context, guid string) (string, error) { + hasShepherd, err := f.CheckForShepherdAPI(ctx) + if err != nil { + f.logger.Warn(fmt.Sprintf("WARNING: Error checking Shepherd API: %v. Falling back to Fence.\n", err)) + } else if hasShepherd { + resp, err := f.Do(ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + shepherdEndpoint + "/objects/" + guid, + Method: http.MethodDelete, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return "", fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode == 204 { + return "Record with GUID " + guid + " has been deleted", nil + } + return "", fmt.Errorf("shepherd delete failed: %d", resp.StatusCode) + } + + resp, err := f.Do(ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + fenceDataEndpoint + "/" + guid, + Method: http.MethodDelete, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return "", fmt.Errorf("request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNoContent { + return "Record with GUID " + guid + " has been deleted", nil + } + + _, err = f.ParseFenceURLResponse(resp) + if err != nil { + return "", err + } + return "Record with GUID " + guid + " has been deleted", nil +} + +func (f *FenceClient) GetDownloadPresignedUrl(ctx context.Context, guid, protocolText string) (string, error) { + hasShepherd, err := f.CheckForShepherdAPI(ctx) + if err == nil && hasShepherd { + return f.resolveFromShepherd(ctx, guid) + } + return f.resolveFromFence(ctx, guid, protocolText) +} + +func (f *FenceClient) resolveFromShepherd(ctx context.Context, guid string) (string, error) { + url := fmt.Sprintf("%s%s/objects/%s/download", f.cred.APIEndpoint, shepherdEndpoint, guid) + resp, err := f.Do(ctx, &request.RequestBuilder{ + Url: url, + Method: http.MethodGet, + Token: f.cred.AccessToken, + }) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("shepherd error: %d", resp.StatusCode) + } + + var result struct { + URL string `json:"url"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return "", fmt.Errorf("failed to decode shepherd response: %w", err) + } + + return result.URL, nil +} + +func (f *FenceClient) resolveFromFence(ctx context.Context, guid, protocolText string) (string, error) { + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + fenceDataDownloadEndpoint + "/" + guid + protocolText, + Method: http.MethodGet, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return "", errors.New("failed to get URL from Fence via Do: " + err.Error()) + } + defer resp.Body.Close() + + msg, err := f.ParseFenceURLResponse(resp) + if err != nil || msg.URL == "" { + return "", errors.New("failed to get URL from Fence via ParseFenceURLResponse: " + err.Error()) + } + + return msg.URL, nil +} + +func (f *FenceClient) GetBucketDetails(ctx context.Context, bucket string) (*S3Bucket, error) { + url := f.cred.APIEndpoint + "/data/buckets" + resp, err := f.Do(ctx, &request.RequestBuilder{ + Method: http.MethodGet, + Url: url, + Token: f.cred.AccessToken, + }) + if err != nil { + return nil, fmt.Errorf("failed to fetch bucket information: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode) + } + + var bucketInfo S3BucketsResponse + if err := json.NewDecoder(resp.Body).Decode(&bucketInfo); err != nil { + return nil, fmt.Errorf("failed to decode bucket information: %w", err) + } + + if info, exists := bucketInfo.S3Buckets[bucket]; exists { + if info.EndpointURL != "" && info.Region != "" { + return info, nil + } + return nil, errors.New("endpoint_url or region not found for bucket") + } + + return nil, nil +} + +func (f *FenceClient) InitUpload(ctx context.Context, filename string, bucket string, guid string) (FenceResponse, error) { + payload := map[string]string{ + "file_name": filename, + } + if bucket != "" { + payload["bucket"] = bucket + } + if guid != "" { + payload["guid"] = guid + } + + buf, err := sycommon.ToJSONReader(payload) + if err != nil { + return FenceResponse{}, err + } + + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Method: http.MethodPost, + Url: f.cred.APIEndpoint + fenceDataUploadEndpoint, + Headers: map[string]string{sycommon.HeaderContentType: sycommon.MIMEApplicationJSON}, + Body: buf, + Token: f.cred.AccessToken, + }) + if err != nil { + return FenceResponse{}, err + } + defer resp.Body.Close() + + return f.ParseFenceURLResponse(resp) +} + +func (f *FenceClient) GetUploadPresignedUrl(ctx context.Context, guid string, filename string, bucket string) (FenceResponse, error) { + endPointPostfix := fenceDataUploadEndpoint + "/" + guid + "?file_name=" + url.QueryEscape(filename) + if bucket != "" { + endPointPostfix += "&bucket=" + bucket + } + + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + endPointPostfix, + Headers: map[string]string{sycommon.HeaderContentType: sycommon.MIMEApplicationJSON}, + Token: f.cred.AccessToken, + Method: http.MethodGet, + }, + ) + if err != nil { + return FenceResponse{}, err + } + defer resp.Body.Close() + + return f.ParseFenceURLResponse(resp) +} + +func (f *FenceClient) InitMultipartUpload(ctx context.Context, filename string, bucket string, guid string) (FenceResponse, error) { + reader, err := sycommon.ToJSONReader( + InitRequestObject{ + Filename: filename, + Bucket: bucket, + GUID: guid, + }, + ) + if err != nil { + return FenceResponse{}, err + } + + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Method: http.MethodPost, + Url: f.cred.APIEndpoint + fenceDataMultipartInitEndpoint, + Headers: map[string]string{sycommon.HeaderContentType: sycommon.MIMEApplicationJSON}, + Body: reader, + Token: f.cred.AccessToken, + }, + ) + + if err != nil { + return FenceResponse{}, err + } + defer resp.Body.Close() + + return f.ParseFenceURLResponse(resp) +} + +func (f *FenceClient) GenerateMultipartPresignedURL(ctx context.Context, key string, uploadID string, partNumber int, bucket string) (string, error) { + reader, err := sycommon.ToJSONReader( + MultipartUploadRequestObject{ + Key: key, + UploadID: uploadID, + PartNumber: partNumber, + Bucket: bucket, + }, + ) + if err != nil { + return "", err + } + + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + fenceDataMultipartUploadEndpoint, + Headers: map[string]string{sycommon.HeaderContentType: sycommon.MIMEApplicationJSON}, + Method: http.MethodPost, + Body: reader, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return "", err + } + defer resp.Body.Close() + + msg, err := f.ParseFenceURLResponse(resp) + if err != nil { + return "", err + } + + return msg.PresignedURL, nil +} + +func (f *FenceClient) CompleteMultipartUpload(ctx context.Context, key string, uploadID string, parts []MultipartPart, bucket string) error { + multipartCompleteObject := MultipartCompleteRequestObject{Key: key, UploadID: uploadID, Parts: parts, Bucket: bucket} + + reader, err := sycommon.ToJSONReader(multipartCompleteObject) + if err != nil { + return err + } + + resp, err := f.Do( + ctx, + &request.RequestBuilder{ + Url: f.cred.APIEndpoint + fenceDataMultipartCompleteEndpoint, + Headers: map[string]string{sycommon.HeaderContentType: sycommon.MIMEApplicationJSON}, + Body: reader, + Method: http.MethodPost, + Token: f.cred.AccessToken, + }, + ) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusCreated || resp.StatusCode == http.StatusNoContent { + return nil + } + + _, err = f.ParseFenceURLResponse(resp) + return err +} + +func (f *FenceClient) ParseFenceURLResponse(resp *http.Response) (FenceResponse, error) { + msg := FenceResponse{} + if resp == nil { + return msg, errors.New("nil response received") + } + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return msg, fmt.Errorf("failed to read response body: %w", err) + } + bodyStr := string(bodyBytes) + strURL := "" + if resp.Request != nil && resp.Request.URL != nil { + strURL = resp.Request.URL.String() + } + + // Handle HTTP error statuses first so plain-text error bodies (for example: + // "Unauthorized") are reported accurately instead of as JSON decode failures. + if !(resp.StatusCode == 200 || resp.StatusCode == 201 || resp.StatusCode == 204) { + switch resp.StatusCode { + case http.StatusUnauthorized: + return msg, fmt.Errorf("401 Unauthorized: %s (URL: %s)", bodyStr, strURL) + case http.StatusForbidden: + return msg, fmt.Errorf("403 Forbidden: %s (URL: %s)", bodyStr, strURL) + case http.StatusNotFound: + return msg, fmt.Errorf("404 Not Found: %s (URL: %s)", bodyStr, strURL) + case http.StatusInternalServerError: + return msg, fmt.Errorf("500 Internal Server Error: %s (URL: %s)", bodyStr, strURL) + case http.StatusServiceUnavailable: + return msg, fmt.Errorf("503 Service Unavailable: %s (URL: %s)", bodyStr, strURL) + case http.StatusBadGateway: + return msg, fmt.Errorf("502 Bad Gateway: %s (URL: %s)", bodyStr, strURL) + default: + return msg, fmt.Errorf("unexpected error (%d): %s (URL: %s)", resp.StatusCode, bodyStr, strURL) + } + } + + if len(bodyBytes) > 0 { + err = json.Unmarshal(bodyBytes, &msg) + if err != nil { + return msg, fmt.Errorf("failed to decode JSON response (status=%d, url=%s): %w (raw body: %s)", resp.StatusCode, strURL, err, bodyStr) + } + } + + if strings.Contains(bodyStr, "Can't find a location for the data") { + return msg, errors.New("the provided GUID is not found") + } + + return msg, nil +} + +func (f *FenceClient) UserPing(ctx context.Context) (*PingResp, error) { + resp, err := f.Do(ctx, &request.RequestBuilder{ + Url: f.cred.APIEndpoint + fenceUserEndpoint, + Method: http.MethodGet, + Token: f.cred.AccessToken, + }) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get user info, status: %d", resp.StatusCode) + } + + var uResp FenceUserResp + if err := json.NewDecoder(resp.Body).Decode(&uResp); err != nil { + return nil, err + } + + bucketResp, err := f.Do(ctx, &request.RequestBuilder{ + Url: f.cred.APIEndpoint + FenceBucketEndpoint, + Method: http.MethodGet, + Token: f.cred.AccessToken, + }) + if err != nil { + return nil, err + } + defer bucketResp.Body.Close() + + if bucketResp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get bucket info, status: %d", bucketResp.StatusCode) + } + + var bResp S3BucketsResponse + if err := json.NewDecoder(bucketResp.Body).Decode(&bResp); err != nil { + return nil, err + } + + return &PingResp{ + Profile: f.cred.Profile, + Username: uResp.Username, + Endpoint: f.cred.APIEndpoint, + BucketPrograms: ParseBucketResp(bResp), + YourAccess: ParseUserResp(uResp), + }, nil +} + +func ParseBucketResp(resp S3BucketsResponse) map[string]string { + bucketsByProgram := make(map[string]string) + + // Check both S3_BUCKETS and s3_buckets + s3Buckets := resp.S3Buckets + if len(s3Buckets) == 0 { + s3Buckets = resp.S3BucketsLower + } + + for bucketName, bucketInfo := range s3Buckets { + var programs strings.Builder + if len(bucketInfo.Programs) > 1 { + for i, p := range bucketInfo.Programs { + if i > 0 { + programs.WriteString(",") + } + programs.WriteString(p) + } + } else if len(bucketInfo.Programs) == 1 { + programs.WriteString(bucketInfo.Programs[0]) + } + bucketsByProgram[bucketName] = programs.String() + } + return bucketsByProgram +} + +func ParseUserResp(resp FenceUserResp) map[string]string { + servicesByPath := make(map[string]string) + for path, permissions := range resp.Authz { + var services strings.Builder + seenServices := make(map[string]bool) + for _, p := range permissions { + if !seenServices[p.Method] { + if services.Len() > 0 { + services.WriteString(",") + } + services.WriteString(p.Method) + seenServices[p.Method] = true + } + } + if services.Len() > 0 { + servicesByPath[path] = services.String() + } + } + return servicesByPath +} diff --git a/fence/client_test.go b/fence/client_test.go new file mode 100644 index 0000000..6d12f30 --- /dev/null +++ b/fence/client_test.go @@ -0,0 +1,253 @@ +package fence + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/calypr/data-client/conf" + "github.com/calypr/data-client/logs" + "github.com/calypr/data-client/request" +) + +type mockFenceServer struct{} + +func (m *mockFenceServer) handler(t *testing.T) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + switch { + case r.Method == http.MethodPost && path == fenceAccessTokenEndpoint: + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]string{"access_token": "new-access-token"}) + return + case r.Method == http.MethodGet && path == fenceUserEndpoint: + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]any{ + "username": "test-user", + "authz": map[string]any{ + "/resource": []map[string]string{ + {"method": "read", "service": "fence"}, + }, + }, + }) + return + case r.Method == http.MethodGet && path == shepherdVersionEndpoint: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`"2.0.0"`)) + return + case r.Method == http.MethodDelete && strings.HasPrefix(path, shepherdEndpoint+"/objects/"): + w.WriteHeader(http.StatusNoContent) + return + case r.Method == http.MethodDelete && strings.HasPrefix(path, fenceDataEndpoint+"/"): + w.WriteHeader(http.StatusNoContent) + return + case r.Method == http.MethodGet && strings.HasSuffix(path, "/download"): + // Shepherd download + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(map[string]string{"url": "https://download.url"}) + return + case r.Method == http.MethodGet && strings.Contains(path, fenceDataDownloadEndpoint+"/"): + // Fence download + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(FenceResponse{URL: "https://download.url"}) + return + case r.Method == http.MethodGet && path == "/data/buckets": + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(S3BucketsResponse{ + S3Buckets: map[string]*S3Bucket{ + "test-bucket": { + EndpointURL: "https://s3.amazonaws.com", + Provider: "s3", + Region: "us-east-1", + }, + }, + }) + return + case r.Method == http.MethodPost && path == fenceDataUploadEndpoint: + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(FenceResponse{GUID: "new-guid", URL: "https://upload.url"}) + return + case r.Method == http.MethodGet && strings.HasPrefix(path, fenceDataUploadEndpoint+"/"): + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(FenceResponse{URL: "https://upload.url"}) + return + } + + w.WriteHeader(http.StatusNotFound) + } +} + +func newTestClient(server *httptest.Server) FenceInterface { + cred := &conf.Credential{APIEndpoint: server.URL, Profile: "test", AccessToken: "test-token", APIKey: "test-key"} + logger, _ := logs.New("test") + config := conf.NewConfigure(logger.Logger) + req := request.NewRequestInterface(logger, cred, config) + return NewFenceClient(req, cred, logger.Logger) +} + +func TestFenceClient_NewAccessToken(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + token, err := client.NewAccessToken(context.Background()) + if err != nil { + t.Fatalf("NewAccessToken error: %v", err) + } + if token != "new-access-token" { + t.Errorf("expected token new-access-token, got %s", token) + } +} + +func TestFenceClient_CheckPrivileges(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + privs, err := client.CheckPrivileges(context.Background()) + if err != nil { + t.Fatalf("CheckPrivileges error: %v", err) + } + if _, ok := privs["/resource"]; !ok { + t.Errorf("expected /resource privilege") + } +} + +func TestFenceClient_CheckForShepherdAPI(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + cred := &conf.Credential{ + APIEndpoint: server.URL, + UseShepherd: "true", + } + logger, _ := logs.New("test") + req := request.NewRequestInterface(logger, cred, conf.NewConfigure(logger.Logger)) + client := NewFenceClient(req, cred, logger.Logger) + + hasShepherd, err := client.CheckForShepherdAPI(context.Background()) + if err != nil { + t.Fatalf("CheckForShepherdAPI error: %v", err) + } + if !hasShepherd { + t.Errorf("expected Shepherd to be detected") + } +} + +func TestFenceClient_DeleteRecord(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + // Test Fence fallback (shepherd check returns false or handled by mock behavior) + msg, err := client.DeleteRecord(context.Background(), "guid-1") + if err != nil { + t.Fatalf("DeleteRecord error: %v", err) + } + if !strings.Contains(msg, "has been deleted") { + t.Errorf("unexpected message: %s", msg) + } +} + +func TestFenceClient_GetBucketDetails(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + info, err := client.GetBucketDetails(context.Background(), "test-bucket") + if err != nil { + t.Fatalf("GetBucketDetails error: %v", err) + } + if info.Region != "us-east-1" { + t.Errorf("expected region us-east-1, got %s", info.Region) + } + if info.Provider != "s3" { + t.Errorf("expected provider s3, got %s", info.Provider) + } + + info, err = client.GetBucketDetails(context.Background(), "unknown-bucket") + if err != nil { + t.Fatalf("unexpected error for unknown bucket: %v", err) + } + if info != nil { + t.Errorf("expected nil info for unknown bucket") + } +} + +func TestFenceClient_UploadFlow(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + resp, err := client.InitUpload(context.Background(), "file.txt", "bucket", "") + if err != nil { + t.Fatalf("InitUpload error: %v", err) + } + if resp.URL != "https://upload.url" { + t.Errorf("expected upload URL, got %s", resp.URL) + } + + resp, err = client.GetUploadPresignedUrl(context.Background(), "guid-1", "file.txt", "bucket") + if err != nil { + t.Fatalf("GetUploadPresignedUrl error: %v", err) + } + if resp.URL != "https://upload.url" { + t.Errorf("expected upload URL, got %s", resp.URL) + } +} + +func TestFenceClient_GetDownloadPresignedUrl_Fence(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + url, err := client.GetDownloadPresignedUrl(context.Background(), "guid-1", "") + if err != nil { + t.Fatalf("GetDownloadPresignedUrl error: %v", err) + } + if url != "https://download.url" { + t.Errorf("expected download URL, got %s", url) + } +} + +func TestFenceClient_UserPing(t *testing.T) { + mock := &mockFenceServer{} + server := httptest.NewServer(mock.handler(t)) + defer server.Close() + + client := newTestClient(server) + + resp, err := client.UserPing(context.Background()) + if err != nil { + t.Fatalf("UserPing error: %v", err) + } + + if resp.Username != "test-user" { + t.Errorf("expected username test-user, got %s", resp.Username) + } + + if _, ok := resp.YourAccess["/resource"]; !ok { + t.Errorf("expected /resource access") + } + + if resp.BucketPrograms["test-bucket"] != "" { + // Our mock for /data/buckets returns a bucket but no programs by default unless we update it + // In my update to types.go, I added Programs to S3Bucket. + } +} diff --git a/fence/types.go b/fence/types.go new file mode 100644 index 0000000..ef4956b --- /dev/null +++ b/fence/types.go @@ -0,0 +1,94 @@ +package fence + +// MultipartPart represents a part of a multipart upload +type MultipartPart struct { + PartNumber int `json:"PartNumber"` + ETag string `json:"ETag"` +} + +// FenceResponse represents the standard response from Fence data endpoints +type FenceResponse struct { + URL string `json:"url"` + UploadURL string `json:"upload_url"` // Alias found in some Fence versions + GUID string `json:"guid"` + UploadID string `json:"uploadId"` + PresignedURL string `json:"presigned_url"` + FileName string `json:"file_name"` + URLs []string `json:"urls"` + Size int64 `json:"size"` +} + +// InitRequestObject represents the payload for initializing an upload +type InitRequestObject struct { + Filename string `json:"file_name"` + Bucket string `json:"bucket,omitempty"` + GUID string `json:"guid,omitempty"` +} + +// MultipartUploadRequestObject represents the payload for getting a presigned URL for a part +type MultipartUploadRequestObject struct { + Key string `json:"key"` + UploadID string `json:"uploadId"` + PartNumber int `json:"partNumber"` + Bucket string `json:"bucket,omitempty"` +} + +// MultipartCompleteRequestObject represents the payload for completing a multipart upload +type MultipartCompleteRequestObject struct { + Key string `json:"key"` + UploadID string `json:"uploadId"` + Parts []MultipartPart `json:"parts"` + Bucket string `json:"bucket,omitempty"` +} + +type S3Bucket struct { + EndpointURL string `json:"endpoint_url"` + Provider string `json:"provider,omitempty"` + Programs []string `json:"programs,omitempty"` + Region string `json:"region"` +} + +type S3BucketsResponse struct { + GSBuckets map[string]any `json:"GS_BUCKETS,omitempty"` + S3Buckets map[string]*S3Bucket `json:"S3_BUCKETS,omitempty"` + // Some versions of fence use lowercase + S3BucketsLower map[string]*S3Bucket `json:"s3_buckets,omitempty"` +} + +type UserPermission struct { + Method string `json:"method"` + Service string `json:"service"` +} + +type FenceUserResp struct { + Active bool `json:"active"` + Authz map[string][]UserPermission `json:"authz"` + Azp *string `json:"azp"` + CertificatesUploaded []any `json:"certificates_uploaded"` + DisplayName string `json:"display_name"` + Email string `json:"email"` + Ga4GhPassportV1 []any `json:"ga4gh_passport_v1"` + Groups []any `json:"groups"` + Idp string `json:"idp"` + IsAdmin bool `json:"is_admin"` + Message string `json:"message"` + Name string `json:"name"` + PhoneNumber string `json:"phone_number"` + PreferredUsername string `json:"preferred_username"` + PrimaryGoogleServiceAccount *string `json:"primary_google_service_account"` + ProjectAccess map[string]any `json:"project_access"` + Resources []string `json:"resources"` + ResourcesGranted []any `json:"resources_granted"` + Role string `json:"role"` + Sub string `json:"sub"` + UserID int `json:"user_id"` + Username string `json:"username"` +} + +type PingResp struct { + Profile string `yaml:"profile" json:"profile"` + Username string `yaml:"username" json:"username"` + Endpoint string `yaml:"endpoint" json:"endpoint"` + BucketPrograms map[string]string `yaml:"bucket_programs" json:"bucket_programs"` + YourAccess map[string]string `yaml:"your_access" json:"your_access"` +} diff --git a/g3client/client.go b/g3client/client.go new file mode 100644 index 0000000..f5c216f --- /dev/null +++ b/g3client/client.go @@ -0,0 +1,263 @@ +package g3client + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/calypr/data-client/conf" + "github.com/calypr/data-client/fence" + "github.com/calypr/data-client/logs" + "github.com/calypr/data-client/request" + "github.com/calypr/data-client/requestor" + "github.com/calypr/data-client/sower" + syconfig "github.com/calypr/syfon/client/config" + version "github.com/hashicorp/go-version" +) + +//go:generate go run go.uber.org/mock/mockgen@v0.6.0 -destination=../mocks/mock_gen3interface.go -package=mocks github.com/calypr/data-client/g3client Gen3Interface + +type Gen3Interface interface { + request.RequestInterface + Logger() *logs.Gen3Logger + Credentials() syconfig.CredentialManager + SyfonClient() SyfonClientInterface + FenceClient() fence.FenceInterface + RequestorClient() requestor.RequestorInterface + SowerClient() sower.SowerInterface +} + +func NewGen3InterfaceFromCredential(cred *conf.Credential, logger *logs.Gen3Logger, opts ...Option) Gen3Interface { + config := conf.NewConfigure(logger.Logger) + reqInterface := request.NewRequestInterface(logger, cred, config) + + client := &Gen3Client{ + config: config, + RequestInterface: reqInterface, + credential: cred, + logger: logger, + } + + for _, opt := range opts { + opt(client) + } + + client.initializeClients() + + return client +} + +func (g *Gen3Client) initializeClients() { + shouldInit := func(ct ClientType) bool { + if len(g.requestedClients) == 0 { + return true + } + for _, c := range g.requestedClients { + if c == ct { + return true + } + } + return false + } + + if shouldInit(FenceClient) { + g.fence = fence.NewFenceClient(g.RequestInterface, g.credential, g.logger.Logger) + } + if shouldInit(SyfonClient) { + g.syfon = buildSyfonClient(g.credential, g.logger, g.RequestInterface) + } + if shouldInit(SowerClient) { + g.sower = sower.NewSowerClient(g.RequestInterface, g.credential.APIEndpoint) + } + if shouldInit(RequestorClient) { + g.requestor = requestor.NewRequestorClient(g.RequestInterface, g.credential) + } +} + +type Gen3Client struct { + Ctx context.Context + fence fence.FenceInterface + syfon SyfonClientInterface + sower sower.SowerInterface + requestor requestor.RequestorInterface + config conf.ManagerInterface + request.RequestInterface + + credential *conf.Credential + creds syconfig.CredentialManager + logger *logs.Gen3Logger + + requestedClients []ClientType +} + +type ClientType string + +const ( + FenceClient ClientType = "fence" + SyfonClient ClientType = "syfon" + SowerClient ClientType = "sower" + RequestorClient ClientType = "requestor" +) + +type Option func(*Gen3Client) + +func WithClients(clients ...ClientType) Option { + return func(g *Gen3Client) { + g.requestedClients = clients + } +} + +func (g *Gen3Client) SyfonClient() SyfonClientInterface { + if g.syfon == nil { + g.syfon = buildSyfonClient(g.credential, g.logger, g.RequestInterface) + } + return g.syfon +} + +func (g *Gen3Client) FenceClient() fence.FenceInterface { + return g.fence +} + +func (g *Gen3Client) RequestorClient() requestor.RequestorInterface { + return g.requestor +} + +func (g *Gen3Client) SowerClient() sower.SowerInterface { + return g.sower +} + +func (g *Gen3Client) exportCredential(ctx context.Context, cred *conf.Credential) error { + if cred.Profile == "" { + return fmt.Errorf("profile name is required") + } + if cred.APIEndpoint == "" { + return fmt.Errorf("API endpoint is required") + } + + // Normalize endpoint + cred.APIEndpoint = strings.TrimSpace(cred.APIEndpoint) + cred.APIEndpoint = strings.TrimSuffix(cred.APIEndpoint, "/") + + // Validate URL format + parsedURL, err := conf.ValidateUrl(cred.APIEndpoint) + if err != nil { + return fmt.Errorf("invalid apiendpoint URL: %w", err) + } + fenceBase := parsedURL.Scheme + "://" + parsedURL.Host + if _, err := g.config.Load(cred.Profile); err != nil && !errors.Is(err, conf.ErrProfileNotFound) { + return err + } + + if cred.APIKey != "" { + // Always refresh the access token — ignore any old one that might be in the struct + token, err := g.fence.NewAccessToken(ctx) + if err != nil { + if strings.Contains(err.Error(), "401") { + return fmt.Errorf("authentication failed (401) for %s — your API key is invalid, revoked, or expired", fenceBase) + } + if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "no such host") { + return fmt.Errorf("cannot reach Fence at %s — is this a valid Gen3 commons?", fenceBase) + } + return fmt.Errorf("failed to refresh access token: %w", err) + } + g.credential.AccessToken = token + cred.AccessToken = token + } else { + g.logger.Warn("WARNING: Your profile will only be valid for 24 hours since you have only provided a refresh token for authentication") + } + + // Clean up shepherd flags + cred.UseShepherd = strings.TrimSpace(cred.UseShepherd) + cred.MinShepherdVersion = strings.TrimSpace(cred.MinShepherdVersion) + + if cred.MinShepherdVersion != "" { + if _, err = version.NewVersion(cred.MinShepherdVersion); err != nil { + return fmt.Errorf("invalid min-shepherd-version: %w", err) + } + } + + if err := g.config.Save(cred); err != nil { + return fmt.Errorf("failed to write config file: %w", err) + } + + return nil +} + +type gen3Credentials struct { + client *Gen3Client +} + +func (c *gen3Credentials) Current() *conf.Credential { + return c.client.credential +} + +func (c *gen3Credentials) Export(ctx context.Context, cred *conf.Credential) error { + return c.client.exportCredential(ctx, cred) +} + +func (g *Gen3Client) Credentials() syconfig.CredentialManager { + if g.creds == nil { + g.creds = &gen3Credentials{client: g} + } + return g.creds +} + +// EnsureValidCredential checks if the credential is valid and refreshes it if the access token is expired but the API key is valid. +// It accepts an optional fClient; if nil, it will initialize one internally if needed for refresh. +func EnsureValidCredential(ctx context.Context, cred *conf.Credential, config conf.ManagerInterface, logger *logs.Gen3Logger, fClient fence.FenceInterface) error { + if valid, err := config.IsCredentialValid(cred); !valid { + if strings.Contains(err.Error(), "access_token is invalid but api_key is valid") { + // Try to refresh the token + if fClient == nil { + reqInterface := request.NewRequestInterface(logger, cred, config) + fClient = fence.NewFenceClient(reqInterface, cred, logger.Logger) + } + newToken, refreshErr := fClient.NewAccessToken(ctx) + if refreshErr == nil { + cred.AccessToken = newToken + err = config.Save(cred) + if err != nil { + logger.Warn(fmt.Sprintf("Failed to save refreshed token: %v", err)) + } + return nil + } + return fmt.Errorf("failed to refresh access token: %v (original error: %v)", refreshErr, err) + } + return fmt.Errorf("invalid credential: %v", err) + } + return nil +} + +// NewGen3Interface returns a Gen3Client that embeds the credential and implements Gen3Interface. +func NewGen3Interface(profile string, logger *logs.Gen3Logger, opts ...Option) (Gen3Interface, error) { + config := conf.NewConfigure(logger.Logger) + cred, err := config.Load(profile) + if err != nil { + return nil, err + } + + reqInterface := request.NewRequestInterface(logger, cred, config) + + // We need a temporary Fence client to refresh tokens if needed + fClient := fence.NewFenceClient(reqInterface, cred, logger.Logger) + if err := EnsureValidCredential(context.Background(), cred, config, logger, fClient); err != nil { + return nil, err + } + + client := &Gen3Client{ + config: config, + RequestInterface: reqInterface, + credential: cred, + logger: logger, + } + + for _, opt := range opts { + opt(client) + } + + client.initializeClients() + + return client, nil +} +func (g *Gen3Client) Logger() *logs.Gen3Logger { return g.logger } diff --git a/g3client/client_test.go b/g3client/client_test.go new file mode 100644 index 0000000..4113a67 --- /dev/null +++ b/g3client/client_test.go @@ -0,0 +1,56 @@ +package g3client + +import ( + "net/http" + "testing" + + "github.com/calypr/data-client/conf" + "github.com/calypr/data-client/logs" + "github.com/calypr/data-client/request" + "github.com/hashicorp/go-retryablehttp" +) + +func TestGen3ClientInitializesSyfonClient(t *testing.T) { + logger, cleanup := logs.New("g3client-test", logs.WithNoConsole(), logs.WithNoMessageFile()) + t.Cleanup(cleanup) + + req := &request.Request{ + RetryClient: &retryablehttp.Client{HTTPClient: &http.Client{}}, + } + cred := &conf.Credential{ + Profile: "test", + APIEndpoint: "https://example.org", + } + client := &Gen3Client{ + RequestInterface: req, + credential: cred, + logger: logger, + requestedClients: []ClientType{SyfonClient}, + } + + client.initializeClients() + + syfon := client.SyfonClient() + if syfon == nil { + t.Fatal("expected syfon client to be initialized") + } + if syfon.Health() == nil || syfon.Data() == nil || syfon.Index() == nil || syfon.DRS() == nil || syfon.Buckets() == nil || syfon.Metrics() == nil || syfon.LFS() == nil { + t.Fatal("expected syfon services to be initialized") + } + + if client.FenceClient() != nil { + t.Fatal("did not expect fence client to be initialized when it was not requested") + } + if client.SowerClient() != nil { + t.Fatal("did not expect sower client to be initialized when it was not requested") + } + if client.RequestorClient() != nil { + t.Fatal("did not expect requestor client to be initialized when it was not requested") + } + + if got := client.Credentials(); got == nil { + t.Fatal("expected credentials manager") + } else if current := got.Current(); current != cred { + t.Fatal("expected credentials manager to return original credential") + } +} diff --git a/g3client/syfon_adapter.go b/g3client/syfon_adapter.go new file mode 100644 index 0000000..697d967 --- /dev/null +++ b/g3client/syfon_adapter.go @@ -0,0 +1,225 @@ +package g3client + +import ( + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + + "github.com/calypr/data-client/conf" + "github.com/calypr/data-client/logs" + "github.com/calypr/data-client/request" + "github.com/calypr/syfon/apigen/client/bucketapi" + "github.com/calypr/syfon/apigen/client/drs" + "github.com/calypr/syfon/apigen/client/internalapi" + "github.com/calypr/syfon/apigen/client/lfsapi" + "github.com/calypr/syfon/apigen/client/metricsapi" + sylogs "github.com/calypr/syfon/client/logs" + syrequest "github.com/calypr/syfon/client/request" + syfonclient "github.com/calypr/syfon/client/services" +) + +// SyfonClientInterface groups the syfon service interfaces that data-client needs. +type SyfonClientInterface interface { + Health() *syfonclient.HealthService + Data() *syfonclient.DataService + Index() *syfonclient.IndexService + DRS() *syfonclient.DRSService + Buckets() *syfonclient.BucketsService + Metrics() *syfonclient.MetricsService + LFS() *syfonclient.LFSService +} + +type syfonClient struct { + health *syfonclient.HealthService + data *syfonclient.DataService + index *syfonclient.IndexService + drs *syfonclient.DRSService + buckets *syfonclient.BucketsService + metrics *syfonclient.MetricsService + lfs *syfonclient.LFSService +} + +func (c *syfonClient) Health() *syfonclient.HealthService { return c.health } +func (c *syfonClient) Data() *syfonclient.DataService { return c.data } +func (c *syfonClient) Index() *syfonclient.IndexService { return c.index } +func (c *syfonClient) DRS() *syfonclient.DRSService { return c.drs } +func (c *syfonClient) Buckets() *syfonclient.BucketsService { return c.buckets } +func (c *syfonClient) Metrics() *syfonclient.MetricsService { return c.metrics } +func (c *syfonClient) LFS() *syfonclient.LFSService { return c.lfs } + +type syfonRequestAdapter struct { + req request.RequestInterface + baseURL string +} + +func (a *syfonRequestAdapter) Do(ctx context.Context, method, path string, body, out any, opts ...syrequest.RequestOption) error { + if a.req == nil { + return fmt.Errorf("request interface is required") + } + + rb := &syrequest.RequestBuilder{ + Method: method, + Url: a.joinURL(path), + Headers: map[string]string{}, + } + for _, opt := range opts { + opt(rb) + } + + if rb.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, rb.Timeout) + defer cancel() + } + + dcRB := a.req.New(method, rb.Url) + for key, value := range rb.Headers { + dcRB.WithHeader(key, value) + } + if rb.SkipAuth { + dcRB.WithSkipAuth(true) + } + if rb.Token != "" { + dcRB.WithToken(rb.Token) + } + if rb.PartSize != 0 { + dcRB.WithPartSize(rb.PartSize) + } + + if body != nil { + if reader, ok := body.(io.Reader); ok { + dcRB.WithBody(reader) + } else { + var err error + dcRB, err = dcRB.WithJSONBody(body) + if err != nil { + return err + } + } + } + + resp, err := a.req.Do(ctx, dcRB) + if err != nil { + return err + } + + switch target := out.(type) { + case **http.Response: + *target = resp + return nil + case *http.Response: + *target = *resp + return nil + } + + defer resp.Body.Close() + data, readErr := io.ReadAll(resp.Body) + if readErr != nil { + return readErr + } + if resp.StatusCode >= http.StatusBadRequest { + return &syrequest.ResponseError{ + Method: method, + URL: resp.Request.URL.String(), + Status: resp.StatusCode, + Body: strings.TrimSpace(string(data)), + Headers: resp.Header.Clone(), + } + } + if out != nil && len(data) > 0 { + if err := json.Unmarshal(data, out); err != nil { + return fmt.Errorf("decode response: %w", err) + } + } + return nil +} + +func (a *syfonRequestAdapter) joinURL(path string) string { + path = strings.TrimSpace(path) + if path == "" { + return a.baseURL + } + if strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") { + return path + } + return strings.TrimRight(a.baseURL, "/") + "/" + strings.TrimLeft(path, "/") +} + +func buildSyfonClient(cred *conf.Credential, logger *logs.Gen3Logger, req request.RequestInterface) SyfonClientInterface { + if cred == nil || req == nil { + return nil + } + + baseReq, ok := req.(*request.Request) + if !ok || baseReq.RetryClient == nil { + return nil + } + + httpClient := baseReq.RetryClient.StandardClient() + var slogLogger *slog.Logger + if logger != nil { + slogLogger = logger.Logger + } + syLogger := sylogs.NewGen3Logger(slogLogger, "", "") + syReq := &syfonRequestAdapter{ + req: req, + baseURL: cred.APIEndpoint, + } + + internalClient, err := internalapi.NewClientWithResponses( + strings.TrimRight(cred.APIEndpoint, "/"), + internalapi.WithHTTPClient(httpClient), + ) + if err != nil { + return nil + } + + drsClient, err := drs.NewClientWithResponses( + strings.TrimRight(cred.APIEndpoint, "/")+"/ga4gh/drs/v1", + drs.WithHTTPClient(httpClient), + ) + if err != nil { + return nil + } + + bucketClient, err := bucketapi.NewClientWithResponses( + strings.TrimRight(cred.APIEndpoint, "/"), + bucketapi.WithHTTPClient(httpClient), + ) + if err != nil { + return nil + } + + metricsClient, err := metricsapi.NewClientWithResponses( + strings.TrimRight(cred.APIEndpoint, "/"), + metricsapi.WithHTTPClient(httpClient), + ) + if err != nil { + return nil + } + + lfsClient, err := lfsapi.NewClientWithResponses( + strings.TrimRight(cred.APIEndpoint, "/"), + lfsapi.WithHTTPClient(httpClient), + ) + if err != nil { + return nil + } + + index := syfonclient.NewIndexService(internalClient, syReq) + drsSvc := syfonclient.NewDRSService(drsClient, index) + dataSvc := syfonclient.NewDataService(internalClient, syReq, syLogger, drsSvc) + return &syfonClient{ + health: syfonclient.NewHealthService(syReq), + data: dataSvc, + index: index, + drs: drsSvc, + buckets: syfonclient.NewBucketsService(bucketClient), + metrics: syfonclient.NewMetricsService(metricsClient), + lfs: syfonclient.NewLFSService(lfsClient), + } +} diff --git a/go.mod b/go.mod index 6515b39..05d0de8 100644 --- a/go.mod +++ b/go.mod @@ -1,26 +1,49 @@ module github.com/calypr/data-client -go 1.24.2 +go 1.26.2 require ( - github.com/golang-jwt/jwt/v5 v5.3.0 - github.com/hashicorp/go-multierror v1.1.1 - github.com/hashicorp/go-version v1.8.0 + github.com/calypr/syfon/apigen v0.2.6-0.20260503003649-cda722e27216 + github.com/calypr/syfon/client v0.2.7-0.20260503003649-cda722e27216 + github.com/golang-jwt/jwt/v5 v5.3.1 + github.com/hashicorp/go-retryablehttp v0.7.8 + github.com/hashicorp/go-version v1.9.0 github.com/spf13/cobra v1.10.2 - github.com/vbauerster/mpb/v8 v8.11.2 go.uber.org/mock v0.6.0 - golang.org/x/mod v0.31.0 - gopkg.in/ini.v1 v1.67.0 + gopkg.in/ini.v1 v1.67.1 + gopkg.in/yaml.v3 v3.0.1 ) require ( github.com/VividCortex/ewma v1.2.0 // indirect github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect - github.com/clipperhouse/stringish v0.1.1 // indirect - github.com/clipperhouse/uax29/v2 v2.3.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect + github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect + github.com/calypr/syfon v0.2.8-0.20260503003649-cda722e27216 // indirect + github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/gofiber/fiber/v3 v3.1.0 // indirect + github.com/gofiber/schema v1.7.0 // indirect + github.com/gofiber/utils/v2 v2.0.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/mattn/go-runewidth v0.0.19 // indirect + github.com/klauspost/compress v1.18.5 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-isatty v0.0.21 // indirect + github.com/mattn/go-runewidth v0.0.23 // indirect + github.com/oapi-codegen/runtime v1.4.0 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/spf13/pflag v1.0.10 // indirect - golang.org/x/sys v0.39.0 // indirect + github.com/tinylib/msgp v1.6.3 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.69.0 // indirect + github.com/vbauerster/mpb/v8 v8.12.0 // indirect + golang.org/x/crypto v0.50.0 // indirect + golang.org/x/net v0.53.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index bae303b..714a133 100644 --- a/go.sum +++ b/go.sum @@ -1,48 +1,127 @@ +github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs= -github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA= -github.com/clipperhouse/uax29/v2 v2.3.0 h1:SNdx9DVUqMoBuBoW3iLOj4FQv3dN5mDtuqwuhIGpJy4= -github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ= +github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/calypr/syfon v0.2.8-0.20260503003649-cda722e27216 h1:f/fr9r4N3yKARlX+RPiZmg6fJG3xuSLpeTvk6as6AxM= +github.com/calypr/syfon v0.2.8-0.20260503003649-cda722e27216/go.mod h1:m2jd8Snb+Gjc32AcOTdioZVjLmhsAXw7F4uzHQTGBtg= +github.com/calypr/syfon/apigen v0.2.6-0.20260503003649-cda722e27216 h1:MuPHiYQXGX7frvN3EQZ4ZM5RjRZ1eGVY0D4iXCNAj0s= +github.com/calypr/syfon/apigen v0.2.6-0.20260503003649-cda722e27216/go.mod h1:9JNwTgR57yKJlWqZpdqP+/l4zCNzH1EIFrW+e20PyMQ= +github.com/calypr/syfon/client v0.2.7-0.20260503003649-cda722e27216 h1:1LFo3dMc4ZQHml9zQsBE5/k2ZEcHT4EpqxL+Hr/ROjo= +github.com/calypr/syfon/client v0.2.7-0.20260503003649-cda722e27216/go.mod h1:DQSqNkxl9V3w08BiMconcJh3xtc+/Je7Xeo7qRH7wto= +github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= +github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= -github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-version v1.8.0 h1:KAkNb1HAiZd1ukkxDFGmokVZe1Xy9HG6NUp+bPle2i4= -github.com/hashicorp/go-version v1.8.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gofiber/fiber/v3 v3.1.0 h1:1p4I820pIa+FGxfwWuQZ5rAyX0WlGZbGT6Hnuxt6hKY= +github.com/gofiber/fiber/v3 v3.1.0/go.mod h1:n2nYQovvL9z3Too/FGOfgtERjW3GQcAUqgfoezGBZdU= +github.com/gofiber/schema v1.7.0 h1:yNM+FNRZjyYEli9Ey0AXRBrAY9jTnb+kmGs3lJGPvKg= +github.com/gofiber/schema v1.7.0/go.mod h1:A/X5Ffyru4p9eBdp99qu+nzviHzQiZ7odLT+TwxWhbk= +github.com/gofiber/utils/v2 v2.0.2 h1:ShRRssz0F3AhTlAQcuEj54OEDtWF7+HJDwEi/aa6QLI= +github.com/gofiber/utils/v2 v2.0.2/go.mod h1:+9Ub4NqQ+IaJoTliq5LfdmOJAA/Hzwf4pXOxOa3RrJ0= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= +github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw= -github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/juju/gnuflag v0.0.0-20171113085948-2ce1bb71843d/go.mod h1:2PavIy+JPciBPrBUjwbNvtwB6RQlve+hkpll6QSNmOE= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.21 h1:xYae+lCNBP7QuW4PUnNG61ffM4hVIfm+zUzDuSzYLGs= +github.com/mattn/go-isatty v0.0.21/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJlRFqopa4= +github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= +github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/oapi-codegen/runtime v1.4.0 h1:KLOSFOp7UzkbS7Cs1ms6NBEKYr0WmH2wZG0KKbd2er4= +github.com/oapi-codegen/runtime v1.4.0/go.mod h1:5sw5fxCDmnOzKNYmkVNF8d34kyUeejJEY8HNT2WaPec= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shamaton/msgpack/v3 v3.1.0 h1:jsk0vEAqVvvS9+fTZ5/EcQ9tz860c9pWxJ4Iwecz8gU= +github.com/shamaton/msgpack/v3 v3.1.0/go.mod h1:DcQG8jrdrQCIxr3HlMYkiXdMhK+KfN2CitkyzsQV4uc= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/vbauerster/mpb/v8 v8.11.2 h1:OqLoHznUVU7SKS/WV+1dB5/hm20YLheYupiHhL5+M1Y= -github.com/vbauerster/mpb/v8 v8.11.2/go.mod h1:mEB/M353al1a7wMUNtiymmPsEkGlJgeJmtlbY5adCJ8= +github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tinylib/msgp v1.6.3 h1:bCSxiTz386UTgyT1i0MSCvdbWjVW+8sG3PjkGsZQt4s= +github.com/tinylib/msgp v1.6.3/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.69.0 h1:fNLLESD2SooWeh2cidsuFtOcrEi4uB4m1mPrkJMZyVI= +github.com/valyala/fasthttp v1.69.0/go.mod h1:4wA4PfAraPlAsJ5jMSqCE2ug5tqUPwKXxVj8oNECGcw= +github.com/vbauerster/mpb/v8 v8.12.0 h1:+gneY3ifzc88tKDzOtfG8k8gfngCx615S2ZmFM4liWg= +github.com/vbauerster/mpb/v8 v8.12.0/go.mod h1:V02YIuMVo301Y1VE9VtZlD8s84OMsk+EKN6mwvf/588= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI= -golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg= -golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= -golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= +golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= +gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/client/logs/factory.go b/logs/factory.go similarity index 64% rename from client/logs/factory.go rename to logs/factory.go index f63a63b..ac44b54 100644 --- a/client/logs/factory.go +++ b/logs/factory.go @@ -2,14 +2,14 @@ package logs import ( "fmt" - "io" + "log/slog" "os" "os/user" "path/filepath" "time" ) -func New(profile string, opts ...Option) (*TeeLogger, func()) { +func New(profile string, opts ...Option) (*Gen3Logger, func()) { cfg := defaults() for _, o := range opts { o(cfg) @@ -19,15 +19,15 @@ func New(profile string, opts ...Option) (*TeeLogger, func()) { logDir := filepath.Join(usr.HomeDir, ".gen3", "logs") os.MkdirAll(logDir, 0755) - var writers []io.Writer + var handlers []slog.Handler var messageFile *os.File if cfg.baseLogger != nil { - writers = append(writers, cfg.baseLogger.Writer()) + handlers = append(handlers, cfg.baseLogger.Handler()) } if cfg.console { - writers = append(writers, os.Stderr) + handlers = append(handlers, slog.NewTextHandler(os.Stderr, nil)) } if cfg.messageFile { @@ -39,15 +39,26 @@ func New(profile string, opts ...Option) (*TeeLogger, func()) { f, err := os.OpenFile(filepath.Join(logDir, filename), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) if err == nil { messageFile = f - writers = append(writers, f) + handlers = append(handlers, slog.NewTextHandler(f, nil)) fmt.Fprintf(f, "[%s] Message log started\n", time.Now().Format(time.RFC3339)) } } - t := NewTeeLogger(logDir, profile, writers...) + var rootHandler slog.Handler + if len(handlers) == 0 { + rootHandler = slog.NewTextHandler(os.Stderr, nil) + } else if len(handlers) == 1 { + rootHandler = handlers[0] + } else { + rootHandler = NewTeeHandler(handlers...) + } + + sl := slog.New(NewProgressHandler(rootHandler)) + + t := NewGen3Logger(sl, logDir, profile) if cfg.enableScoreboard { - t.scoreboard = NewSB(5, t) + t.scoreboard = NewSB(5, t.Logger) } if cfg.failedLog { diff --git a/logs/handler.go b/logs/handler.go new file mode 100644 index 0000000..ac7ed48 --- /dev/null +++ b/logs/handler.go @@ -0,0 +1,102 @@ +package logs + +import ( + "context" + "log/slog" + + sycommon "github.com/calypr/syfon/client/common" +) + +// ProgressHandler is a slog.Handler that captures log messages and +// forwards them to a ProgressCallback if one is present in the context. +type ProgressHandler struct { + next slog.Handler +} + +func NewProgressHandler(next slog.Handler) *ProgressHandler { + if next == nil { + next = slog.Default().Handler() + } + return &ProgressHandler{next: next} +} + +func (h *ProgressHandler) Enabled(ctx context.Context, level slog.Level) bool { + return h.next.Enabled(ctx, level) +} + +func (h *ProgressHandler) Handle(ctx context.Context, r slog.Record) error { + // Call the next handler in the chain (original logging) + err := h.next.Handle(ctx, r) + + // In addition, try to bubble up to progress callback + cb := sycommon.GetProgress(ctx) + if cb != nil { + oid := sycommon.GetOid(ctx) + // We send an event of type "log" + attrs := make(map[string]any) + r.Attrs(func(a slog.Attr) bool { + attrs[a.Key] = a.Value.Any() + return true + }) + _ = cb(sycommon.ProgressEvent{ + Event: "log", + Oid: oid, + Message: r.Message, + Level: r.Level.String(), + Attrs: attrs, + }) + } + + return err +} + +func (h *ProgressHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return &ProgressHandler{next: h.next.WithAttrs(attrs)} +} + +func (h *ProgressHandler) WithGroup(name string) slog.Handler { + return &ProgressHandler{next: h.next.WithGroup(name)} +} + +// TeeHandler fans out log records to multiple handlers +type TeeHandler struct { + handlers []slog.Handler +} + +func NewTeeHandler(handlers ...slog.Handler) slog.Handler { + return &TeeHandler{handlers: handlers} +} + +func (h *TeeHandler) Enabled(ctx context.Context, level slog.Level) bool { + for _, hand := range h.handlers { + if hand.Enabled(ctx, level) { + return true + } + } + return false +} + +func (h *TeeHandler) Handle(ctx context.Context, r slog.Record) error { + for _, hand := range h.handlers { + if hand.Enabled(ctx, r.Level) { + _ = hand.Handle(ctx, r) + } + } + return nil +} + +func (h *TeeHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + newHandlers := make([]slog.Handler, len(h.handlers)) + for i, hand := range h.handlers { + newHandlers[i] = hand.WithAttrs(attrs) + } + return &TeeHandler{handlers: newHandlers} +} + +func (h *TeeHandler) WithGroup(name string) slog.Handler { + newHandlers := make([]slog.Handler, len(h.handlers)) + for i, hand := range h.handlers { + newHandlers[i] = hand.WithGroup(name) + } + return &TeeHandler{handlers: newHandlers} +} diff --git a/logs/logger.go b/logs/logger.go new file mode 100644 index 0000000..f6a55f0 --- /dev/null +++ b/logs/logger.go @@ -0,0 +1,35 @@ +package logs + +import ( + "log/slog" +) + +type Option func(*config) + +type config struct { + console bool + messageFile bool + failedLog bool + succeededLog bool + enableScoreboard bool + baseLogger *slog.Logger +} + +func WithConsole() Option { return func(c *config) { c.console = true } } +func WithNoConsole() Option { return func(c *config) { c.console = false } } +func WithMessageFile() Option { return func(c *config) { c.messageFile = true } } +func WithNoMessageFile() Option { return func(c *config) { c.messageFile = false } } +func WithFailedLog() Option { return func(c *config) { c.failedLog = true } } +func WithSucceededLog() Option { return func(c *config) { c.succeededLog = true } } +func WithScoreboard() Option { return func(c *config) { c.enableScoreboard = true } } +func WithBaseLogger(base *slog.Logger) Option { return func(c *config) { c.baseLogger = base } } + +func defaults() *config { + return &config{ + console: true, + messageFile: true, + failedLog: true, + succeededLog: true, + baseLogger: nil, + } +} diff --git a/logs/logger_test.go b/logs/logger_test.go new file mode 100644 index 0000000..382d2b5 --- /dev/null +++ b/logs/logger_test.go @@ -0,0 +1,208 @@ +package logs + +import ( + "io" + "log/slog" + "os" + "testing" +) + +func TestNewSlogNoOpLogger(t *testing.T) { + logger := NewSlogNoOpLogger() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + // Verify it's a valid slog.Logger + logger.Info("test message") // Should not panic + logger.Error("test error") // Should not panic +} + +func TestNew_WithDefaults(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + if logger.Logger == nil { + t.Error("Expected non-nil embedded slog logger") + } +} + +func TestNew_WithConsoleOption(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile, WithConsole()) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + // Test that we can log without errors + logger.Info("test console message") +} + +func TestNew_WithMessageFileOption(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile, WithMessageFile()) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + // Test that we can log without errors + logger.Info("test file message") +} + +func TestNew_WithScoreboardOption(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile, WithScoreboard()) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + if logger.Scoreboard() == nil { + t.Error("Expected non-nil scoreboard when WithScoreboard option is used") + } +} + +func TestNew_WithFailedLogOption(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile, WithFailedLog()) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + // Ensure failed-log helpers remain callable with syfon-backed logger. + _ = logger.GetFailedLogMap() +} + +func TestNew_WithSucceededLogOption(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile, WithSucceededLog()) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + // Ensure succeeded-log helpers remain callable with syfon-backed logger. + _ = logger.GetSucceededLogMap() +} + +func TestNew_WithBaseLogger(t *testing.T) { + profile := "test-profile" + baseLogger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + logger, cleanup := New(profile, WithBaseLogger(baseLogger)) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + // Test that we can log without errors + logger.Info("test with base logger") +} + +func TestNew_WithMultipleOptions(t *testing.T) { + profile := "test-profile" + baseLogger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + + logger, cleanup := New(profile, + WithBaseLogger(baseLogger), + WithConsole(), + WithMessageFile(), + WithScoreboard(), + ) + defer cleanup() + + if logger == nil { + t.Fatal("Expected non-nil logger") + } + + if logger.Logger == nil { + t.Error("Expected non-nil embedded slog logger") + } + + if logger.Scoreboard() == nil { + t.Error("Expected non-nil scoreboard") + } + + // Test that we can log without errors + logger.Info("test with multiple options") +} + +func TestGen3Logger_Info(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + // Should not panic + logger.Info("test info message") +} + +func TestGen3Logger_Error(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + // Should not panic + logger.Error("test error message") +} + +func TestGen3Logger_Warn(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + // Should not panic + logger.Warn("test warning message") +} + +func TestGen3Logger_Debug(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + // Should not panic + logger.Debug("test debug message") +} + +func TestGen3Logger_Printf(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + // Should not panic + logger.Printf("test printf message: %s", "value") +} + +func TestGen3Logger_Println(t *testing.T) { + profile := "test-profile" + logger, cleanup := New(profile) + defer cleanup() + + // Should not panic + logger.Println("test println message") +} + +// testLogger implements the Logger interface for testing +type testLogger struct { + writer io.Writer +} + +func (l *testLogger) Printf(format string, v ...any) {} +func (l *testLogger) Println(v ...any) {} +func (l *testLogger) Fatalf(format string, v ...any) {} +func (l *testLogger) Fatal(v ...any) {} +func (l *testLogger) Writer() io.Writer { return l.writer } diff --git a/logs/noop.go b/logs/noop.go new file mode 100644 index 0000000..f705772 --- /dev/null +++ b/logs/noop.go @@ -0,0 +1,11 @@ +package logs + +import ( + "io" + "log/slog" +) + +// NewSlogNoOpLogger creates a no-op slog logger for testing. +func NewSlogNoOpLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(io.Discard, nil)) +} diff --git a/client/logs/scoreboard.go b/logs/scoreboard.go similarity index 62% rename from client/logs/scoreboard.go rename to logs/scoreboard.go index a73117c..738a47a 100644 --- a/client/logs/scoreboard.go +++ b/logs/scoreboard.go @@ -1,33 +1,33 @@ package logs import ( - "context" "fmt" + "io" + "log/slog" + "os" "sync" "text/tabwriter" ) -type key int - -const scoreboardKey key = 0 - // Scoreboard holds retry statistics type Scoreboard struct { mu sync.Mutex Counts []int // index 0 = success on first try, 1 = after 1 retry, ..., last = failed - log Logger + logger *slog.Logger + writer io.Writer } -// New creates a new scoreboard +// NewSB creates a new scoreboard // maxRetryCount = how many retries you allow before giving up -func NewSB(maxRetryCount int, log Logger) *Scoreboard { +func NewSB(maxRetryCount int, logger *slog.Logger) *Scoreboard { return &Scoreboard{ Counts: make([]int, maxRetryCount+2), // +2: one for success-on-first, one for final failure - log: log, + logger: logger, + writer: os.Stderr, } } -// Increment records a result after `retryCount` attempts +// IncrementSB records a result after `retryCount` attempts // retryCount == 0 → succeeded on first try // retryCount == max → final failure func (s *Scoreboard) IncrementSB(retryCount int) { @@ -43,7 +43,7 @@ func (s *Scoreboard) IncrementSB(retryCount int) { s.Counts[retryCount]++ } -// Print the beautiful table at the end +// PrintSB prints the beautiful table at the end func (s *Scoreboard) PrintSB() { s.mu.Lock() defer s.mu.Unlock() @@ -56,8 +56,8 @@ func (s *Scoreboard) PrintSB() { return } - s.log.Println("\n\nSubmission Results") - w := tabwriter.NewWriter(s.log.Writer(), 0, 0, 2, ' ', 0) + s.logger.Info("Submission Results") + w := tabwriter.NewWriter(s.writer, 0, 0, 2, ' ', 0) for i, count := range s.Counts { if i == 0 { @@ -73,16 +73,3 @@ func (s *Scoreboard) PrintSB() { fmt.Fprintf(w, "TOTAL\t%d\n", total) w.Flush() } - -// Context helpers — so you don't have to pass scoreboard around - -func NewSBContext(parent context.Context, sb *Scoreboard) context.Context { - return context.WithValue(parent, scoreboardKey, sb) -} - -func FromSBContext(ctx context.Context) (*Scoreboard, error) { - if sb, ok := ctx.Value(scoreboardKey).(*Scoreboard); ok { - return sb, nil - } - return nil, fmt.Errorf("Scoreboard is not of type Scoreboard") -} diff --git a/logs/tee_logger.go b/logs/tee_logger.go new file mode 100644 index 0000000..998a195 --- /dev/null +++ b/logs/tee_logger.go @@ -0,0 +1,217 @@ +package logs + +import ( + "context" + "encoding/json" + "fmt" + "io" + "maps" + "os" + "runtime" + "sync" + "time" + + "log/slog" + + sycommon "github.com/calypr/syfon/client/common" +) + +// --- Gen3Logger Implementation --- +type Gen3Logger struct { + *slog.Logger + mu sync.RWMutex + scoreboard *Scoreboard + + failedMu sync.Mutex + FailedMap map[string]sycommon.RetryObject // Maps filePath to FileMetadata + failedPath string + + succeededMu sync.Mutex + succeededMap map[string]string // Maps filePath to GUID + succeededPath string +} + +// NewGen3Logger creates a new Gen3Logger wrapping the provided slog.Logger. +func NewGen3Logger(logger *slog.Logger, logDir, profile string) *Gen3Logger { + if logger == nil { + logger = slog.New(slog.NewTextHandler(os.Stdout, nil)) + } + return &Gen3Logger{ + Logger: logger, + FailedMap: make(map[string]sycommon.RetryObject), + succeededMap: make(map[string]string), + } +} + +// loadJSON is an internal helper to load JSON from a file path. +func loadJSON(path string, v any) { + data, _ := os.ReadFile(path) + if len(data) > 0 { + json.Unmarshal(data, v) + } +} + +// --- Core logging helper --- + +// logWithSkip logs a message at the given level, skipping `skip` stack frames for source attribution. +func (t *Gen3Logger) logWithSkip(ctx context.Context, level slog.Level, skip int, msg string, args ...any) { + if !t.Enabled(ctx, level) { + return + } + var pcs [1]uintptr + runtime.Callers(skip, pcs[:]) + r := slog.NewRecord(time.Now(), level, msg, pcs[0]) + r.Add(args...) + _ = t.Handler().Handle(ctx, r) +} + +// --- slog.Logger Method Overrides for accurate source attribution --- + +func (t *Gen3Logger) Info(msg string, args ...any) { + t.logWithSkip(context.Background(), slog.LevelInfo, 3, msg, args...) +} + +func (t *Gen3Logger) InfoContext(ctx context.Context, msg string, args ...any) { + t.logWithSkip(ctx, slog.LevelInfo, 3, msg, args...) +} + +func (t *Gen3Logger) Error(msg string, args ...any) { + t.logWithSkip(context.Background(), slog.LevelError, 3, msg, args...) +} + +func (t *Gen3Logger) ErrorContext(ctx context.Context, msg string, args ...any) { + t.logWithSkip(ctx, slog.LevelError, 3, msg, args...) +} + +func (t *Gen3Logger) Warn(msg string, args ...any) { + t.logWithSkip(context.Background(), slog.LevelWarn, 3, msg, args...) +} + +func (t *Gen3Logger) WarnContext(ctx context.Context, msg string, args ...any) { + t.logWithSkip(ctx, slog.LevelWarn, 3, msg, args...) +} + +func (t *Gen3Logger) Debug(msg string, args ...any) { + t.logWithSkip(context.Background(), slog.LevelDebug, 3, msg, args...) +} + +func (t *Gen3Logger) DebugContext(ctx context.Context, msg string, args ...any) { + t.logWithSkip(ctx, slog.LevelDebug, 3, msg, args...) +} + +// --- Legacy fmt-style methods --- + +func (t *Gen3Logger) Printf(format string, v ...any) { + t.logWithSkip(context.Background(), slog.LevelInfo, 3, fmt.Sprintf(format, v...)) +} + +func (t *Gen3Logger) Println(v ...any) { + t.logWithSkip(context.Background(), slog.LevelInfo, 3, fmt.Sprint(v...)) +} + +func (t *Gen3Logger) Fatalf(format string, v ...any) { + t.logWithSkip(context.Background(), slog.LevelError, 3, fmt.Sprintf(format, v...)) + os.Exit(1) +} + +func (t *Gen3Logger) Fatal(v ...any) { + t.logWithSkip(context.Background(), slog.LevelError, 3, fmt.Sprint(v...)) + os.Exit(1) +} + +// Writer returns os.Stderr for legacy compatibility (used by Scoreboard's tabwriter). +func (t *Gen3Logger) Writer() io.Writer { + return os.Stderr +} + +// Scoreboard returns the embedded Scoreboard. +func (t *Gen3Logger) Scoreboard() *Scoreboard { + return t.scoreboard +} + +// --- Succeeded/Failed log map methods --- + +func (t *Gen3Logger) GetSucceededLogMap() map[string]string { + t.succeededMu.Lock() + defer t.succeededMu.Unlock() + copiedMap := make(map[string]string, len(t.succeededMap)) + maps.Copy(copiedMap, t.succeededMap) + return copiedMap +} + +func (t *Gen3Logger) GetFailedLogMap() map[string]sycommon.RetryObject { + t.failedMu.Lock() + defer t.failedMu.Unlock() + copiedMap := make(map[string]sycommon.RetryObject, len(t.FailedMap)) + maps.Copy(copiedMap, t.FailedMap) + return copiedMap +} + +func (t *Gen3Logger) DeleteFromFailedLog(path string) { + t.failedMu.Lock() + defer t.failedMu.Unlock() + delete(t.FailedMap, path) +} + +func (t *Gen3Logger) GetSucceededCount() int { + return len(t.succeededMap) +} + +func (t *Gen3Logger) writeFailedSync(e sycommon.RetryObject) { + t.failedMu.Lock() + defer t.failedMu.Unlock() + t.FailedMap[e.SourcePath] = e + data, _ := json.MarshalIndent(t.FailedMap, "", " ") + os.WriteFile(t.failedPath, data, 0644) +} + +func (t *Gen3Logger) writeSucceededSync(path, guid string) { + t.succeededMu.Lock() + defer t.succeededMu.Unlock() + t.succeededMap[path] = guid + data, _ := json.MarshalIndent(t.succeededMap, "", " ") + os.WriteFile(t.succeededPath, data, 0644) +} + +// --- Tracking Methods --- + +// --- Tracking Methods --- + +func (t *Gen3Logger) Failed(filePath, filename string, metadata sycommon.FileMetadata, guid string, retryCount int, multipart bool) { + t.failedHelper(context.Background(), filePath, filename, metadata, guid, retryCount, multipart, 4) +} + +func (t *Gen3Logger) FailedContext(ctx context.Context, filePath, filename string, metadata sycommon.FileMetadata, guid string, retryCount int, multipart bool) { + t.failedHelper(ctx, filePath, filename, metadata, guid, retryCount, multipart, 4) +} + +func (t *Gen3Logger) failedHelper(ctx context.Context, filePath, filename string, metadata sycommon.FileMetadata, guid string, retryCount int, multipart bool, skip int) { + msg := fmt.Sprintf("Failed: %s (GUID: %s, Retry: %d)", filePath, guid, retryCount) + t.logWithSkip(ctx, slog.LevelError, skip, msg) + if t.failedPath != "" { + t.writeFailedSync(sycommon.RetryObject{ + SourcePath: filePath, + ObjectKey: filename, + FileMetadata: metadata, + GUID: guid, + RetryCount: retryCount, + Multipart: multipart, + }) + } +} + +func (t *Gen3Logger) Succeeded(filePath, guid string) { + t.succeededHelper(context.Background(), filePath, guid, 4) +} + +func (t *Gen3Logger) SucceededContext(ctx context.Context, filePath, guid string) { + t.succeededHelper(ctx, filePath, guid, 4) +} + +func (t *Gen3Logger) succeededHelper(ctx context.Context, filePath, guid string, skip int) { + msg := fmt.Sprintf("Succeeded: %s (GUID: %s)", filePath, guid) + t.logWithSkip(ctx, slog.LevelDebug, skip, msg) + if t.succeededPath != "" { + t.writeSucceededSync(filePath, guid) + } +} diff --git a/main.go b/main.go index 00bb0f7..dd6e829 100644 --- a/main.go +++ b/main.go @@ -1,9 +1,9 @@ package main import ( - "github.com/calypr/data-client/client/g3cmd" + "github.com/calypr/data-client/cmd" ) func main() { - g3cmd.Execute() + cmd.Execute() } diff --git a/mocks/mock_configure.go b/mocks/mock_configure.go new file mode 100644 index 0000000..dac723e --- /dev/null +++ b/mocks/mock_configure.go @@ -0,0 +1,129 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/calypr/data-client/conf (interfaces: ManagerInterface) +// +// Generated by this command: +// +// mockgen -destination=../mocks/mock_configure.go -package=mocks github.com/calypr/data-client/conf ManagerInterface +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + reflect "reflect" + + conf "github.com/calypr/data-client/conf" + gomock "go.uber.org/mock/gomock" +) + +// MockManagerInterface is a mock of ManagerInterface interface. +type MockManagerInterface struct { + ctrl *gomock.Controller + recorder *MockManagerInterfaceMockRecorder + isgomock struct{} +} + +// MockManagerInterfaceMockRecorder is the mock recorder for MockManagerInterface. +type MockManagerInterfaceMockRecorder struct { + mock *MockManagerInterface +} + +// NewMockManagerInterface creates a new mock instance. +func NewMockManagerInterface(ctrl *gomock.Controller) *MockManagerInterface { + mock := &MockManagerInterface{ctrl: ctrl} + mock.recorder = &MockManagerInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockManagerInterface) EXPECT() *MockManagerInterfaceMockRecorder { + return m.recorder +} + +// EnsureExists mocks base method. +func (m *MockManagerInterface) EnsureExists() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureExists") + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureExists indicates an expected call of EnsureExists. +func (mr *MockManagerInterfaceMockRecorder) EnsureExists() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureExists", reflect.TypeOf((*MockManagerInterface)(nil).EnsureExists)) +} + +// Import mocks base method. +func (m *MockManagerInterface) Import(filePath, fenceToken string) (*conf.Credential, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Import", filePath, fenceToken) + ret0, _ := ret[0].(*conf.Credential) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Import indicates an expected call of Import. +func (mr *MockManagerInterfaceMockRecorder) Import(filePath, fenceToken any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Import", reflect.TypeOf((*MockManagerInterface)(nil).Import), filePath, fenceToken) +} + +// IsCredentialValid mocks base method. +func (m *MockManagerInterface) IsCredentialValid(arg0 *conf.Credential) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsCredentialValid", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsCredentialValid indicates an expected call of IsCredentialValid. +func (mr *MockManagerInterfaceMockRecorder) IsCredentialValid(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsCredentialValid", reflect.TypeOf((*MockManagerInterface)(nil).IsCredentialValid), arg0) +} + +// IsTokenValid mocks base method. +func (m *MockManagerInterface) IsTokenValid(arg0 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsTokenValid", arg0) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsTokenValid indicates an expected call of IsTokenValid. +func (mr *MockManagerInterfaceMockRecorder) IsTokenValid(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsTokenValid", reflect.TypeOf((*MockManagerInterface)(nil).IsTokenValid), arg0) +} + +// Load mocks base method. +func (m *MockManagerInterface) Load(profile string) (*conf.Credential, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Load", profile) + ret0, _ := ret[0].(*conf.Credential) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Load indicates an expected call of Load. +func (mr *MockManagerInterfaceMockRecorder) Load(profile any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Load", reflect.TypeOf((*MockManagerInterface)(nil).Load), profile) +} + +// Save mocks base method. +func (m *MockManagerInterface) Save(cred *conf.Credential) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Save", cred) + ret0, _ := ret[0].(error) + return ret0 +} + +// Save indicates an expected call of Save. +func (mr *MockManagerInterfaceMockRecorder) Save(cred any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockManagerInterface)(nil).Save), cred) +} diff --git a/mocks/mock_fence.go b/mocks/mock_fence.go new file mode 100644 index 0000000..004b633 --- /dev/null +++ b/mocks/mock_fence.go @@ -0,0 +1,281 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/calypr/data-client/fence (interfaces: FenceInterface) +// +// Generated by this command: +// +// mockgen -destination=../mocks/mock_fence.go -package=mocks github.com/calypr/data-client/fence FenceInterface +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + http "net/http" + reflect "reflect" + + fence "github.com/calypr/data-client/fence" + request "github.com/calypr/data-client/request" + gomock "go.uber.org/mock/gomock" +) + +// MockFenceInterface is a mock of FenceInterface interface. +type MockFenceInterface struct { + ctrl *gomock.Controller + recorder *MockFenceInterfaceMockRecorder + isgomock struct{} +} + +// MockFenceInterfaceMockRecorder is the mock recorder for MockFenceInterface. +type MockFenceInterfaceMockRecorder struct { + mock *MockFenceInterface +} + +// NewMockFenceInterface creates a new mock instance. +func NewMockFenceInterface(ctrl *gomock.Controller) *MockFenceInterface { + mock := &MockFenceInterface{ctrl: ctrl} + mock.recorder = &MockFenceInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFenceInterface) EXPECT() *MockFenceInterfaceMockRecorder { + return m.recorder +} + +// CheckForShepherdAPI mocks base method. +func (m *MockFenceInterface) CheckForShepherdAPI(ctx context.Context) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckForShepherdAPI", ctx) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CheckForShepherdAPI indicates an expected call of CheckForShepherdAPI. +func (mr *MockFenceInterfaceMockRecorder) CheckForShepherdAPI(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckForShepherdAPI", reflect.TypeOf((*MockFenceInterface)(nil).CheckForShepherdAPI), ctx) +} + +// CheckPrivileges mocks base method. +func (m *MockFenceInterface) CheckPrivileges(ctx context.Context) (map[string]any, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckPrivileges", ctx) + ret0, _ := ret[0].(map[string]any) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CheckPrivileges indicates an expected call of CheckPrivileges. +func (mr *MockFenceInterfaceMockRecorder) CheckPrivileges(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckPrivileges", reflect.TypeOf((*MockFenceInterface)(nil).CheckPrivileges), ctx) +} + +// CompleteMultipartUpload mocks base method. +func (m *MockFenceInterface) CompleteMultipartUpload(ctx context.Context, key, uploadID string, parts []fence.MultipartPart, bucket string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CompleteMultipartUpload", ctx, key, uploadID, parts, bucket) + ret0, _ := ret[0].(error) + return ret0 +} + +// CompleteMultipartUpload indicates an expected call of CompleteMultipartUpload. +func (mr *MockFenceInterfaceMockRecorder) CompleteMultipartUpload(ctx, key, uploadID, parts, bucket any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CompleteMultipartUpload", reflect.TypeOf((*MockFenceInterface)(nil).CompleteMultipartUpload), ctx, key, uploadID, parts, bucket) +} + +// DeleteRecord mocks base method. +func (m *MockFenceInterface) DeleteRecord(ctx context.Context, guid string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteRecord", ctx, guid) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteRecord indicates an expected call of DeleteRecord. +func (mr *MockFenceInterfaceMockRecorder) DeleteRecord(ctx, guid any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRecord", reflect.TypeOf((*MockFenceInterface)(nil).DeleteRecord), ctx, guid) +} + +// Do mocks base method. +func (m *MockFenceInterface) Do(ctx context.Context, req *request.RequestBuilder) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Do", ctx, req) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Do indicates an expected call of Do. +func (mr *MockFenceInterfaceMockRecorder) Do(ctx, req any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Do", reflect.TypeOf((*MockFenceInterface)(nil).Do), ctx, req) +} + +// GenerateMultipartPresignedURL mocks base method. +func (m *MockFenceInterface) GenerateMultipartPresignedURL(ctx context.Context, key, uploadID string, partNumber int, bucket string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GenerateMultipartPresignedURL", ctx, key, uploadID, partNumber, bucket) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GenerateMultipartPresignedURL indicates an expected call of GenerateMultipartPresignedURL. +func (mr *MockFenceInterfaceMockRecorder) GenerateMultipartPresignedURL(ctx, key, uploadID, partNumber, bucket any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GenerateMultipartPresignedURL", reflect.TypeOf((*MockFenceInterface)(nil).GenerateMultipartPresignedURL), ctx, key, uploadID, partNumber, bucket) +} + +// GetBucketDetails mocks base method. +func (m *MockFenceInterface) GetBucketDetails(ctx context.Context, bucket string) (*fence.S3Bucket, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBucketDetails", ctx, bucket) + ret0, _ := ret[0].(*fence.S3Bucket) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetBucketDetails indicates an expected call of GetBucketDetails. +func (mr *MockFenceInterfaceMockRecorder) GetBucketDetails(ctx, bucket any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBucketDetails", reflect.TypeOf((*MockFenceInterface)(nil).GetBucketDetails), ctx, bucket) +} + +// GetDownloadPresignedUrl mocks base method. +func (m *MockFenceInterface) GetDownloadPresignedUrl(ctx context.Context, guid, protocolText string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDownloadPresignedUrl", ctx, guid, protocolText) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDownloadPresignedUrl indicates an expected call of GetDownloadPresignedUrl. +func (mr *MockFenceInterfaceMockRecorder) GetDownloadPresignedUrl(ctx, guid, protocolText any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDownloadPresignedUrl", reflect.TypeOf((*MockFenceInterface)(nil).GetDownloadPresignedUrl), ctx, guid, protocolText) +} + +// GetUploadPresignedUrl mocks base method. +func (m *MockFenceInterface) GetUploadPresignedUrl(ctx context.Context, guid, filename, bucket string) (fence.FenceResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetUploadPresignedUrl", ctx, guid, filename, bucket) + ret0, _ := ret[0].(fence.FenceResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetUploadPresignedUrl indicates an expected call of GetUploadPresignedUrl. +func (mr *MockFenceInterfaceMockRecorder) GetUploadPresignedUrl(ctx, guid, filename, bucket any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetUploadPresignedUrl", reflect.TypeOf((*MockFenceInterface)(nil).GetUploadPresignedUrl), ctx, guid, filename, bucket) +} + +// InitMultipartUpload mocks base method. +func (m *MockFenceInterface) InitMultipartUpload(ctx context.Context, filename, bucket, guid string) (fence.FenceResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitMultipartUpload", ctx, filename, bucket, guid) + ret0, _ := ret[0].(fence.FenceResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// InitMultipartUpload indicates an expected call of InitMultipartUpload. +func (mr *MockFenceInterfaceMockRecorder) InitMultipartUpload(ctx, filename, bucket, guid any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitMultipartUpload", reflect.TypeOf((*MockFenceInterface)(nil).InitMultipartUpload), ctx, filename, bucket, guid) +} + +// InitUpload mocks base method. +func (m *MockFenceInterface) InitUpload(ctx context.Context, filename, bucket, guid string) (fence.FenceResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InitUpload", ctx, filename, bucket, guid) + ret0, _ := ret[0].(fence.FenceResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// InitUpload indicates an expected call of InitUpload. +func (mr *MockFenceInterfaceMockRecorder) InitUpload(ctx, filename, bucket, guid any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InitUpload", reflect.TypeOf((*MockFenceInterface)(nil).InitUpload), ctx, filename, bucket, guid) +} + +// New mocks base method. +func (m *MockFenceInterface) New(method, url string) *request.RequestBuilder { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "New", method, url) + ret0, _ := ret[0].(*request.RequestBuilder) + return ret0 +} + +// New indicates an expected call of New. +func (mr *MockFenceInterfaceMockRecorder) New(method, url any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "New", reflect.TypeOf((*MockFenceInterface)(nil).New), method, url) +} + +// NewAccessToken mocks base method. +func (m *MockFenceInterface) NewAccessToken(ctx context.Context) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewAccessToken", ctx) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NewAccessToken indicates an expected call of NewAccessToken. +func (mr *MockFenceInterfaceMockRecorder) NewAccessToken(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewAccessToken", reflect.TypeOf((*MockFenceInterface)(nil).NewAccessToken), ctx) +} + +// ParseFenceURLResponse mocks base method. +func (m *MockFenceInterface) ParseFenceURLResponse(resp *http.Response) (fence.FenceResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ParseFenceURLResponse", resp) + ret0, _ := ret[0].(fence.FenceResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ParseFenceURLResponse indicates an expected call of ParseFenceURLResponse. +func (mr *MockFenceInterfaceMockRecorder) ParseFenceURLResponse(resp any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ParseFenceURLResponse", reflect.TypeOf((*MockFenceInterface)(nil).ParseFenceURLResponse), resp) +} + +// RefreshToken mocks base method. +func (m *MockFenceInterface) RefreshToken(ctx context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RefreshToken", ctx) + ret0, _ := ret[0].(error) + return ret0 +} + +// RefreshToken indicates an expected call of RefreshToken. +func (mr *MockFenceInterfaceMockRecorder) RefreshToken(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RefreshToken", reflect.TypeOf((*MockFenceInterface)(nil).RefreshToken), ctx) +} + +// UserPing mocks base method. +func (m *MockFenceInterface) UserPing(ctx context.Context) (*fence.PingResp, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UserPing", ctx) + ret0, _ := ret[0].(*fence.PingResp) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// UserPing indicates an expected call of UserPing. +func (mr *MockFenceInterfaceMockRecorder) UserPing(ctx any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UserPing", reflect.TypeOf((*MockFenceInterface)(nil).UserPing), ctx) +} diff --git a/mocks/mock_gen3interface.go b/mocks/mock_gen3interface.go new file mode 100644 index 0000000..adb4446 --- /dev/null +++ b/mocks/mock_gen3interface.go @@ -0,0 +1,162 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/calypr/data-client/g3client (interfaces: Gen3Interface) +// +// Generated by this command: +// +// mockgen -destination=../mocks/mock_gen3interface.go -package=mocks github.com/calypr/data-client/g3client Gen3Interface +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + http "net/http" + reflect "reflect" + + fence "github.com/calypr/data-client/fence" + g3client "github.com/calypr/data-client/g3client" + logs "github.com/calypr/data-client/logs" + request "github.com/calypr/data-client/request" + requestor "github.com/calypr/data-client/requestor" + sower "github.com/calypr/data-client/sower" + syconfig "github.com/calypr/syfon/client/config" + gomock "go.uber.org/mock/gomock" +) + +// MockGen3Interface is a mock of Gen3Interface interface. +type MockGen3Interface struct { + ctrl *gomock.Controller + recorder *MockGen3InterfaceMockRecorder + isgomock struct{} +} + +// MockGen3InterfaceMockRecorder is the mock recorder for MockGen3Interface. +type MockGen3InterfaceMockRecorder struct { + mock *MockGen3Interface +} + +// NewMockGen3Interface creates a new mock instance. +func NewMockGen3Interface(ctrl *gomock.Controller) *MockGen3Interface { + mock := &MockGen3Interface{ctrl: ctrl} + mock.recorder = &MockGen3InterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockGen3Interface) EXPECT() *MockGen3InterfaceMockRecorder { + return m.recorder +} + +// Credentials mocks base method. +func (m *MockGen3Interface) Credentials() syconfig.CredentialManager { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Credentials") + ret0, _ := ret[0].(syconfig.CredentialManager) + return ret0 +} + +// Credentials indicates an expected call of Credentials. +func (mr *MockGen3InterfaceMockRecorder) Credentials() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Credentials", reflect.TypeOf((*MockGen3Interface)(nil).Credentials)) +} + +// SyfonClient mocks base method. +func (m *MockGen3Interface) SyfonClient() g3client.SyfonClientInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SyfonClient") + ret0, _ := ret[0].(g3client.SyfonClientInterface) + return ret0 +} + +// SyfonClient indicates an expected call of SyfonClient. +func (mr *MockGen3InterfaceMockRecorder) SyfonClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyfonClient", reflect.TypeOf((*MockGen3Interface)(nil).SyfonClient)) +} + +// Do mocks base method. +func (m *MockGen3Interface) Do(ctx context.Context, req *request.RequestBuilder) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Do", ctx, req) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Do indicates an expected call of Do. +func (mr *MockGen3InterfaceMockRecorder) Do(ctx, req any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Do", reflect.TypeOf((*MockGen3Interface)(nil).Do), ctx, req) +} + +// FenceClient mocks base method. +func (m *MockGen3Interface) FenceClient() fence.FenceInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FenceClient") + ret0, _ := ret[0].(fence.FenceInterface) + return ret0 +} + +// FenceClient indicates an expected call of FenceClient. +func (mr *MockGen3InterfaceMockRecorder) FenceClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FenceClient", reflect.TypeOf((*MockGen3Interface)(nil).FenceClient)) +} + +// Logger mocks base method. +func (m *MockGen3Interface) Logger() *logs.Gen3Logger { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Logger") + ret0, _ := ret[0].(*logs.Gen3Logger) + return ret0 +} + +// Logger indicates an expected call of Logger. +func (mr *MockGen3InterfaceMockRecorder) Logger() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Logger", reflect.TypeOf((*MockGen3Interface)(nil).Logger)) +} + +// New mocks base method. +func (m *MockGen3Interface) New(method, url string) *request.RequestBuilder { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "New", method, url) + ret0, _ := ret[0].(*request.RequestBuilder) + return ret0 +} + +// New indicates an expected call of New. +func (mr *MockGen3InterfaceMockRecorder) New(method, url any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "New", reflect.TypeOf((*MockGen3Interface)(nil).New), method, url) +} + +// RequestorClient mocks base method. +func (m *MockGen3Interface) RequestorClient() requestor.RequestorInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RequestorClient") + ret0, _ := ret[0].(requestor.RequestorInterface) + return ret0 +} + +// RequestorClient indicates an expected call of RequestorClient. +func (mr *MockGen3InterfaceMockRecorder) RequestorClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RequestorClient", reflect.TypeOf((*MockGen3Interface)(nil).RequestorClient)) +} + +// SowerClient mocks base method. +func (m *MockGen3Interface) SowerClient() sower.SowerInterface { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SowerClient") + ret0, _ := ret[0].(sower.SowerInterface) + return ret0 +} + +// SowerClient indicates an expected call of SowerClient. +func (mr *MockGen3InterfaceMockRecorder) SowerClient() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SowerClient", reflect.TypeOf((*MockGen3Interface)(nil).SowerClient)) +} diff --git a/mocks/mock_request.go b/mocks/mock_request.go new file mode 100644 index 0000000..d0d0fff --- /dev/null +++ b/mocks/mock_request.go @@ -0,0 +1,72 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/calypr/data-client/request (interfaces: RequestInterface) +// +// Generated by this command: +// +// mockgen -destination=./mocks/mock_request.go -package=mocks github.com/calypr/data-client/request RequestInterface +// + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + http "net/http" + reflect "reflect" + + request "github.com/calypr/syfon/client/request" + gomock "go.uber.org/mock/gomock" +) + +// MockRequestInterface is a mock of RequestInterface interface. +type MockRequestInterface struct { + ctrl *gomock.Controller + recorder *MockRequestInterfaceMockRecorder + isgomock struct{} +} + +// MockRequestInterfaceMockRecorder is the mock recorder for MockRequestInterface. +type MockRequestInterfaceMockRecorder struct { + mock *MockRequestInterface +} + +// NewMockRequestInterface creates a new mock instance. +func NewMockRequestInterface(ctrl *gomock.Controller) *MockRequestInterface { + mock := &MockRequestInterface{ctrl: ctrl} + mock.recorder = &MockRequestInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRequestInterface) EXPECT() *MockRequestInterfaceMockRecorder { + return m.recorder +} + +// Do mocks base method. +func (m *MockRequestInterface) Do(ctx context.Context, req *request.RequestBuilder) (*http.Response, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Do", ctx, req) + ret0, _ := ret[0].(*http.Response) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Do indicates an expected call of Do. +func (mr *MockRequestInterfaceMockRecorder) Do(ctx, req any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Do", reflect.TypeOf((*MockRequestInterface)(nil).Do), ctx, req) +} + +// New mocks base method. +func (m *MockRequestInterface) New(method, url string) *request.RequestBuilder { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "New", method, url) + ret0, _ := ret[0].(*request.RequestBuilder) + return ret0 +} + +// New indicates an expected call of New. +func (mr *MockRequestInterfaceMockRecorder) New(method, url any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "New", reflect.TypeOf((*MockRequestInterface)(nil).New), method, url) +} diff --git a/request/auth.go b/request/auth.go new file mode 100644 index 0000000..0a1bcec --- /dev/null +++ b/request/auth.go @@ -0,0 +1,116 @@ +package request + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "sync" + + "github.com/calypr/data-client/conf" + sycommon "github.com/calypr/syfon/client/common" +) + +func (t *AuthTransport) NewAccessToken(ctx context.Context) error { + if t.Cred.APIKey == "" { + return errors.New("APIKey is required to refresh access token") + } + + refreshClient := &http.Client{Transport: t.Base} + + payload := map[string]string{"api_key": t.Cred.APIKey} + reader, err := sycommon.ToJSONReader(payload) + if err != nil { + return err + } + + refreshUrl := t.Cred.APIEndpoint + sycommon.DataAccessTokenEndpoint + req, err := http.NewRequestWithContext(ctx, http.MethodPost, refreshUrl, reader) + if err != nil { + return err + } + req.Header.Set(sycommon.HeaderContentType, sycommon.MIMEApplicationJSON) + + resp, err := refreshClient.Do(req) + if err != nil { + return fmt.Errorf("refresh request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return errors.New("failed to refresh token, status: " + strconv.Itoa(resp.StatusCode)) + } + + var result struct { + AccessToken string `json:"access_token"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + return err + } + + t.mu.Lock() + t.Cred.AccessToken = result.AccessToken + if t.Manager != nil { + t.Manager.Save(t.Cred) + } + t.mu.Unlock() + return nil +} + +type AuthTransport struct { + Manager conf.ManagerInterface + Base http.RoundTripper + Cred *conf.Credential + mu sync.RWMutex + refreshMu sync.Mutex +} + +func (t *AuthTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Header.Get("X-Skip-Auth") == "true" { + req.Header.Del("X-Skip-Auth") + return t.Base.RoundTrip(req) + } + + t.mu.RLock() + token := t.Cred.AccessToken + t.mu.RUnlock() + + // Just add the header and pass it down + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + return t.Base.RoundTrip(req) +} + +func extractBearerToken(header string) string { + if !strings.HasPrefix(strings.ToLower(header), "bearer ") { + return "" + } + return strings.TrimSpace(header[len("Bearer "):]) +} + +func (t *AuthTransport) refreshOnce(ctx context.Context, failedToken string) error { + t.refreshMu.Lock() + defer t.refreshMu.Unlock() + + t.mu.RLock() + currentToken := t.Cred.AccessToken + t.mu.RUnlock() + + // Another goroutine may already have refreshed the token after the failing + // request was sent. If the credential token has changed, use that one and + // avoid a redundant refresh call. + if failedToken != "" && currentToken != "" && currentToken != failedToken { + return nil + } + + if currentToken != "" && failedToken == "" { + // Without the failed token, we cannot distinguish a stale token from a + // freshly refreshed one, so fall through and refresh explicitly. + } + + return t.NewAccessToken(ctx) +} diff --git a/request/auth_test.go b/request/auth_test.go new file mode 100644 index 0000000..08a5a74 --- /dev/null +++ b/request/auth_test.go @@ -0,0 +1,98 @@ +package request + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + + "github.com/calypr/data-client/conf" + "github.com/calypr/data-client/logs" +) + +type trackingConfigManager struct { + mockConfigManager + saveCalls int + lastSaved *conf.Credential +} + +func (m *trackingConfigManager) Save(cred *conf.Credential) error { + m.saveCalls++ + if cred != nil { + copyCred := *cred + m.lastSaved = ©Cred + } + return nil +} + +func TestRequestRefreshesExpiredBearerTokenOnUnauthorized(t *testing.T) { + const ( + oldToken = "expired-token" + newToken = "fresh-token" + apiKey = "refresh-api-key" + ) + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/user/credentials/api/access_token": + body, err := io.ReadAll(r.Body) + if err != nil { + t.Fatalf("read refresh body: %v", err) + } + if !strings.Contains(string(body), apiKey) { + t.Fatalf("refresh request missing api key: %s", string(body)) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"access_token":"` + newToken + `"}`)) + case "/protected": + switch r.Header.Get("Authorization") { + case "Bearer " + oldToken: + http.Error(w, "expired", http.StatusUnauthorized) + case "Bearer " + newToken: + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + default: + t.Fatalf("unexpected authorization header: %q", r.Header.Get("Authorization")) + } + default: + t.Fatalf("unexpected path: %s", r.URL.Path) + } + })) + defer server.Close() + + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + Profile: "test", + APIKey: apiKey, + AccessToken: oldToken, + APIEndpoint: server.URL, + } + cfg := &trackingConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, cfg) + req := reqInterface.(*Request) + builder := req.New(http.MethodGet, server.URL+"/protected") + + resp, err := req.Do(context.Background(), builder) + if err != nil { + t.Fatalf("request failed: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if cred.AccessToken != newToken { + t.Fatalf("credential token = %q, want %q", cred.AccessToken, newToken) + } + if cfg.saveCalls != 1 { + t.Fatalf("save calls = %d, want 1", cfg.saveCalls) + } + if cfg.lastSaved == nil || cfg.lastSaved.AccessToken != newToken { + t.Fatalf("saved credential token = %#v, want %q", cfg.lastSaved, newToken) + } +} diff --git a/request/builder.go b/request/builder.go new file mode 100644 index 0000000..104ac47 --- /dev/null +++ b/request/builder.go @@ -0,0 +1,65 @@ +package request + +import ( + "io" + + sycommon "github.com/calypr/syfon/client/common" +) + +// New addition to your request package +type RequestBuilder struct { + //Req *Request // the underlying retry client holder + Method string + Url string + Body io.Reader // store as []byte for easy reuse + Headers map[string]string + Token string + PartSize int64 + SkipAuth bool +} + +func (r *Request) New(method, url string) *RequestBuilder { + return &RequestBuilder{ + //Req: r, + Method: method, + Url: url, + Headers: make(map[string]string), + } +} + +func (ar *RequestBuilder) WithToken(token string) *RequestBuilder { + ar.Token = token + return ar +} + +func (ar *RequestBuilder) WithJSONBody(v any) (*RequestBuilder, error) { + reader, err := sycommon.ToJSONReader(v) + if err != nil { + return nil, err + } + + ar.Body = reader + ar.Headers[sycommon.HeaderContentType] = sycommon.MIMEApplicationJSON + return ar, nil + +} + +func (ar *RequestBuilder) WithBody(body io.Reader) *RequestBuilder { + ar.Body = body + return ar +} + +func (ar *RequestBuilder) WithHeader(key, value string) *RequestBuilder { + ar.Headers[key] = value + return ar +} + +func (ar *RequestBuilder) WithSkipAuth(skip bool) *RequestBuilder { + ar.SkipAuth = skip + return ar +} + +func (ar *RequestBuilder) WithPartSize(size int64) *RequestBuilder { + ar.PartSize = size + return ar +} diff --git a/request/request.go b/request/request.go new file mode 100644 index 0000000..d496d0f --- /dev/null +++ b/request/request.go @@ -0,0 +1,119 @@ +package request + +//go:generate mockgen -destination=../mocks/mock_request.go -package=mocks github.com/calypr/syfon/client/request RequestInterface + +import ( + "context" + "errors" + "net" + "net/http" + "strings" + "time" + + "github.com/calypr/data-client/conf" + "github.com/calypr/data-client/logs" + "github.com/hashicorp/go-retryablehttp" +) + +type Request struct { + Logs *logs.Gen3Logger + RetryClient *retryablehttp.Client +} + +type RequestInterface interface { + New(method, url string) *RequestBuilder + Do(ctx context.Context, req *RequestBuilder) (*http.Response, error) +} + +func NewRequestInterface( + logger *logs.Gen3Logger, + cred *conf.Credential, + conf conf.ManagerInterface, +) RequestInterface { + retryClient := retryablehttp.NewClient() + retryClient.RetryMax = 5 + retryClient.Logger = logger + retryClient.RetryWaitMin = 5 * time.Second + retryClient.RetryWaitMax = 15 * time.Second + baseTransport := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + TLSHandshakeTimeout: 30 * time.Second, + ResponseHeaderTimeout: 60 * time.Second, + } + + authTransport := &AuthTransport{ + Base: baseTransport, + Cred: cred, + Manager: conf, + } + retryClient.HTTPClient = &http.Client{ + Timeout: 0, + Transport: authTransport, + } + + retryClient.CheckRetry = func(ctx context.Context, resp *http.Response, err error) (bool, error) { + shouldRetry, retryErr := + retryablehttp.DefaultRetryPolicy(ctx, resp, err) + + if resp != nil && + (resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusBadGateway) { + failedToken := "" + if resp.Request != nil { + failedToken = extractBearerToken(resp.Request.Header.Get("Authorization")) + } + err := authTransport.refreshOnce(ctx, strings.TrimSpace(failedToken)) + if err != nil { + return false, err + } + return true, nil + } + return shouldRetry, retryErr + } + + return &Request{ + RetryClient: retryClient, + Logs: logger, + } +} + +func (r *Request) Do(ctx context.Context, rb *RequestBuilder) (*http.Response, error) { + // Prepare body reader + + httpReq, err := http.NewRequestWithContext(ctx, rb.Method, rb.Url, rb.Body) + if err != nil { + return nil, errors.New("failed to create HTTP request: " + err.Error()) + } + + for key, value := range rb.Headers { + httpReq.Header.Add(key, value) + } + + if rb.SkipAuth { + httpReq.Header.Set("X-Skip-Auth", "true") + } + + if rb.Token != "" { + httpReq.Header.Set("Authorization", "Bearer "+rb.Token) + } + + if rb.PartSize != 0 { + httpReq.ContentLength = rb.PartSize + } + // Convert to retryablehttp.Request + retryReq, err := retryablehttp.FromRequest(httpReq) + if err != nil { + return nil, err + } + + resp, err := r.RetryClient.Do(retryReq) + if err != nil { + return resp, errors.New("request failed after retries: " + err.Error()) + } + + return resp, nil +} diff --git a/request/request_test.go b/request/request_test.go new file mode 100644 index 0000000..019b097 --- /dev/null +++ b/request/request_test.go @@ -0,0 +1,263 @@ +package request + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" + "time" + + "github.com/calypr/data-client/conf" + "github.com/calypr/data-client/logs" +) + +func TestNewRequestInterface(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: "https://example.com", + } + + // Create a mock config manager + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + + if reqInterface == nil { + t.Fatal("Expected non-nil request interface") + } + + req, ok := reqInterface.(*Request) + if !ok { + t.Fatal("Expected request interface to be of type *Request") + } + + if req.RetryClient == nil { + t.Error("Expected non-nil retry client") + } + + if req.Logs == nil { + t.Error("Expected non-nil logger") + } +} + +func TestRequestBuilder_New(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: "https://example.com", + } + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + req := reqInterface.(*Request) + + builder := req.New("GET", "https://example.com/api/test") + + if builder == nil { + t.Fatal("Expected non-nil request builder") + } + + if builder.Method != "GET" { + t.Errorf("Expected method 'GET', got '%s'", builder.Method) + } + + if builder.Url != "https://example.com/api/test" { + t.Errorf("Expected URL 'https://example.com/api/test', got '%s'", builder.Url) + } +} + +func TestRequestBuilder_WithHeaders(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: "https://example.com", + } + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + req := reqInterface.(*Request) + + builder := req.New("GET", "https://example.com/api/test") + builder = builder.WithHeader("Content-Type", "application/json") + builder = builder.WithHeader("X-Custom-Header", "test-value") + + if len(builder.Headers) != 2 { + t.Errorf("Expected 2 headers, got %d", len(builder.Headers)) + } + + if builder.Headers["Content-Type"] != "application/json" { + t.Error("Expected Content-Type header to be set") + } + + if builder.Headers["X-Custom-Header"] != "test-value" { + t.Error("Expected X-Custom-Header to be set") + } +} + +func TestRequestBuilder_WithToken(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: "https://example.com", + } + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + req := reqInterface.(*Request) + + token := "test-bearer-token-12345" + builder := req.New("GET", "https://example.com/api/test") + builder = builder.WithToken(token) + + if builder.Token != token { + t.Errorf("Expected token '%s', got '%s'", token, builder.Token) + } +} + +func TestRequestBuilder_WithBody(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: "https://example.com", + } + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + req := reqInterface.(*Request) + + body := strings.NewReader("test body content") + builder := req.New("POST", "https://example.com/api/test") + builder = builder.WithBody(body) + + if builder.Body == nil { + t.Error("Expected non-nil body") + } +} + +func TestRequest_Do_Success(t *testing.T) { + // Create a test server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Verify the request + if r.Method != "GET" { + t.Errorf("Expected GET method, got %s", r.Method) + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "success"}`)) + })) + defer server.Close() + + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: server.URL, + } + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + req := reqInterface.(*Request) + + builder := req.New("GET", server.URL+"/api/test") + builder = builder.WithToken("test-token") + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + resp, err := req.Do(ctx, builder) + + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if resp == nil { + t.Fatal("Expected non-nil response") + } + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if !strings.Contains(string(body), "success") { + t.Error("Expected response body to contain 'success'") + } +} + +func TestRequest_Do_WithCustomHeaders(t *testing.T) { + // Create a test server that checks for custom headers + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + customHeader := r.Header.Get("X-Custom-Header") + if customHeader != "test-value" { + t.Errorf("Expected X-Custom-Header 'test-value', got '%s'", customHeader) + } + + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + logger := slog.New(slog.NewTextHandler(os.Stdout, nil)) + cred := &conf.Credential{ + KeyID: "test-key", + APIKey: "test-secret", + APIEndpoint: server.URL, + } + mockConf := &mockConfigManager{} + + reqInterface := NewRequestInterface(logs.NewGen3Logger(logger, "", ""), cred, mockConf) + req := reqInterface.(*Request) + + builder := req.New("GET", server.URL+"/api/test") + builder = builder.WithHeader("X-Custom-Header", "test-value") + + ctx := context.Background() + resp, err := req.Do(ctx, builder) + + if err != nil { + t.Fatalf("Expected no error, got: %v", err) + } + + if resp.StatusCode != http.StatusOK { + t.Errorf("Expected status 200, got %d", resp.StatusCode) + } + + resp.Body.Close() +} + +// Mock config manager for testing +type mockConfigManager struct{} + +func (m *mockConfigManager) Import(filePath, fenceToken string) (*conf.Credential, error) { + return &conf.Credential{}, nil +} + +func (m *mockConfigManager) Load(profile string) (*conf.Credential, error) { + return &conf.Credential{}, nil +} + +func (m *mockConfigManager) Save(cred *conf.Credential) error { + return nil +} + +func (m *mockConfigManager) EnsureExists() error { + return nil +} + +func (m *mockConfigManager) IsCredentialValid(cred *conf.Credential) (bool, error) { + return true, nil +} + +func (m *mockConfigManager) IsTokenValid(token string) (bool, error) { + return true, nil +} diff --git a/requestor/client.go b/requestor/client.go new file mode 100644 index 0000000..223861d --- /dev/null +++ b/requestor/client.go @@ -0,0 +1,437 @@ +package requestor + +import ( + "context" + "embed" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "sort" + "strings" + + "github.com/calypr/data-client/conf" + "github.com/calypr/data-client/request" + "gopkg.in/yaml.v3" +) + +//go:embed policies/*.yaml +var policyFS embed.FS + +type RequestorClient struct { + request.RequestInterface + Endpoint string +} + +func NewRequestorClient(req request.RequestInterface, creds *conf.Credential) *RequestorClient { + return &RequestorClient{ + RequestInterface: req, + Endpoint: creds.APIEndpoint, + } +} + +// Ensure interface compliance +var _ RequestorInterface = &RequestorClient{} + +type RequestorInterface interface { + ListRequests(ctx context.Context, mine bool, active bool, username string) ([]Request, error) + CreateRequest(ctx context.Context, req CreateRequestRequest, revoke bool) (*Request, error) + UpdateRequest(ctx context.Context, requestID string, status string) (*Request, error) + AddUser(ctx context.Context, projectID string, username string, write bool, guppy bool) ([]Request, error) + AddUserToResources(ctx context.Context, resources []ProjectResource, username string, write bool, guppy bool) ([]Request, error) + RemoveUser(ctx context.Context, projectID string, username string) ([]Request, error) +} + +func (c *RequestorClient) ListRequests(ctx context.Context, mine bool, active bool, username string) ([]Request, error) { + url := c.Endpoint + "/requestor/request" + if mine { + url += "/user" + } + + params := []string{} + if active { + params = append(params, "active") + } + if username != "" && !mine { + params = append(params, fmt.Sprintf("username=%s", username)) + } + + if len(params) > 0 { + url += "?" + strings.Join(params, "&") + } + + rb := c.New(http.MethodGet, url) + resp, err := c.Do(ctx, rb) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to list requests: status %d", resp.StatusCode) + } + + var requests []Request + if err := json.NewDecoder(resp.Body).Decode(&requests); err != nil { + return nil, err + } + return requests, nil +} + +func (c *RequestorClient) CreateRequest(ctx context.Context, reqPayload CreateRequestRequest, revoke bool) (*Request, error) { + url := c.Endpoint + "/requestor/request" + if revoke { + url += "?revoke" + } + + rb := c.New(http.MethodPost, url) + rb, err := rb.WithJSONBody(reqPayload) + if err != nil { + return nil, err + } + + resp, err := c.Do(ctx, rb) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("failed to create request: status %d, body: %s", resp.StatusCode, string(bodyBytes)) + } + + var createdRequest Request + if err := json.NewDecoder(resp.Body).Decode(&createdRequest); err != nil { + return nil, err + } + return &createdRequest, nil +} + +func (c *RequestorClient) UpdateRequest(ctx context.Context, requestID string, status string) (*Request, error) { + url := fmt.Sprintf("%s/requestor/request/%s", c.Endpoint, requestID) + payload := UpdateRequestRequest{Status: status} + + rb := c.New(http.MethodPut, url) + rb, err := rb.WithJSONBody(payload) + if err != nil { + return nil, err + } + + resp, err := c.Do(ctx, rb) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + bodyBytes, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("failed to update request: status %d, body: %s", resp.StatusCode, string(bodyBytes)) + } + + var updatedRequest Request + if err := json.NewDecoder(resp.Body).Decode(&updatedRequest); err != nil { + return nil, err + } + return &updatedRequest, nil +} + +func loadPolicies(filename string) ([]CreateRequestRequest, error) { + content, err := policyFS.ReadFile("policies/" + filename) + if err != nil { + return nil, err + } + + var config PolicyConfig + if err := yaml.Unmarshal(content, &config); err != nil { + return nil, err + } + return config.Policies, nil +} + +func formatPolicy(policy CreateRequestRequest, projectID string, username string) CreateRequestRequest { + p := policy + if username != "" { + p.Username = username + } + + if projectID != "" { + parts := strings.Split(projectID, "-") + if len(parts) >= 2 { + program := parts[0] + project := parts[1] + + newPaths := make([]string, len(p.ResourcePaths)) + for i, path := range p.ResourcePaths { + r := strings.ReplaceAll(path, "PROGRAM", program) + r = strings.ReplaceAll(r, "PROJECT", project) + newPaths[i] = r + } + p.ResourcePaths = newPaths + } + p.ResourceDisplayName = projectID + } + return p +} + +func formatPolicyForResource(policy CreateRequestRequest, resource ProjectResource, username string) CreateRequestRequest { + p := policy + if username != "" { + p.Username = username + } + + if resource.ResourcePath != "" { + newPaths := make([]string, len(p.ResourcePaths)) + for i, path := range p.ResourcePaths { + if strings.Contains(path, "PROGRAM") || strings.Contains(path, "PROJECT") { + newPaths[i] = resource.ResourcePath + continue + } + newPaths[i] = path + } + p.ResourcePaths = newPaths + p.ResourceDisplayName = resource.DisplayName() + } + return p +} + +func ParseProjectResources(raw string) ([]ProjectResource, error) { + raw = extractProjectResourceList(raw) + parts := strings.FieldsFunc(raw, func(r rune) bool { + return r == ',' || r == '\n' || r == '\r' + }) + + seen := map[string]struct{}{} + resources := make([]ProjectResource, 0, len(parts)) + for _, part := range parts { + trimmed := strings.TrimSpace(strings.Trim(part, `"'`)) + if trimmed == "" || strings.HasPrefix(trimmed, "(") { + continue + } + resource, err := ParseProjectResource(part) + if err != nil { + return nil, err + } + if _, ok := seen[resource.ResourcePath]; ok { + continue + } + seen[resource.ResourcePath] = struct{}{} + resources = append(resources, resource) + } + if len(resources) == 0 { + return nil, fmt.Errorf("no project resources found") + } + return resources, nil +} + +func extractProjectResourceList(raw string) string { + markers := []string{ + "copy/paste scope list:", + "missing organization/project scopes:", + "denied organization/project scopes:", + "denied resource paths:", + } + lower := strings.ToLower(raw) + start := -1 + markerLen := 0 + for _, marker := range markers { + if idx := strings.LastIndex(lower, marker); idx >= 0 && idx >= start { + start = idx + markerLen = len(marker) + } + } + if start >= 0 { + raw = raw[start+markerLen:] + } + raw = strings.TrimSpace(raw) + if idx := strings.Index(raw, "\r\n\r\n"); idx >= 0 { + raw = raw[:idx] + } + if idx := strings.Index(raw, "\n\n"); idx >= 0 { + raw = raw[:idx] + } + return raw +} + +func ParseProjectResource(raw string) (ProjectResource, error) { + candidate := strings.TrimSpace(raw) + candidate = strings.Trim(candidate, `"'`) + if idx := strings.Index(candidate, "/programs/"); idx >= 0 { + candidate = candidate[idx:] + } else if idx := strings.LastIndex(candidate, ":"); idx >= 0 { + candidate = candidate[idx+1:] + } + candidate = strings.TrimSpace(candidate) + candidate = strings.Trim(candidate, `"'`) + candidate = strings.TrimSuffix(candidate, ".") + if candidate == "" || strings.HasPrefix(candidate, "(") { + return ProjectResource{}, fmt.Errorf("empty resource path") + } + + if parsed, err := url.Parse(candidate); err == nil && parsed.Path != "" { + candidate = parsed.Path + } + + trimmed := strings.Trim(candidate, "/") + if !strings.HasPrefix(trimmed, "programs/") && strings.Count(trimmed, "/") == 1 { + pair := strings.SplitN(trimmed, "/", 2) + return projectResource(pair[0], pair[1]) + } + + parts := strings.Split(trimmed, "/") + if len(parts) < 4 || parts[0] != "programs" || parts[2] != "projects" { + return ProjectResource{}, fmt.Errorf("invalid project resource %q; expected /programs//projects/", raw) + } + return projectResource(parts[1], parts[3]) +} + +func projectResource(program, project string) (ProjectResource, error) { + program = strings.TrimSpace(program) + project = strings.TrimSpace(project) + if program == "" || project == "" { + return ProjectResource{}, fmt.Errorf("project resource requires both organization and project") + } + if strings.ContainsAny(program, " \t") || strings.ContainsAny(project, " \t") { + return ProjectResource{}, fmt.Errorf("project resource contains whitespace: %s/%s", program, project) + } + return ProjectResource{ + Program: program, + Project: project, + ResourcePath: "/programs/" + program + "/projects/" + project, + }, nil +} + +func (c *RequestorClient) getPolicyKey(p CreateRequestRequest) string { + roles := make([]string, len(p.RoleIDs)) + copy(roles, p.RoleIDs) + sort.Strings(roles) + + paths := make([]string, len(p.ResourcePaths)) + copy(paths, p.ResourcePaths) + sort.Strings(paths) + + return fmt.Sprintf("%s:%s:%s", p.PolicyID, strings.Join(roles, ","), strings.Join(paths, ",")) +} + +func (c *RequestorClient) AddUser(ctx context.Context, projectID string, username string, write bool, guppy bool) ([]Request, error) { + uniquePolicies := make(map[string]CreateRequestRequest) + + addFrom := func(fileName string) error { + pols, err := loadPolicies(fileName) + if err != nil { + return err + } + for _, p := range pols { + formatted := formatPolicy(p, projectID, username) + key := c.getPolicyKey(formatted) + uniquePolicies[key] = formatted + } + return nil + } + + // Always add read + if err := addFrom("add-user-read.yaml"); err != nil { + return nil, fmt.Errorf("failed to load read policy: %w", err) + } + + if write { + if err := addFrom("add-user-write.yaml"); err != nil { + return nil, fmt.Errorf("failed to load write policy: %w", err) + } + } + if guppy { + if err := addFrom("add-user-guppy-admin.yaml"); err != nil { + return nil, fmt.Errorf("failed to load guppy policy: %w", err) + } + } + + var createdRequests []Request + for _, formatted := range uniquePolicies { + req, err := c.CreateRequest(ctx, formatted, false) + if err != nil { + return createdRequests, fmt.Errorf("failed to create request for policy %v: %w", formatted, err) + } + createdRequests = append(createdRequests, *req) + } + return createdRequests, nil +} + +func (c *RequestorClient) AddUserToResources(ctx context.Context, resources []ProjectResource, username string, write bool, guppy bool) ([]Request, error) { + uniquePolicies := make(map[string]CreateRequestRequest) + + addFrom := func(fileName string, resource ProjectResource) error { + pols, err := loadPolicies(fileName) + if err != nil { + return err + } + for _, p := range pols { + formatted := formatPolicyForResource(p, resource, username) + key := c.getPolicyKey(formatted) + uniquePolicies[key] = formatted + } + return nil + } + + for _, resource := range resources { + if err := addFrom("add-user-read.yaml", resource); err != nil { + return nil, fmt.Errorf("failed to load read policy for %s: %w", resource.DisplayName(), err) + } + if write { + if err := addFrom("add-user-write.yaml", resource); err != nil { + return nil, fmt.Errorf("failed to load write policy for %s: %w", resource.DisplayName(), err) + } + } + if guppy { + if err := addFrom("add-user-guppy-admin.yaml", resource); err != nil { + return nil, fmt.Errorf("failed to load guppy policy for %s: %w", resource.DisplayName(), err) + } + } + } + + var createdRequests []Request + for _, formatted := range uniquePolicies { + req, err := c.CreateRequest(ctx, formatted, false) + if err != nil { + return createdRequests, fmt.Errorf("failed to create request for policy %v: %w", formatted, err) + } + createdRequests = append(createdRequests, *req) + } + return createdRequests, nil +} + +func (c *RequestorClient) RemoveUser(ctx context.Context, projectID string, username string) ([]Request, error) { + uniquePolicies := make(map[string]CreateRequestRequest) + + addFrom := func(fileName string) error { + pols, err := loadPolicies(fileName) + if err != nil { + return err + } + for _, p := range pols { + formatted := formatPolicy(p, projectID, username) + key := c.getPolicyKey(formatted) + uniquePolicies[key] = formatted + } + return nil + } + + // Revoke read and write + if err := addFrom("add-user-read.yaml"); err != nil { + return nil, fmt.Errorf("failed to load read policy: %w", err) + } + + if err := addFrom("add-user-write.yaml"); err != nil { + return nil, fmt.Errorf("failed to load write policy: %w", err) + } + + var createdRequests []Request + for _, formatted := range uniquePolicies { + req, err := c.CreateRequest(ctx, formatted, true) // revoke=true + if err != nil { + return createdRequests, fmt.Errorf("failed to revoke request: %w", err) + } + createdRequests = append(createdRequests, *req) + } + return createdRequests, nil +} diff --git a/requestor/client_test.go b/requestor/client_test.go new file mode 100644 index 0000000..3d04e64 --- /dev/null +++ b/requestor/client_test.go @@ -0,0 +1,346 @@ +package requestor + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/calypr/data-client/request" +) + +type fakeRequest struct { + doFn func(rb *request.RequestBuilder) (*http.Response, error) +} + +func (f *fakeRequest) New(method, u string) *request.RequestBuilder { + return &request.RequestBuilder{Method: method, Url: u, Headers: map[string]string{}} +} + +func (f *fakeRequest) Do(ctx context.Context, rb *request.RequestBuilder) (*http.Response, error) { + return f.doFn(rb) +} + +func jsonResponse(status int, v any) *http.Response { + var body io.ReadCloser = http.NoBody + if v != nil { + buf, _ := json.Marshal(v) + body = io.NopCloser(bytes.NewReader(buf)) + } + return &http.Response{StatusCode: status, Body: body} +} + +func TestGetPolicyKey(t *testing.T) { + c := &RequestorClient{} + + p1 := CreateRequestRequest{ + PolicyID: "p1", + RoleIDs: []string{"reader"}, + ResourcePaths: []string{"/path1"}, + } + p2 := CreateRequestRequest{ + PolicyID: "p1", + RoleIDs: []string{"reader"}, + ResourcePaths: []string{"/path1"}, + } + p3 := CreateRequestRequest{ + PolicyID: "p2", + RoleIDs: []string{"reader"}, + ResourcePaths: []string{"/path1"}, + } + + if c.getPolicyKey(p1) != c.getPolicyKey(p2) { + t.Errorf("Expected p1 and p2 to have same key") + } + if c.getPolicyKey(p1) == c.getPolicyKey(p3) { + t.Errorf("Expected p1 and p3 to have different keys (PolicyID differs)") + } + + p4 := CreateRequestRequest{ + RoleIDs: []string{"a", "b"}, + ResourcePaths: []string{"/p1", "/p2"}, + } + p5 := CreateRequestRequest{ + RoleIDs: []string{"b", "a"}, + ResourcePaths: []string{"/p2", "/p1"}, + } + if c.getPolicyKey(p4) != c.getPolicyKey(p5) { + t.Errorf("Expected p4 and p5 to have same key (sorting check)") + } + + p6 := CreateRequestRequest{ + RoleIDs: []string{"reader"}, + ResourcePaths: []string{"/path1"}, + } + p7 := CreateRequestRequest{ + RoleIDs: []string{"reader"}, + ResourcePaths: []string{"/path1"}, + } + if c.getPolicyKey(p6) != c.getPolicyKey(p7) { + t.Errorf("Expected p6 and p7 (empty PolicyID) to have same key") + } +} + +func TestLoadPoliciesAndFormatPolicy(t *testing.T) { + policies, err := loadPolicies("add-user-read.yaml") + if err != nil { + t.Fatalf("loadPolicies returned error: %v", err) + } + if len(policies) == 0 { + t.Fatal("expected add-user-read.yaml to contain at least one policy") + } + + if _, err := loadPolicies("missing.yaml"); err == nil { + t.Fatal("expected missing policy file to return an error") + } + + formatted := formatPolicy( + CreateRequestRequest{ + PolicyID: "policy-1", + ResourcePaths: []string{"/PROGRAM/projects/PROJECT/data"}, + ResourceDisplayName: "original", + }, + "demo-program-demo-project", + "alice", + ) + if formatted.Username != "alice" { + t.Fatalf("expected username to be replaced, got %q", formatted.Username) + } + if formatted.ResourceDisplayName != "demo-program-demo-project" { + t.Fatalf("expected display name to be replaced, got %q", formatted.ResourceDisplayName) + } + if got := formatted.ResourcePaths[0]; got != "/demo/projects/program/data" { + t.Fatalf("unexpected formatted path: %q", got) + } +} + +func TestParseProjectResources(t *testing.T) { + resources, err := ParseProjectResources(`denied resource paths: /programs/HTAN_INT/projects/BForePC, /programs/cbds/projects/git_drs_test, aced/evotypes`) + if err != nil { + t.Fatalf("ParseProjectResources returned error: %v", err) + } + if len(resources) != 3 { + t.Fatalf("expected 3 resources, got %+v", resources) + } + want := []string{ + "/programs/HTAN_INT/projects/BForePC", + "/programs/cbds/projects/git_drs_test", + "/programs/aced/projects/evotypes", + } + for i, expected := range want { + if resources[i].ResourcePath != expected { + t.Fatalf("resource %d = %q, want %q", i, resources[i].ResourcePath, expected) + } + } +} + +func TestParseProjectResourcesFromScopeMessage(t *testing.T) { + resources, err := ParseProjectResources(`denied organization/project scopes: HTAN_INT/BForePC, cbds/git_drs_test`) + if err != nil { + t.Fatalf("ParseProjectResources returned error: %v", err) + } + if len(resources) != 2 { + t.Fatalf("expected 2 resources, got %+v", resources) + } + if got := resources[0].ResourcePath; got != "/programs/HTAN_INT/projects/BForePC" { + t.Fatalf("unexpected first resource: %q", got) + } +} + +func TestParseProjectResourcesFromPreflightCopyPasteBlock(t *testing.T) { + raw := `migration import preflight failed: missing create access for 1320/82628 records across 323 scopes +first denied record: 04ae835f-9a18-5867-8a63-fa527dedf425 + +Copy/paste scope list: +Ellrott_Lab/embedding_rotation, HTAN_INT/TestUpload1, cbds/017549802cd04d108524ae3f196ffacd + +2026/05/01 12:01:42 ERROR command execution failed err="target authorization preflight failed: missing create access"` + + resources, err := ParseProjectResources(raw) + if err != nil { + t.Fatalf("ParseProjectResources returned error: %v", err) + } + if len(resources) != 3 { + t.Fatalf("expected 3 resources, got %+v", resources) + } + want := []string{ + "/programs/Ellrott_Lab/projects/embedding_rotation", + "/programs/HTAN_INT/projects/TestUpload1", + "/programs/cbds/projects/017549802cd04d108524ae3f196ffacd", + } + for i, expected := range want { + if resources[i].ResourcePath != expected { + t.Fatalf("resource %d = %q, want %q", i, resources[i].ResourcePath, expected) + } + } +} + +func TestRequestorClientListCreateAndUpdate(t *testing.T) { + client := &RequestorClient{ + RequestInterface: &fakeRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + u, err := url.Parse(rb.Url) + if err != nil { + return nil, err + } + + switch { + case rb.Method == http.MethodGet && u.Path == "/requestor/request": + if !u.Query().Has("active") { + t.Fatalf("expected active query parameter") + } + if got := u.Query().Get("username"); got != "bob" { + t.Fatalf("expected username query bob, got %q", got) + } + return jsonResponse(http.StatusOK, []Request{{RequestID: "req-list", Status: "active"}}), nil + case rb.Method == http.MethodPost && u.Path == "/requestor/request": + body, _ := io.ReadAll(rb.Body) + var payload CreateRequestRequest + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("decode create payload: %v", err) + } + if payload.PolicyID == "" { + t.Fatal("expected create payload policy id") + } + return jsonResponse(http.StatusOK, Request{RequestID: "req-create", Status: "open"}), nil + case rb.Method == http.MethodPut && strings.HasPrefix(u.Path, "/requestor/request/"): + body, _ := io.ReadAll(rb.Body) + var payload UpdateRequestRequest + if err := json.Unmarshal(body, &payload); err != nil { + t.Fatalf("decode update payload: %v", err) + } + if payload.Status != "approved" { + t.Fatalf("expected approved status, got %q", payload.Status) + } + return jsonResponse(http.StatusOK, Request{RequestID: "req-update", Status: payload.Status}), nil + default: + return jsonResponse(http.StatusNotFound, nil), nil + } + }, + }, + Endpoint: "https://example.org", + } + + list, err := client.ListRequests(context.Background(), false, true, "bob") + if err != nil { + t.Fatalf("ListRequests failed: %v", err) + } + if len(list) != 1 || list[0].RequestID != "req-list" { + t.Fatalf("unexpected list response: %+v", list) + } + + created, err := client.CreateRequest(context.Background(), CreateRequestRequest{ + PolicyID: "policy-1", + ResourcePaths: []string{"/path"}, + RoleIDs: []string{"reader"}, + }, false) + if err != nil { + t.Fatalf("CreateRequest failed: %v", err) + } + if created.RequestID != "req-create" { + t.Fatalf("unexpected create response: %+v", created) + } + + updated, err := client.UpdateRequest(context.Background(), "req-1", "approved") + if err != nil { + t.Fatalf("UpdateRequest failed: %v", err) + } + if updated.Status != "approved" { + t.Fatalf("unexpected update response: %+v", updated) + } +} + +func TestRequestorClientAddAndRemoveUser(t *testing.T) { + var revokeCount int + + client := &RequestorClient{ + RequestInterface: &fakeRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + u, err := url.Parse(rb.Url) + if err != nil { + return nil, err + } + if rb.Method != http.MethodPost || u.Path != "/requestor/request" { + return jsonResponse(http.StatusNotFound, nil), nil + } + if u.Query().Has("revoke") { + revokeCount++ + } + return jsonResponse(http.StatusOK, Request{RequestID: "req-ok", Status: "open"}), nil + }, + }, + Endpoint: "https://example.org", + } + + added, err := client.AddUser(context.Background(), "demo-program-demo-project", "bob", true, true) + if err != nil { + t.Fatalf("AddUser failed: %v", err) + } + if len(added) == 0 { + t.Fatal("expected AddUser to create at least one request") + } + + revoked, err := client.RemoveUser(context.Background(), "demo-program-demo-project", "bob") + if err != nil { + t.Fatalf("RemoveUser failed: %v", err) + } + if len(revoked) == 0 { + t.Fatal("expected RemoveUser to create at least one revocation request") + } + if revokeCount == 0 { + t.Fatal("expected revoke query parameter to be seen for revocation requests") + } +} + +func TestAddUserToResourcesCreatesRequestsPerResource(t *testing.T) { + var payloads []CreateRequestRequest + client := &RequestorClient{ + RequestInterface: &fakeRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + if rb.Method != http.MethodPost { + return jsonResponse(http.StatusMethodNotAllowed, nil), nil + } + var payload CreateRequestRequest + if err := json.NewDecoder(rb.Body).Decode(&payload); err != nil { + return nil, err + } + payloads = append(payloads, payload) + return jsonResponse(http.StatusOK, Request{RequestID: "req-ok", Status: "open"}), nil + }, + }, + Endpoint: "https://example.org", + } + + resources, err := ParseProjectResources("/programs/cbds/projects/git_drs_test, /programs/aced/projects/evotypes") + if err != nil { + t.Fatalf("ParseProjectResources returned error: %v", err) + } + created, err := client.AddUserToResources(context.Background(), resources, "bob@example.org", true, true) + if err != nil { + t.Fatalf("AddUserToResources failed: %v", err) + } + if len(created) != 5 { + t.Fatalf("expected 5 deduplicated requests, got %d", len(created)) + } + + seen := map[string]bool{} + for _, payload := range payloads { + if payload.Username != "bob@example.org" { + t.Fatalf("unexpected username in payload: %+v", payload) + } + seen[strings.Join(payload.RoleIDs, ",")+"|"+strings.Join(payload.ResourcePaths, ",")] = true + } + if !seen["reader|/programs/cbds/projects/git_drs_test"] { + t.Fatalf("missing reader request for cbds/git_drs_test: %+v", payloads) + } + if !seen["writer|/programs/aced/projects/evotypes"] { + t.Fatalf("missing writer request for aced/evotypes: %+v", payloads) + } + if !seen["guppy_admin_user|/guppy_admin"] { + t.Fatalf("missing deduplicated guppy admin request: %+v", payloads) + } +} diff --git a/requestor/policies/add-user-guppy-admin.yaml b/requestor/policies/add-user-guppy-admin.yaml new file mode 100644 index 0000000..fb544df --- /dev/null +++ b/requestor/policies/add-user-guppy-admin.yaml @@ -0,0 +1,9 @@ +policies: +- role_ids: + - writer + resource_paths: + - /programs/PROGRAM/projects/PROJECT +- role_ids: + - guppy_admin_user + resource_paths: + - /guppy_admin diff --git a/requestor/policies/add-user-read.yaml b/requestor/policies/add-user-read.yaml new file mode 100644 index 0000000..e7acb34 --- /dev/null +++ b/requestor/policies/add-user-read.yaml @@ -0,0 +1,5 @@ +policies: +- role_ids: + - reader + resource_paths: + - /programs/PROGRAM/projects/PROJECT diff --git a/requestor/policies/add-user-write.yaml b/requestor/policies/add-user-write.yaml new file mode 100644 index 0000000..8fda383 --- /dev/null +++ b/requestor/policies/add-user-write.yaml @@ -0,0 +1,5 @@ +policies: +- role_ids: + - writer + resource_paths: + - /programs/PROGRAM/projects/PROJECT diff --git a/requestor/types.go b/requestor/types.go new file mode 100644 index 0000000..c3c39c0 --- /dev/null +++ b/requestor/types.go @@ -0,0 +1,47 @@ +package requestor + +// Request represents a requestor request object +type Request struct { + RequestID string `json:"request_id,omitempty" yaml:"request_id,omitempty"` + Username string `json:"username,omitempty" yaml:"username,omitempty"` + PolicyID string `json:"policy_id,omitempty" yaml:"policy_id,omitempty"` + ResourcePaths []string `json:"resource_paths,omitempty" yaml:"resource_paths,omitempty"` + RoleIDs []string `json:"role_ids,omitempty" yaml:"role_ids,omitempty"` + ResourceID string `json:"resource_id,omitempty" yaml:"resource_id,omitempty"` + ResourceDisplay string `json:"resource_display_name,omitempty" yaml:"resource_display_name,omitempty"` + Status string `json:"status,omitempty" yaml:"status,omitempty"` + CreatedTime string `json:"created_time,omitempty" yaml:"created_time,omitempty"` + UpdatedTime string `json:"updated_time,omitempty" yaml:"updated_time,omitempty"` + Revoke bool `json:"revoke,omitempty" yaml:"revoke,omitempty"` +} + +// CreateRequestRequest represents the payload to create a request +type CreateRequestRequest struct { + Username string `json:"username,omitempty" yaml:"username,omitempty"` + PolicyID string `json:"policy_id,omitempty" yaml:"policy_id,omitempty"` + ResourcePaths []string `json:"resource_paths,omitempty" yaml:"resource_paths,omitempty"` + RoleIDs []string `json:"role_ids,omitempty" yaml:"role_ids,omitempty"` + ResourceDisplayName string `json:"resource_display_name,omitempty" yaml:"resource_display_name,omitempty"` +} + +// UpdateRequestRequest represents the payload to update a request +type UpdateRequestRequest struct { + Status string `json:"status" yaml:"status"` +} + +type PolicyConfig struct { + Policies []CreateRequestRequest `yaml:"policies"` +} + +type ProjectResource struct { + Program string + Project string + ResourcePath string +} + +func (r ProjectResource) DisplayName() string { + if r.Program == "" || r.Project == "" { + return r.ResourcePath + } + return r.Program + "/" + r.Project +} diff --git a/sower/client.go b/sower/client.go new file mode 100644 index 0000000..770e891 --- /dev/null +++ b/sower/client.go @@ -0,0 +1,148 @@ +package sower + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + + "github.com/calypr/data-client/request" +) + +const ( + sowerDispatch = "/job/dispatch" + sowerStatus = "/job/status" + sowerList = "/job/list" + sowerJobOutput = "/job/output" +) + +type SowerInterface interface { + DispatchJob(ctx context.Context, name string, args *DispatchArgs) (*StatusResp, error) + Status(ctx context.Context, uid string) (*StatusResp, error) + List(ctx context.Context) ([]StatusResp, error) + Output(ctx context.Context, uid string) (*OutputResp, error) +} + +type SowerClient struct { + request.RequestInterface + Endpoint string +} + +func NewSowerClient(req request.RequestInterface, endpoint string) *SowerClient { + return &SowerClient{ + RequestInterface: req, + Endpoint: endpoint, + } +} + +func (sc *SowerClient) fullURL(path string) string { + u, _ := url.Parse(sc.Endpoint) + u.Path = path + return u.String() +} + +func (sc *SowerClient) DispatchJob(ctx context.Context, name string, args *DispatchArgs) (*StatusResp, error) { + body := JobArgs{ + Action: name, + Input: *args, + } + + rb := sc.New(http.MethodPost, sc.fullURL(sowerDispatch)) + rb, err := rb.WithJSONBody(body) + if err != nil { + return nil, err + } + + resp, err := sc.Do(ctx, rb) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("sower dispatch failed: %d %s", resp.StatusCode, string(b)) + } + + statusResp := &StatusResp{} + err = json.NewDecoder(resp.Body).Decode(statusResp) + if err != nil { + return nil, err + } + return statusResp, nil +} + +func (sc *SowerClient) Status(ctx context.Context, uid string) (*StatusResp, error) { + u, _ := url.Parse(sc.fullURL(sowerStatus)) + q := u.Query() + q.Add("UID", uid) + u.RawQuery = q.Encode() + + rb := sc.New(http.MethodGet, u.String()) + resp, err := sc.Do(ctx, rb) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("sower status failed: %d %s", resp.StatusCode, string(b)) + } + + statusResp := &StatusResp{} + err = json.NewDecoder(resp.Body).Decode(statusResp) + if err != nil { + return nil, err + } + return statusResp, nil +} + +func (sc *SowerClient) Output(ctx context.Context, uid string) (*OutputResp, error) { + u, _ := url.Parse(sc.fullURL(sowerJobOutput)) + q := u.Query() + q.Add("UID", uid) + u.RawQuery = q.Encode() + + rb := sc.New(http.MethodGet, u.String()) + resp, err := sc.Do(ctx, rb) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("sower output failed: %d %s", resp.StatusCode, string(b)) + } + + var outputResp OutputResp + err = json.NewDecoder(resp.Body).Decode(&outputResp) + if err != nil { + return nil, err + } + return &outputResp, nil +} + +func (sc *SowerClient) List(ctx context.Context) ([]StatusResp, error) { + rb := sc.New(http.MethodGet, sc.fullURL(sowerList)) + resp, err := sc.Do(ctx, rb) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("sower list failed: %d %s", resp.StatusCode, string(b)) + } + + var listResp []StatusResp + err = json.NewDecoder(resp.Body).Decode(&listResp) + if err != nil { + return nil, err + } + return listResp, nil +} diff --git a/sower/client_test.go b/sower/client_test.go new file mode 100644 index 0000000..77788c7 --- /dev/null +++ b/sower/client_test.go @@ -0,0 +1,122 @@ +package sower + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/url" + "testing" + + "github.com/calypr/data-client/request" + "github.com/hashicorp/go-retryablehttp" +) + +type fakeSowerRequest struct { + doFn func(rb *request.RequestBuilder) (*http.Response, error) +} + +func (f *fakeSowerRequest) New(method, u string) *request.RequestBuilder { + return &request.RequestBuilder{Method: method, Url: u, Headers: map[string]string{}} +} + +func (f *fakeSowerRequest) Do(ctx context.Context, rb *request.RequestBuilder) (*http.Response, error) { + return f.doFn(rb) +} + +func jsonResp(status int, v any) *http.Response { + var body io.ReadCloser = http.NoBody + if v != nil { + buf, _ := json.Marshal(v) + body = io.NopCloser(bytes.NewReader(buf)) + } + return &http.Response{StatusCode: status, Body: body} +} + +func TestSowerClientOperations(t *testing.T) { + client := &SowerClient{ + RequestInterface: &fakeSowerRequest{ + doFn: func(rb *request.RequestBuilder) (*http.Response, error) { + u, err := url.Parse(rb.Url) + if err != nil { + return nil, err + } + switch { + case rb.Method == http.MethodPost && u.Path == sowerDispatch: + var payload JobArgs + if err := json.NewDecoder(rb.Body).Decode(&payload); err != nil { + t.Fatalf("decode dispatch payload: %v", err) + } + if payload.Action != "dispatch" { + t.Fatalf("unexpected dispatch action: %q", payload.Action) + } + if payload.Input.Profile != "profile-a" { + t.Fatalf("unexpected profile: %q", payload.Input.Profile) + } + return jsonResp(http.StatusOK, StatusResp{Uid: "uid-1", Name: "job-1", Status: "running"}), nil + case rb.Method == http.MethodGet && u.Path == sowerStatus: + if got := u.Query().Get("UID"); got != "uid-1" { + t.Fatalf("unexpected UID query: %q", got) + } + return jsonResp(http.StatusOK, StatusResp{Uid: "uid-1", Name: "job-1", Status: "complete"}), nil + case rb.Method == http.MethodGet && u.Path == sowerJobOutput: + if got := u.Query().Get("UID"); got != "uid-1" { + t.Fatalf("unexpected UID query: %q", got) + } + return jsonResp(http.StatusOK, OutputResp{Output: "done"}), nil + case rb.Method == http.MethodGet && u.Path == sowerList: + return jsonResp(http.StatusOK, []StatusResp{{Uid: "uid-1", Name: "job-1", Status: "complete"}}), nil + default: + return jsonResp(http.StatusNotFound, nil), nil + } + }, + }, + Endpoint: "https://example.org", + } + + status, err := client.DispatchJob(context.Background(), "dispatch", &DispatchArgs{ + Profile: "profile-a", + APIEndpoint: "https://example.org", + }) + if err != nil { + t.Fatalf("DispatchJob failed: %v", err) + } + if status.Uid != "uid-1" { + t.Fatalf("unexpected dispatch result: %+v", status) + } + + gotStatus, err := client.Status(context.Background(), "uid-1") + if err != nil { + t.Fatalf("Status failed: %v", err) + } + if gotStatus.Status != "complete" { + t.Fatalf("unexpected status result: %+v", gotStatus) + } + + output, err := client.Output(context.Background(), "uid-1") + if err != nil { + t.Fatalf("Output failed: %v", err) + } + if output.Output != "done" { + t.Fatalf("unexpected output result: %+v", output) + } + + list, err := client.List(context.Background()) + if err != nil { + t.Fatalf("List failed: %v", err) + } + if len(list) != 1 || list[0].Uid != "uid-1" { + t.Fatalf("unexpected list result: %+v", list) + } +} + +func TestNewSowerClientBuildsEndpoint(t *testing.T) { + req := &request.Request{ + RetryClient: &retryablehttp.Client{HTTPClient: &http.Client{}}, + } + client := NewSowerClient(req, "https://example.org") + if client.fullURL(sowerDispatch) != "https://example.org/job/dispatch" { + t.Fatalf("unexpected full URL: %s", client.fullURL(sowerDispatch)) + } +} diff --git a/sower/types.go b/sower/types.go new file mode 100644 index 0000000..7b735a7 --- /dev/null +++ b/sower/types.go @@ -0,0 +1,33 @@ +package sower + +type StatusResp struct { + Uid string `json:"uid"` + Name string `json:"name"` + Status string `json:"status"` +} + +type OutputResp struct { + Output string `json:"output"` +} + +type File struct { + FileTitle string `json:"fileTitle,omitempty"` + FilePath string `json:"filePath"` +} + +type DispatchArgs struct { + Method string `json:"method"` + ProjectId string `json:"projectId"` + Profile string `json:"profile"` + BucketName string `json:"bucketName"` + APIEndpoint string `json:"APIEndpoint"` + GHCommitHash string `json:"ghCommitHash"` + GHPAccessToken string `json:"ghToken"` + GHUserName string `json:"ghUserName"` + GHRepoURL string `json:"ghRepoUrl"` +} + +type JobArgs struct { + Input DispatchArgs `json:"input"` + Action string `json:"action"` +} diff --git a/tests/download-multiple_test.go b/tests/download-multiple_test.go index 0113935..aca1d49 100644 --- a/tests/download-multiple_test.go +++ b/tests/download-multiple_test.go @@ -1,183 +1,94 @@ package tests import ( + "context" "fmt" - "io" - "net/http" - "os" - "strings" "testing" - "github.com/calypr/data-client/client/common" - g3cmd "github.com/calypr/data-client/client/g3cmd" - "github.com/calypr/data-client/client/jwt" - "github.com/calypr/data-client/client/logs" - "github.com/calypr/data-client/client/mocks" - "go.uber.org/mock/gomock" + sylogs "github.com/calypr/syfon/client/logs" + "github.com/calypr/syfon/client/transfer" + "github.com/calypr/syfon/client/transfer/download" ) -// Add all other methods required by your logs.Logger interface! +type fakeResolver struct { + obj *transfer.ResolvedObject + err error +} + +func (f *fakeResolver) Resolve(ctx context.Context, id string) (*transfer.ResolvedObject, error) { + return f.obj, f.err +} -// If Shepherd is deployed, attempt to get the filename from the Shepherd API. func Test_askGen3ForFileInfo_withShepherd(t *testing.T) { - // -- SETUP -- testGUID := "000000-0000000-0000000-000000" testFileName := "test-file" testFileSize := int64(120) - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Expect AskGen3ForFileInfo to call shepherd looking for testGUID: respond with a valid file. - testBody := `{ - "record": { - "file_name": "test-file", - "size": 120, - "did": "000000-0000000-0000000-000000" - }, - "metadata": { - "_file_type": "PFB", - "_resource_paths": ["/open"], - "_uploader_id": 42, - "_bucket": "s3://gen3-bucket" + + logger := sylogs.NewGen3Logger(nil, "", "test") + + skipped := []download.RenamedOrSkippedFileInfo{} + resolver := &fakeResolver{obj: &transfer.ResolvedObject{Id: testGUID, Name: testFileName, Size: testFileSize}} + info, err := download.GetFileInfo(context.Background(), resolver, logger, testGUID, "", "", "original", true, &skipped) + if err != nil { + t.Error(err) } -}` - testResponse := http.Response{ - StatusCode: 200, - Body: io.NopCloser(strings.NewReader(testBody)), + + if info.Name != testFileName { + t.Errorf("Wanted filename %v, got %v", testFileName, info.Name) } - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(true, nil) - mockGen3Interface. - EXPECT(). - GetResponse(common.ShepherdEndpoint+"/objects/"+testGUID, "GET", "", nil). - Return("", &testResponse, nil) - // ---------- - - // Expect AskGen3ForFileInfo to return the correct filename and filesize from shepherd. - fileName, fileSize := g3cmd.AskGen3ForFileInfo(mockGen3Interface, testGUID, "", "", "original", true, &[]g3cmd.RenamedOrSkippedFileInfo{}) - if fileName != testFileName { - t.Errorf("Wanted filename %v, got %v", testFileName, fileName) + if info.Size != testFileSize { + t.Errorf("Wanted filesize %v, got %v", testFileSize, info.Size) } - if fileSize != testFileSize { - t.Errorf("Wanted filesize %v, got %v", testFileSize, fileSize) + if len(skipped) != 0 { + t.Errorf("Expected no skipped files, got %v", skipped) } } -// If there's an error while getting the filename from Shepherd, add the guid -// to *renamedFiles, which tracks which files have errored. func Test_askGen3ForFileInfo_withShepherd_shepherdError(t *testing.T) { - // -- SETUP -- testGUID := "000000-0000000-0000000-000000" - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Expect AskGen3ForFileInfo to call indexd looking for testGUID: - // Respond with an error. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(true, nil) - mockGen3Interface. - EXPECT(). - GetResponse(common.ShepherdEndpoint+"/objects/"+testGUID, "GET", "", nil). - Return("", nil, fmt.Errorf("Error getting metadata from Shepherd")) - // ---------- - - mockGen3Interface. - EXPECT(). - Logger(). - Return(logs.NewTeeLogger("", "test", os.Stdout)). // Or your appropriate dummy logger - AnyTimes() - - // Expect AskGen3ForFileInfo to add this file's GUID to the renamedOrSkippedFiles array. - skipped := []g3cmd.RenamedOrSkippedFileInfo{} - fileName, _ := g3cmd.AskGen3ForFileInfo(mockGen3Interface, testGUID, "", "", "original", true, &skipped) - expected := g3cmd.RenamedOrSkippedFileInfo{GUID: testGUID, OldFilename: "N/A", NewFilename: testGUID} - if skipped[0] != expected { - t.Errorf("Wanted skipped files list to contain %v, got %v", expected, skipped) + + logger := sylogs.NewGen3Logger(nil, "", "test") + + skipped := []download.RenamedOrSkippedFileInfo{} + resolver := &fakeResolver{err: fmt.Errorf("Indexd error")} + info, err := download.GetFileInfo(context.Background(), resolver, logger, testGUID, "", "", "original", true, &skipped) + if err != nil { + t.Fatal(err) + } + + if info == nil { + t.Fatal("AskGen3ForFileInfo returned nil when both Shepherd and Indexd failed. Expected fallback FileInfo with Name = GUID") } - // Expect the returned filename to be the file's GUID. - if fileName != testGUID { - t.Errorf("Wanted filename %v, got %v", testGUID, fileName) + + if info.Name != testGUID { + t.Errorf("Wanted fallback filename %v, got %v", testGUID, info.Name) + } + + if len(skipped) != 1 { + t.Errorf("Expected exactly 1 skipped file, got %d", len(skipped)) + } else if skipped[0].GUID != testGUID || skipped[0].NewFilename != testGUID { + t.Errorf("Skipped entry mismatch: %+v", skipped[0]) } } -// If Shepherd is not deployed, attempt to get the filename from indexd. func Test_askGen3ForFileInfo_noShepherd(t *testing.T) { - // -- SETUP -- testGUID := "000000-0000000-0000000-000000" testFileName := "test-file" testFileSize := int64(120) - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Expect AskGen3ForFileInfo to call indexd looking for testGUID: respond with a valid file. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(false, nil) - mockGen3Interface. - EXPECT(). - DoRequestWithSignedHeader(common.IndexdIndexEndpoint+"/"+testGUID, "", nil). - Return(jwt.JsonMessage{FileName: testFileName, Size: testFileSize}, nil) - // ---------- - - mockGen3Interface. - EXPECT(). - Logger(). - Return(logs.NewTeeLogger("", "test", os.Stdout)). // Or your appropriate dummy logger - AnyTimes() - - // Expect AskGen3ForFileInfo to return the correct filename and filesize from indexd. - fileName, fileSize := g3cmd.AskGen3ForFileInfo(mockGen3Interface, testGUID, "", "", "original", true, &[]g3cmd.RenamedOrSkippedFileInfo{}) - if fileName != testFileName { - t.Errorf("Wanted filename %v, got %v", testFileName, fileName) - } - if fileSize != testFileSize { - t.Errorf("Wanted filesize %v, got %v", testFileSize, fileSize) + + logger := sylogs.NewGen3Logger(nil, "", "test") + + skipped := []download.RenamedOrSkippedFileInfo{} + resolver := &fakeResolver{obj: &transfer.ResolvedObject{Id: testGUID, Name: testFileName, Size: testFileSize}} + info, err := download.GetFileInfo(context.Background(), resolver, logger, testGUID, "", "", "original", true, &skipped) + if err != nil { + t.Fatal(err) } -} -// If there's an error while getting the filename from indexd, add the guid -// to *renamedFiles, which tracks which files have errored. -func Test_askGen3ForFileInfo_noShepherd_indexdError(t *testing.T) { - // -- SETUP -- - testGUID := "000000-0000000-0000000-000000" - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Expect AskGen3ForFileInfo to call indexd looking for testGUID: - // Respond with an error. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(false, nil) - mockGen3Interface. - EXPECT(). - DoRequestWithSignedHeader(common.IndexdIndexEndpoint+"/"+testGUID, "", nil). - Return(jwt.JsonMessage{}, fmt.Errorf("Error downloading file from Indexd")) - // ---------- - mockGen3Interface. - EXPECT(). - Logger(). - Return(logs.NewTeeLogger("", "test", os.Stdout)). // Or your appropriate dummy logger - AnyTimes() - - // Expect AskGen3ForFileInfo to add this file's GUID to the renamedOrSkippedFiles array. - skipped := []g3cmd.RenamedOrSkippedFileInfo{} - fileName, _ := g3cmd.AskGen3ForFileInfo(mockGen3Interface, testGUID, "", "", "original", true, &skipped) - expected := g3cmd.RenamedOrSkippedFileInfo{GUID: testGUID, OldFilename: "N/A", NewFilename: testGUID} - if skipped[0] != expected { - t.Errorf("Wanted skipped files list to contain %v, got %v", expected, skipped) + if info.Name != testFileName { + t.Errorf("Wanted filename %v, got %v", testFileName, info.Name) } - // Expect the returned filename to be the file's GUID. - if fileName != testGUID { - t.Errorf("Wanted filename %v, got %v", testGUID, fileName) + if info.Size != testFileSize { + t.Errorf("Wanted filesize %v, got %v", testFileSize, info.Size) } } diff --git a/tests/functions_test.go b/tests/functions_test.go deleted file mode 100755 index d1e0982..0000000 --- a/tests/functions_test.go +++ /dev/null @@ -1,254 +0,0 @@ -package tests - -import ( - "bytes" - "fmt" - "io" - "net/http" - "reflect" - "strings" - "testing" - - "github.com/calypr/data-client/client/jwt" - "github.com/calypr/data-client/client/mocks" - "go.uber.org/mock/gomock" -) - -func TestDoRequestWithSignedHeaderNoProfile(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "", AccessToken: "", APIEndpoint: ""} - - _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) - - if err == nil { - t.Fail() - } -} - -func TestDoRequestWithSignedHeaderGoodToken(t *testing.T) { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - mockRequest := mocks.NewMockRequestInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig, Request: mockRequest} - - profileConfig := jwt.Credential{Profile: "test", KeyId: "", APIKey: "fake_api_key", AccessToken: "non_expired_token", APIEndpoint: "http://www.test.com", UseShepherd: "false", MinShepherdVersion: ""} - mockedResp := &http.Response{ - Body: io.NopCloser(bytes.NewBufferString("{\"url\": \"http://www.test.com/user/data/download/test_uuid\"}")), - StatusCode: 200, - } - - mockRequest.EXPECT().MakeARequest("GET", "http://www.test.com/user/data/download/test_uuid", "non_expired_token", "", gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - - _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) - - if err != nil { - t.Fail() - } -} - -func TestDoRequestWithSignedHeaderCreateNewToken(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - mockRequest := mocks.NewMockRequestInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig, Request: mockRequest} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "", APIEndpoint: "http://www.test.com"} - mockedResp := &http.Response{ - Body: io.NopCloser(bytes.NewBufferString("{\"url\": \"www.test.com/user/data/download/\"}")), - StatusCode: 200, - } - - mockConfig.EXPECT().UpdateConfigFile(profileConfig).Times(1) - mockRequest.EXPECT().RequestNewAccessToken("http://www.test.com/user/credentials/api/access_token", &profileConfig).Return(nil).Times(1) - mockRequest.EXPECT().MakeARequest(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - - _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) - - if err != nil { - t.Fail() - } -} - -func TestDoRequestWithSignedHeaderRefreshToken(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - mockRequest := mocks.NewMockRequestInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig, Request: mockRequest} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "expired_token", APIEndpoint: "http://www.test.com"} - mockedResp := &http.Response{ - Body: io.NopCloser(bytes.NewBufferString("{\"url\": \"www.test.com/user/data/download/\"}")), - StatusCode: 401, - } - - mockConfig.EXPECT().UpdateConfigFile(profileConfig).Times(1) - mockRequest.EXPECT().RequestNewAccessToken("http://www.test.com/user/credentials/api/access_token", &profileConfig).Return(nil).Times(1) - mockRequest.EXPECT().MakeARequest(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(2) - - _, err := testFunction.DoRequestWithSignedHeader(&profileConfig, "/user/data/download/test_uuid", "", nil) - - if err != nil && !strings.Contains(err.Error(), "401") { - t.Fail() - } - -} - -func TestCheckPrivilegesNoProfile(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "", AccessToken: "", APIEndpoint: ""} - - _, _, err := testFunction.CheckPrivileges(&profileConfig) - - if err == nil { - t.Errorf("Expected an error on missing credentials in configuration, but not received") - } -} - -func TestCheckPrivilegesNoAccess(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - mockRequest := mocks.NewMockRequestInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig, Request: mockRequest} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "non_expired_token", APIEndpoint: "http://www.test.com"} - mockedResp := &http.Response{ - Body: io.NopCloser(bytes.NewBufferString("{\"project_access\": {}}")), - StatusCode: 200, - } - - mockRequest.EXPECT().MakeARequest("GET", "http://www.test.com/user/user", "non_expired_token", "", gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - - _, receivedAccess, err := testFunction.CheckPrivileges(&profileConfig) - - expectedAccess := make(map[string]any) - - if err != nil { - t.Errorf("Expected no errors, received an error \"%v\"", err) - } else if !reflect.DeepEqual(receivedAccess, expectedAccess) { - t.Errorf("Expected no user access, received %v", receivedAccess) - } -} - -func TestCheckPrivilegesGrantedAccess(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - mockRequest := mocks.NewMockRequestInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig, Request: mockRequest} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "non_expired_token", APIEndpoint: "http://www.test.com"} - - grantedAccessJSON := `{ - "project_access": - { - "test_project": ["read", "create","read-storage","update","delete"] - } - }` - - mockedResp := &http.Response{ - Body: io.NopCloser(bytes.NewBufferString(grantedAccessJSON)), - StatusCode: 200, - } - - mockRequest.EXPECT().MakeARequest("GET", "http://www.test.com/user/user", "non_expired_token", "", gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - - _, expectedAccess, err := testFunction.CheckPrivileges(&profileConfig) - - receivedAccess := make(map[string]any) - receivedAccess["test_project"] = []any{ - "read", - "create", - "read-storage", - "update", - "delete"} - - if err != nil { - t.Errorf("Expected no errors, received an error \"%v\"", err) - } else if !reflect.DeepEqual(expectedAccess, receivedAccess) { - t.Errorf(`Expected user access and received user access are not the same. - Expected: %v - Received: %v`, expectedAccess, receivedAccess) - } -} - -// If both `authz` and `project_access` section exists, `authz` takes precedence -func TestCheckPrivilegesGrantedAccessAuthz(t *testing.T) { - - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - mockConfig := mocks.NewMockConfigureInterface(mockCtrl) - mockRequest := mocks.NewMockRequestInterface(mockCtrl) - testFunction := &jwt.Functions{Config: mockConfig, Request: mockRequest} - - profileConfig := jwt.Credential{KeyId: "", APIKey: "fake_api_key", AccessToken: "non_expired_token", APIEndpoint: "http://www.test.com"} - - grantedAccessJSON := `{ - "authz": { - "test_project":[ - {"method":"create", "service":"*"}, - {"method":"delete", "service":"*"}, - {"method":"read", "service":"*"}, - {"method":"read-storage", "service":"*"}, - {"method":"update", "service":"*"}, - {"method":"upload", "service":"*"} - ] - }, - "project_access": { - "test_project": ["read", "create","read-storage","update","delete"] - } - }` - - mockedResp := &http.Response{ - Body: io.NopCloser(bytes.NewBufferString(grantedAccessJSON)), - StatusCode: 200, - } - - mockRequest.EXPECT().MakeARequest("GET", "http://www.test.com/user/user", "non_expired_token", "", gomock.Any(), gomock.Any(), false).Return(mockedResp, nil).Times(1) - - _, expectedAccess, err := testFunction.CheckPrivileges(&profileConfig) - - receivedAccess := make(map[string]any) - receivedAccess["test_project"] = []map[string]any{ - {"method": "create", "service": "*"}, - {"method": "delete", "service": "*"}, - {"method": "read", "service": "*"}, - {"method": "read-storage", "service": "*"}, - {"method": "update", "service": "*"}, - {"method": "upload", "service": "*"}, - } - - if err != nil { - t.Errorf("Expected no errors, received an error \"%v\"", err) - // don't use DeepEqual since expectedAccess is []interface {} and receivedAccess is []map[string]interface {}, just check for contents - } else if fmt.Sprint(expectedAccess) != fmt.Sprint(receivedAccess) { - t.Errorf(`Expected user access and received user access are not the same. - Expected: %v - Received: %v`, expectedAccess, receivedAccess) - } -} diff --git a/tests/utils_test.go b/tests/utils_test.go index ae2c387..4b80f88 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -1,241 +1,88 @@ package tests import ( - "encoding/json" - "fmt" + "context" "io" "net/http" - "strings" "testing" - "github.com/calypr/data-client/client/common" - g3cmd "github.com/calypr/data-client/client/g3cmd" - "github.com/calypr/data-client/client/jwt" - "github.com/calypr/data-client/client/mocks" - "go.uber.org/mock/gomock" + internalapi "github.com/calypr/syfon/apigen/client/internalapi" + sycommon "github.com/calypr/syfon/client/common" + sylogs "github.com/calypr/syfon/client/logs" + "github.com/calypr/syfon/client/transfer" + "github.com/calypr/syfon/client/transfer/upload" ) -// Expect GetDownloadResponse to: -// 1. get the file download URL from Shepherd if it's deployed -// 2. add the file download URL to the FileDownloadResponseObject -// 3. GET the file download URL, and add the response to the FileDownloadResponseObject -func TestGetDownloadResponse_withShepherd(t *testing.T) { - // -- SETUP -- - testGUID := "000000-0000000-0000000-000000" - testFilename := "test-file" - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Mock the request that checks if Shepherd is deployed. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(true, nil) - - // Mock the request to Shepherd for the download URL of this file. - mockDownloadURL := "https://example.com/example.pfb" - downloadURLBody := fmt.Sprintf(`{ - "url": "%v" - }`, mockDownloadURL) - mockDownloadURLResponse := http.Response{ - StatusCode: 200, - Body: io.NopCloser(strings.NewReader(downloadURLBody)), - } - mockGen3Interface. - EXPECT(). - GetResponse(common.ShepherdEndpoint+"/objects/"+testGUID+"/download", "GET", "", nil). - Return("", &mockDownloadURLResponse, nil) - - // Mock the request for the file at mockDownloadURL. - mockFileResponse := http.Response{ - StatusCode: 200, - Body: io.NopCloser(strings.NewReader("It work")), - } - mockGen3Interface. - EXPECT(). - MakeARequest(http.MethodGet, mockDownloadURL, "", "", map[string]string{}, nil, true). - Return(&mockFileResponse, nil) - // ---------- - - mockFDRObj := common.FileDownloadResponseObject{ - Filename: testFilename, - GUID: testGUID, - Range: 0, - } - err := g3cmd.GetDownloadResponse(mockGen3Interface, &mockFDRObj, "") - if err != nil { - t.Error(err) - } - if mockFDRObj.URL != mockDownloadURL { - t.Errorf("Wanted the DownloadPath to be set to %v, got %v", mockDownloadURL, mockFDRObj.DownloadPath) - } - if mockFDRObj.Response != &mockFileResponse { - t.Errorf("Wanted download response to be %v, got %v", mockFileResponse, mockFDRObj.Response) - } +type fakeDownloader struct { + resolveFn func(ctx context.Context, guid, accessID string) (string, error) + downloadFn func(ctx context.Context, url string, rangeStart, rangeEnd *int64) (*http.Response, error) } -// Expect GetDownloadResponse to: -// 1. get the file download URL from Fence if Shepherd is not deployed -// 2. add the file download URL to the FileDownloadResponseObject -// 3. GET the file download URL, and add the response to the FileDownloadResponseObject -func TestGetDownloadResponse_noShepherd(t *testing.T) { - // -- SETUP -- - testGUID := "000000-0000000-0000000-000000" - testFilename := "test-file" - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Mock the request that checks if Shepherd is deployed. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(false, nil) - - // Mock the request to Fence for the download URL of this file. - mockDownloadURL := "https://example.com/example.pfb" - mockDownloadURLResponse := jwt.JsonMessage{ - URL: mockDownloadURL, - } - mockGen3Interface. - EXPECT(). - DoRequestWithSignedHeader(common.FenceDataDownloadEndpoint+"/"+testGUID, "", nil). - Return(mockDownloadURLResponse, nil) - - // Mock the request for the file at mockDownloadURL. - mockFileResponse := http.Response{ - StatusCode: 200, - Body: io.NopCloser(strings.NewReader("It work")), - } - mockGen3Interface. - EXPECT(). - MakeARequest(http.MethodGet, mockDownloadURL, "", "", map[string]string{}, nil, true). - Return(&mockFileResponse, nil) - // ---------- - - mockFDRObj := common.FileDownloadResponseObject{ - Filename: testFilename, - GUID: testGUID, - Range: 0, - } - err := g3cmd.GetDownloadResponse(mockGen3Interface, &mockFDRObj, "") - if err != nil { - t.Error(err) - } - if mockFDRObj.URL != mockDownloadURL { - t.Errorf("Wanted the DownloadPath to be set to %v, got %v", mockDownloadURL, mockFDRObj.DownloadPath) - } - if mockFDRObj.Response != &mockFileResponse { - t.Errorf("Wanted download response to be %v, got %v", mockFileResponse, mockFDRObj.Response) - } +func (f *fakeDownloader) Name() string { return "fake-downloader" } +func (f *fakeDownloader) Logger() transfer.TransferLogger { + return sylogs.NewGen3Logger(nil, "", "test") +} +func (f *fakeDownloader) ResolveDownloadURL(ctx context.Context, guid, accessID string) (string, error) { + return f.resolveFn(ctx, guid, accessID) +} +func (f *fakeDownloader) Download(ctx context.Context, url string, rangeStart, rangeEnd *int64) (*http.Response, error) { + return f.downloadFn(ctx, url, rangeStart, rangeEnd) } -// If Shepherd is not deployed, expect GeneratePresignedURL to hit fence's data upload -// endpoint and return the presigned URL and guid. -func TestGeneratePresignedURL_noShepherd(t *testing.T) { - // -- SETUP -- - testFilename := "test-file" - testBucketname := "test-bucket" - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Mock the request that checks if Shepherd is deployed. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(false, nil) +type fakeUploader struct { + resolveFn func(ctx context.Context, guid, filename string, metadata sycommon.FileMetadata, bucket string) (string, error) +} - // Mock the request to Fence's data upload endpoint to create a presigned url for this file name. - expectedReqBody := []byte(fmt.Sprintf(`{"file_name":"%v","bucket":"%v"}`, testFilename, testBucketname)) - mockPresignedURL := "https://example.com/example.pfb" - mockGUID := "000000-0000000-0000000-000000" - mockUploadURLResponse := jwt.JsonMessage{ - URL: mockPresignedURL, - GUID: mockGUID, - } - mockGen3Interface. - EXPECT(). - DoRequestWithSignedHeader(common.FenceDataUploadEndpoint, "application/json", expectedReqBody). - Return(mockUploadURLResponse, nil) - // ---------- +func (f *fakeUploader) Name() string { return "fake-uploader" } +func (f *fakeUploader) Logger() transfer.TransferLogger { return sylogs.NewGen3Logger(nil, "", "test") } - url, guid, err := g3cmd.GeneratePresignedURL(mockGen3Interface, testFilename, common.FileMetadata{}, testBucketname) - if err != nil { - t.Error(err) - } - if url != mockPresignedURL { - t.Errorf("Wanted the presignedURL to be set to %v, got %v", mockPresignedURL, url) - } - if guid != mockGUID { - t.Errorf("Wanted generated GUID to be %v, got %v", mockGUID, guid) - } +func (f *fakeUploader) ResolveUploadURL(ctx context.Context, guid, filename string, metadata sycommon.FileMetadata, bucket string) (string, error) { + return f.resolveFn(ctx, guid, filename, metadata, bucket) +} +func (f *fakeUploader) InitMultipartUpload(ctx context.Context, guid string, filename string, bucket string) (string, string, error) { + return "", "", nil +} +func (f *fakeUploader) GetMultipartUploadURL(ctx context.Context, key string, uploadID string, partNumber int32, bucket string) (string, error) { + return "", nil +} +func (f *fakeUploader) CompleteMultipartUpload(ctx context.Context, key string, uploadID string, parts []internalapi.InternalMultipartPart, bucket string) error { + return nil +} +func (f *fakeUploader) Upload(ctx context.Context, url string, body io.Reader, size int64) error { + return nil +} +func (f *fakeUploader) UploadPart(ctx context.Context, url string, body io.Reader, size int64) (string, error) { + return "", nil +} +func (f *fakeUploader) DeleteFile(ctx context.Context, guid string) (string, error) { + return "", nil +} +func (f *fakeUploader) CanonicalObjectURL(signedURL, bucketHint, fallbackDID string) (string, error) { + return signedURL, nil } -// If Shepherd is deployed, expect GeneratePresignedURL to hit Shepherd's data upload -// endpoint with the file name and file metadata. GeneratePresignedURL should then -// return the guid and file name that it gets from the endpoint. -func TestGeneratePresignedURL_withShepherd(t *testing.T) { - // -- SETUP -- +func TestGeneratePresignedUploadURL(t *testing.T) { testFilename := "test-file" - testBucketname := "test-bucket" - testMetadata := common.FileMetadata{ - Aliases: []string{"test-alias-1", "test-alias-2"}, - Authz: []string{"authz-resource-1", "authz-resource-2"}, - Metadata: map[string]any{"arbitrary": "metadata"}, - } - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - - // Mock the request that checks if Shepherd is deployed. - mockGen3Interface := mocks.NewMockGen3Interface(mockCtrl) - mockGen3Interface. - EXPECT(). - CheckForShepherdAPI(). - Return(true, nil) - - // Mock the request to Fence's data upload endpoint to create a presigned url for this file name. - expectedReq := g3cmd.ShepherdInitRequestObject{ - Filename: testFilename, - Authz: struct { - Version string `json:"version"` - ResourcePaths []string `json:"resource_paths"` - }{ - "0", - testMetadata.Authz, + testBucket := "test-bucket" + mockUploadURL := "https://example.com/upload" + + bk := &fakeUploader{ + resolveFn: func(ctx context.Context, guid, filename string, metadata sycommon.FileMetadata, bucket string) (string, error) { + if filename != testFilename { + t.Fatalf("unexpected filename: %s", filename) + } + if bucket != testBucket { + t.Fatalf("unexpected bucket: %s", bucket) + } + return mockUploadURL, nil }, - Aliases: testMetadata.Aliases, - Metadata: testMetadata.Metadata, - } - expectedReqBody, err := json.Marshal(expectedReq) - if err != nil { - t.Error(err) - } - mockPresignedURL := "https://example.com/example.pfb" - mockGUID := "000000-0000000-0000000-000000" - presignedURLBody := fmt.Sprintf(`{ - "guid": "%v", - "upload_url": "%v" - }`, mockGUID, mockPresignedURL) - mockUploadURLResponse := http.Response{ - StatusCode: 201, - Body: io.NopCloser(strings.NewReader(presignedURLBody)), } - mockGen3Interface. - EXPECT(). - GetResponse(common.ShepherdEndpoint+"/objects", "POST", "", expectedReqBody). - Return("", &mockUploadURLResponse, nil) - // ---------- - url, guid, err := g3cmd.GeneratePresignedURL(mockGen3Interface, testFilename, testMetadata, testBucketname) + resp, err := upload.GeneratePresignedUploadURL(context.Background(), bk, testFilename, sycommon.FileMetadata{}, testBucket) if err != nil { - t.Error(err) - } - if url != mockPresignedURL { - t.Errorf("Wanted the presignedURL to be set to %v, got %v", mockPresignedURL, url) + t.Fatalf("unexpected error: %v", err) } - if guid != mockGUID { - t.Errorf("Wanted generated GUID to be %v, got %v", mockGUID, guid) + if resp != mockUploadURL { + t.Errorf("wanted URL %s, got %s", mockUploadURL, resp) } }