diff --git a/.github/workflows/weekly-report.lock.yml b/.github/workflows/weekly-report.lock.yml new file mode 100644 index 000000000000..2cac0bdb9e37 --- /dev/null +++ b/.github/workflows/weekly-report.lock.yml @@ -0,0 +1,1083 @@ +# +# ___ _ _ +# / _ \ | | (_) +# | |_| | __ _ ___ _ __ | |_ _ ___ +# | _ |/ _` |/ _ \ '_ \| __| |/ __| +# | | | | (_| | __/ | | | |_| | (__ +# \_| |_/\__, |\___|_| |_|\__|_|\___| +# __/ | +# _ _ |___/ +# | | | | / _| | +# | | | | ___ _ __ _ __| |_| | _____ ____ +# | |/\| |/ _ \ '__| |/ /| _| |/ _ \ \ /\ / / ___| +# \ /\ / (_) | | | | ( | | | | (_) \ V V /\__ \ +# \/ \/ \___/|_| |_|\_\|_| |_|\___/ \_/\_/ |___/ +# +# This file was automatically generated by gh-aw (v0.56.0). DO NOT EDIT. +# +# To update this file, edit the corresponding .md file and run: +# gh aw compile +# Not all edits will cause changes to this file. +# +# For more information: https://github.github.com/gh-aw/introduction/overview/ +# +# Generates a weekly read-only report covering PRs, Issues, and Branch Hygiene for the previous calendar week (Asia/Shanghai time) and posts it to GitHub Discussions. +# +# gh-aw-metadata: {"schema_version":"v2","frontmatter_hash":"ccab0d1c5ce8e55c0bb40e8abde7085c10496c0963cef44d3ea33aa12d851191","compiler_version":"v0.56.0","strict":true} + +name: "Weekly Repository Report" +"on": + schedule: + - cron: "0 22 * * 0" + workflow_dispatch: + +permissions: {} + +concurrency: + group: "gh-aw-${{ github.workflow }}" + +run-name: "Weekly Repository Report" + +jobs: + activation: + runs-on: ubuntu-slim + permissions: + contents: read + outputs: + comment_id: "" + comment_repo: "" + model: ${{ steps.generate_aw_info.outputs.model }} + secret_verification_result: ${{ steps.validate-secret.outputs.verification_result }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@046e81c42fe2a9d91f47596660fcc69f48f5c70a # v0.56.0 + with: + destination: /opt/gh-aw/actions + - name: Generate agentic run info + id: generate_aw_info + env: + GH_AW_INFO_ENGINE_ID: "copilot" + GH_AW_INFO_ENGINE_NAME: "GitHub Copilot CLI" + GH_AW_INFO_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_INFO_VERSION: "" + GH_AW_INFO_AGENT_VERSION: "latest" + GH_AW_INFO_CLI_VERSION: "v0.56.0" + GH_AW_INFO_WORKFLOW_NAME: "Weekly Repository Report" + GH_AW_INFO_EXPERIMENTAL: "false" + GH_AW_INFO_SUPPORTS_TOOLS_ALLOWLIST: "true" + GH_AW_INFO_STAGED: "false" + GH_AW_INFO_ALLOWED_DOMAINS: '["defaults"]' + GH_AW_INFO_FIREWALL_ENABLED: "true" + GH_AW_INFO_AWF_VERSION: "v0.23.0" + GH_AW_INFO_AWMG_VERSION: "" + GH_AW_INFO_FIREWALL_TYPE: "squid" + GH_AW_COMPILED_STRICT: "true" + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { main } = require('/opt/gh-aw/actions/generate_aw_info.cjs'); + await main(core, context); + - name: Validate COPILOT_GITHUB_TOKEN secret + id: validate-secret + run: /opt/gh-aw/actions/validate_multi_secret.sh COPILOT_GITHUB_TOKEN 'GitHub Copilot CLI' https://github.github.com/gh-aw/reference/engines/#github-copilot-default + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + - name: Checkout .github and .agents folders + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + sparse-checkout: | + .github + .agents + sparse-checkout-cone-mode: true + fetch-depth: 1 + persist-credentials: false + - name: Check workflow file timestamps + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_WORKFLOW_FILE: "weekly-report.lock.yml" + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/check_workflow_timestamp_api.cjs'); + await main(); + - name: Create prompt with built-in context + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + run: | + bash /opt/gh-aw/actions/create_prompt_first.sh + { + cat << 'GH_AW_PROMPT_EOF' + + GH_AW_PROMPT_EOF + cat "/opt/gh-aw/prompts/xpia.md" + cat "/opt/gh-aw/prompts/temp_folder_prompt.md" + cat "/opt/gh-aw/prompts/markdown.md" + cat "/opt/gh-aw/prompts/safe_outputs_prompt.md" + cat << 'GH_AW_PROMPT_EOF' + + Tools: create_discussion, missing_tool, missing_data, noop + + + The following GitHub context information is available for this workflow: + {{#if __GH_AW_GITHUB_ACTOR__ }} + - **actor**: __GH_AW_GITHUB_ACTOR__ + {{/if}} + {{#if __GH_AW_GITHUB_REPOSITORY__ }} + - **repository**: __GH_AW_GITHUB_REPOSITORY__ + {{/if}} + {{#if __GH_AW_GITHUB_WORKSPACE__ }} + - **workspace**: __GH_AW_GITHUB_WORKSPACE__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ }} + - **issue-number**: #__GH_AW_GITHUB_EVENT_ISSUE_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ }} + - **discussion-number**: #__GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ }} + - **pull-request-number**: #__GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER__ + {{/if}} + {{#if __GH_AW_GITHUB_EVENT_COMMENT_ID__ }} + - **comment-id**: __GH_AW_GITHUB_EVENT_COMMENT_ID__ + {{/if}} + {{#if __GH_AW_GITHUB_RUN_ID__ }} + - **workflow-run-id**: __GH_AW_GITHUB_RUN_ID__ + {{/if}} + + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' + + GH_AW_PROMPT_EOF + cat << 'GH_AW_PROMPT_EOF' + {{#runtime-import .github/workflows/weekly-report.md}} + GH_AW_PROMPT_EOF + } > "$GH_AW_PROMPT" + - name: Interpolate variables and render templates + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/interpolate_prompt.cjs'); + await main(); + - name: Substitute placeholders + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_GITHUB_ACTOR: ${{ github.actor }} + GH_AW_GITHUB_EVENT_COMMENT_ID: ${{ github.event.comment.id }} + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: ${{ github.event.discussion.number }} + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: ${{ github.event.issue.number }} + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }} + GH_AW_GITHUB_REPOSITORY: ${{ github.repository }} + GH_AW_GITHUB_RUN_ID: ${{ github.run_id }} + GH_AW_GITHUB_WORKSPACE: ${{ github.workspace }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + + const substitutePlaceholders = require('/opt/gh-aw/actions/substitute_placeholders.cjs'); + + // Call the substitution function + return await substitutePlaceholders({ + file: process.env.GH_AW_PROMPT, + substitutions: { + GH_AW_GITHUB_ACTOR: process.env.GH_AW_GITHUB_ACTOR, + GH_AW_GITHUB_EVENT_COMMENT_ID: process.env.GH_AW_GITHUB_EVENT_COMMENT_ID, + GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER: process.env.GH_AW_GITHUB_EVENT_DISCUSSION_NUMBER, + GH_AW_GITHUB_EVENT_ISSUE_NUMBER: process.env.GH_AW_GITHUB_EVENT_ISSUE_NUMBER, + GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER: process.env.GH_AW_GITHUB_EVENT_PULL_REQUEST_NUMBER, + GH_AW_GITHUB_REPOSITORY: process.env.GH_AW_GITHUB_REPOSITORY, + GH_AW_GITHUB_RUN_ID: process.env.GH_AW_GITHUB_RUN_ID, + GH_AW_GITHUB_WORKSPACE: process.env.GH_AW_GITHUB_WORKSPACE + } + }); + - name: Validate prompt placeholders + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/validate_prompt_placeholders.sh + - name: Print prompt + env: + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + run: bash /opt/gh-aw/actions/print_prompt_summary.sh + - name: Upload activation artifact + if: success() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: activation + path: | + /tmp/gh-aw/aw_info.json + /tmp/gh-aw/aw-prompts/prompt.txt + retention-days: 1 + + agent: + needs: activation + runs-on: ubuntu-latest + permissions: + contents: read + discussions: read + issues: read + pull-requests: read + concurrency: + group: "gh-aw-copilot-${{ github.workflow }}" + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GH_AW_ASSETS_ALLOWED_EXTS: "" + GH_AW_ASSETS_BRANCH: "" + GH_AW_ASSETS_MAX_SIZE_KB: 0 + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + GH_AW_SAFE_OUTPUTS: /opt/gh-aw/safeoutputs/outputs.jsonl + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + GH_AW_WORKFLOW_ID_SANITIZED: weeklyreport + outputs: + checkout_pr_success: ${{ steps.checkout-pr.outputs.checkout_pr_success || 'true' }} + detection_conclusion: ${{ steps.detection_conclusion.outputs.conclusion }} + detection_success: ${{ steps.detection_conclusion.outputs.success }} + has_patch: ${{ steps.collect_output.outputs.has_patch }} + inference_access_error: ${{ steps.detect-inference-error.outputs.inference_access_error || 'false' }} + model: ${{ needs.activation.outputs.model }} + output: ${{ steps.collect_output.outputs.output }} + output_types: ${{ steps.collect_output.outputs.output_types }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@046e81c42fe2a9d91f47596660fcc69f48f5c70a # v0.56.0 + with: + destination: /opt/gh-aw/actions + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Create gh-aw temp directory + run: bash /opt/gh-aw/actions/create_gh_aw_tmp_dir.sh + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Checkout PR branch + id: checkout-pr + if: | + (github.event.pull_request) || (github.event.issue.pull_request) + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + with: + github-token: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/checkout_pr_branch.cjs'); + await main(); + - name: Install GitHub Copilot CLI + run: /opt/gh-aw/actions/install_copilot_cli.sh latest + - name: Install awf binary + run: bash /opt/gh-aw/actions/install_awf_binary.sh v0.23.0 + - name: Download container images + run: bash /opt/gh-aw/actions/download_docker_images.sh ghcr.io/github/gh-aw-firewall/agent:0.23.0 ghcr.io/github/gh-aw-firewall/api-proxy:0.23.0 ghcr.io/github/gh-aw-firewall/squid:0.23.0 ghcr.io/github/gh-aw-mcpg:v0.1.8 node:lts-alpine + - name: Write Safe Outputs Config + run: | + mkdir -p /opt/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/safeoutputs + mkdir -p /tmp/gh-aw/mcp-logs/safeoutputs + cat > /opt/gh-aw/safeoutputs/config.json << 'GH_AW_SAFE_OUTPUTS_CONFIG_EOF' + {"create_discussion":{"expires":168,"max":1},"missing_data":{},"missing_tool":{},"noop":{"max":1}} + GH_AW_SAFE_OUTPUTS_CONFIG_EOF + cat > /opt/gh-aw/safeoutputs/tools.json << 'GH_AW_SAFE_OUTPUTS_TOOLS_EOF' + [ + { + "description": "Create a GitHub discussion for announcements, Q\u0026A, reports, status updates, or community conversations. Use this for content that benefits from threaded replies, doesn't require task tracking, or serves as documentation. For actionable work items that need assignment and status tracking, use create_issue instead. CONSTRAINTS: Maximum 1 discussion(s) can be created.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "body": { + "description": "Discussion content in Markdown. Do NOT repeat the title as a heading since it already appears as the discussion's h1. Include all relevant context, findings, or questions.", + "type": "string" + }, + "category": { + "description": "Discussion category by name (e.g., 'General'), slug (e.g., 'general'), or ID. If omitted, uses the first available category. Category must exist in the repository.", + "type": "string" + }, + "integrity": { + "description": "Trustworthiness level of the message source (e.g., \"low\", \"medium\", \"high\").", + "type": "string" + }, + "secrecy": { + "description": "Confidentiality level of the message content (e.g., \"public\", \"internal\", \"private\").", + "type": "string" + }, + "title": { + "description": "Concise discussion title summarizing the topic. The title appears as the main heading, so keep it brief and descriptive.", + "type": "string" + } + }, + "required": [ + "title", + "body" + ], + "type": "object" + }, + "name": "create_discussion" + }, + { + "description": "Report that a tool or capability needed to complete the task is not available, or share any information you deem important about missing functionality or limitations. Use this when you cannot accomplish what was requested because the required functionality is missing or access is restricted.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "integrity": { + "description": "Trustworthiness level of the message source (e.g., \"low\", \"medium\", \"high\").", + "type": "string" + }, + "reason": { + "description": "Explanation of why this tool is needed or what information you want to share about the limitation (max 256 characters).", + "type": "string" + }, + "secrecy": { + "description": "Confidentiality level of the message content (e.g., \"public\", \"internal\", \"private\").", + "type": "string" + }, + "tool": { + "description": "Optional: Name or description of the missing tool or capability (max 128 characters). Be specific about what functionality is needed.", + "type": "string" + } + }, + "required": [ + "reason" + ], + "type": "object" + }, + "name": "missing_tool" + }, + { + "description": "Log a transparency message when no significant actions are needed. Use this to confirm workflow completion and provide visibility when analysis is complete but no changes or outputs are required (e.g., 'No issues found', 'All checks passed'). This ensures the workflow produces human-visible output even when no other actions are taken.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "integrity": { + "description": "Trustworthiness level of the message source (e.g., \"low\", \"medium\", \"high\").", + "type": "string" + }, + "message": { + "description": "Status or completion message to log. Should explain what was analyzed and the outcome (e.g., 'Code review complete - no issues found', 'Analysis complete - all tests passing').", + "type": "string" + }, + "secrecy": { + "description": "Confidentiality level of the message content (e.g., \"public\", \"internal\", \"private\").", + "type": "string" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "name": "noop" + }, + { + "description": "Report that data or information needed to complete the task is not available. Use this when you cannot accomplish what was requested because required data, context, or information is missing.", + "inputSchema": { + "additionalProperties": false, + "properties": { + "alternatives": { + "description": "Any workarounds, manual steps, or alternative approaches the user could take (max 256 characters).", + "type": "string" + }, + "context": { + "description": "Additional context about the missing data or where it should come from (max 256 characters).", + "type": "string" + }, + "data_type": { + "description": "Type or description of the missing data or information (max 128 characters). Be specific about what data is needed.", + "type": "string" + }, + "integrity": { + "description": "Trustworthiness level of the message source (e.g., \"low\", \"medium\", \"high\").", + "type": "string" + }, + "reason": { + "description": "Explanation of why this data is needed to complete the task (max 256 characters).", + "type": "string" + }, + "secrecy": { + "description": "Confidentiality level of the message content (e.g., \"public\", \"internal\", \"private\").", + "type": "string" + } + }, + "required": [], + "type": "object" + }, + "name": "missing_data" + } + ] + GH_AW_SAFE_OUTPUTS_TOOLS_EOF + cat > /opt/gh-aw/safeoutputs/validation.json << 'GH_AW_SAFE_OUTPUTS_VALIDATION_EOF' + { + "create_discussion": { + "defaultMax": 1, + "fields": { + "body": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + }, + "category": { + "type": "string", + "sanitize": true, + "maxLength": 128 + }, + "repo": { + "type": "string", + "maxLength": 256 + }, + "title": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "missing_data": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "context": { + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "data_type": { + "type": "string", + "sanitize": true, + "maxLength": 128 + }, + "reason": { + "type": "string", + "sanitize": true, + "maxLength": 256 + } + } + }, + "missing_tool": { + "defaultMax": 20, + "fields": { + "alternatives": { + "type": "string", + "sanitize": true, + "maxLength": 512 + }, + "reason": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 256 + }, + "tool": { + "type": "string", + "sanitize": true, + "maxLength": 128 + } + } + }, + "noop": { + "defaultMax": 1, + "fields": { + "message": { + "required": true, + "type": "string", + "sanitize": true, + "maxLength": 65000 + } + } + } + } + GH_AW_SAFE_OUTPUTS_VALIDATION_EOF + - name: Generate Safe Outputs MCP Server Config + id: safe-outputs-config + run: | + # Generate a secure random API key (360 bits of entropy, 40+ chars) + # Mask immediately to prevent timing vulnerabilities + API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${API_KEY}" + + PORT=3001 + + # Set outputs for next steps + { + echo "safe_outputs_api_key=${API_KEY}" + echo "safe_outputs_port=${PORT}" + } >> "$GITHUB_OUTPUT" + + echo "Safe Outputs MCP server will run on port ${PORT}" + + - name: Start Safe Outputs MCP HTTP Server + id: safe-outputs-start + env: + DEBUG: '*' + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-config.outputs.safe_outputs_port }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-config.outputs.safe_outputs_api_key }} + GH_AW_SAFE_OUTPUTS_TOOLS_PATH: /opt/gh-aw/safeoutputs/tools.json + GH_AW_SAFE_OUTPUTS_CONFIG_PATH: /opt/gh-aw/safeoutputs/config.json + GH_AW_MCP_LOG_DIR: /tmp/gh-aw/mcp-logs/safeoutputs + run: | + # Environment variables are set above to prevent template injection + export DEBUG + export GH_AW_SAFE_OUTPUTS_PORT + export GH_AW_SAFE_OUTPUTS_API_KEY + export GH_AW_SAFE_OUTPUTS_TOOLS_PATH + export GH_AW_SAFE_OUTPUTS_CONFIG_PATH + export GH_AW_MCP_LOG_DIR + + bash /opt/gh-aw/actions/start_safe_outputs_server.sh + + - name: Start MCP Gateway + id: start-mcp-gateway + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_SAFE_OUTPUTS_API_KEY: ${{ steps.safe-outputs-start.outputs.api_key }} + GH_AW_SAFE_OUTPUTS_PORT: ${{ steps.safe-outputs-start.outputs.port }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + run: | + set -eo pipefail + mkdir -p /tmp/gh-aw/mcp-config + + # Export gateway environment variables for MCP config and gateway script + export MCP_GATEWAY_PORT="80" + export MCP_GATEWAY_DOMAIN="host.docker.internal" + MCP_GATEWAY_API_KEY=$(openssl rand -base64 45 | tr -d '/+=') + echo "::add-mask::${MCP_GATEWAY_API_KEY}" + export MCP_GATEWAY_API_KEY + export MCP_GATEWAY_PAYLOAD_DIR="/tmp/gh-aw/mcp-payloads" + mkdir -p "${MCP_GATEWAY_PAYLOAD_DIR}" + export MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD="524288" + export DEBUG="*" + + export GH_AW_ENGINE="copilot" + export GITHUB_PERSONAL_ACCESS_TOKEN="$GITHUB_MCP_SERVER_TOKEN" + export MCP_GATEWAY_DOCKER_COMMAND='docker run -i --rm --network host -v /var/run/docker.sock:/var/run/docker.sock -e MCP_GATEWAY_PORT -e MCP_GATEWAY_DOMAIN -e MCP_GATEWAY_API_KEY -e MCP_GATEWAY_PAYLOAD_DIR -e MCP_GATEWAY_PAYLOAD_SIZE_THRESHOLD -e DEBUG -e MCP_GATEWAY_LOG_DIR -e GH_AW_MCP_LOG_DIR -e GH_AW_SAFE_OUTPUTS -e GH_AW_SAFE_OUTPUTS_CONFIG_PATH -e GH_AW_SAFE_OUTPUTS_TOOLS_PATH -e GH_AW_ASSETS_BRANCH -e GH_AW_ASSETS_MAX_SIZE_KB -e GH_AW_ASSETS_ALLOWED_EXTS -e DEFAULT_BRANCH -e GITHUB_MCP_SERVER_TOKEN -e GITHUB_PERSONAL_ACCESS_TOKEN -e GITHUB_MCP_LOCKDOWN -e GITHUB_REPOSITORY -e GITHUB_SERVER_URL -e GITHUB_SHA -e GITHUB_WORKSPACE -e GITHUB_TOKEN -e GITHUB_RUN_ID -e GITHUB_RUN_NUMBER -e GITHUB_RUN_ATTEMPT -e GITHUB_JOB -e GITHUB_ACTION -e GITHUB_EVENT_NAME -e GITHUB_EVENT_PATH -e GITHUB_ACTOR -e GITHUB_ACTOR_ID -e GITHUB_TRIGGERING_ACTOR -e GITHUB_WORKFLOW -e GITHUB_WORKFLOW_REF -e GITHUB_WORKFLOW_SHA -e GITHUB_REF -e GITHUB_REF_NAME -e GITHUB_REF_TYPE -e GITHUB_HEAD_REF -e GITHUB_BASE_REF -e GH_AW_SAFE_OUTPUTS_PORT -e GH_AW_SAFE_OUTPUTS_API_KEY -v /tmp/gh-aw/mcp-payloads:/tmp/gh-aw/mcp-payloads:rw -v /opt:/opt:ro -v /tmp:/tmp:rw -v '"${GITHUB_WORKSPACE}"':'"${GITHUB_WORKSPACE}"':rw ghcr.io/github/gh-aw-mcpg:v0.1.8' + + mkdir -p /home/runner/.copilot + cat << GH_AW_MCP_CONFIG_EOF | bash /opt/gh-aw/actions/start_mcp_gateway.sh + { + "mcpServers": { + "github": { + "type": "http", + "url": "https://api.githubcopilot.com/mcp/", + "headers": { + "Authorization": "Bearer \${GITHUB_PERSONAL_ACCESS_TOKEN}", + "X-MCP-Readonly": "true", + "X-MCP-Toolsets": "context,repos,issues,pull_requests" + }, + "env": { + "GITHUB_PERSONAL_ACCESS_TOKEN": "\${GITHUB_MCP_SERVER_TOKEN}" + } + }, + "safeoutputs": { + "type": "http", + "url": "http://host.docker.internal:$GH_AW_SAFE_OUTPUTS_PORT", + "headers": { + "Authorization": "\${GH_AW_SAFE_OUTPUTS_API_KEY}" + } + } + }, + "gateway": { + "port": $MCP_GATEWAY_PORT, + "domain": "${MCP_GATEWAY_DOMAIN}", + "apiKey": "${MCP_GATEWAY_API_KEY}", + "payloadDir": "${MCP_GATEWAY_PAYLOAD_DIR}" + } + } + GH_AW_MCP_CONFIG_EOF + - name: Download activation artifact + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8 + with: + name: activation + path: /tmp/gh-aw + - name: Clean git credentials + run: bash /opt/gh-aw/actions/clean_git_credentials.sh + - name: Execute GitHub Copilot CLI + id: agentic_execution + # Copilot CLI tool arguments (sorted): + timeout-minutes: 20 + run: | + set -o pipefail + touch /tmp/gh-aw/agent-step-summary.md + # shellcheck disable=SC1003 + sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.23.0 --skip-pull --enable-api-proxy \ + -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-all-tools --allow-all-paths --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/agent-stdio.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_AGENT_COPILOT || '' }} + GH_AW_MCP_CONFIG: /home/runner/.copilot/mcp-config.json + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN || secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + XDG_CONFIG_HOME: /home/runner + - name: Detect inference access error + id: detect-inference-error + if: always() + continue-on-error: true + run: bash /opt/gh-aw/actions/detect_inference_access_error.sh + - name: Configure Git credentials + env: + REPO_NAME: ${{ github.repository }} + SERVER_URL: ${{ github.server_url }} + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git config --global am.keepcr true + # Re-authenticate git with GitHub token + SERVER_URL_STRIPPED="${SERVER_URL#https://}" + git remote set-url origin "https://x-access-token:${{ github.token }}@${SERVER_URL_STRIPPED}/${REPO_NAME}.git" + echo "Git configured with standard GitHub Actions identity" + - name: Copy Copilot session state files to logs + if: always() + continue-on-error: true + run: | + # Copy Copilot session state files to logs folder for artifact collection + # This ensures they are in /tmp/gh-aw/ where secret redaction can scan them + SESSION_STATE_DIR="$HOME/.copilot/session-state" + LOGS_DIR="/tmp/gh-aw/sandbox/agent/logs" + + if [ -d "$SESSION_STATE_DIR" ]; then + echo "Copying Copilot session state files from $SESSION_STATE_DIR to $LOGS_DIR" + mkdir -p "$LOGS_DIR" + cp -v "$SESSION_STATE_DIR"/*.jsonl "$LOGS_DIR/" 2>/dev/null || true + echo "Session state files copied successfully" + else + echo "No session-state directory found at $SESSION_STATE_DIR" + fi + - name: Stop MCP Gateway + if: always() + continue-on-error: true + env: + MCP_GATEWAY_PORT: ${{ steps.start-mcp-gateway.outputs.gateway-port }} + MCP_GATEWAY_API_KEY: ${{ steps.start-mcp-gateway.outputs.gateway-api-key }} + GATEWAY_PID: ${{ steps.start-mcp-gateway.outputs.gateway-pid }} + run: | + bash /opt/gh-aw/actions/stop_mcp_gateway.sh "$GATEWAY_PID" + - name: Redact secrets in logs + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/redact_secrets.cjs'); + await main(); + env: + GH_AW_SECRET_NAMES: 'COPILOT_GITHUB_TOKEN,GH_AW_GITHUB_MCP_SERVER_TOKEN,GH_AW_GITHUB_TOKEN,GITHUB_TOKEN' + SECRET_COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + SECRET_GH_AW_GITHUB_MCP_SERVER_TOKEN: ${{ secrets.GH_AW_GITHUB_MCP_SERVER_TOKEN }} + SECRET_GH_AW_GITHUB_TOKEN: ${{ secrets.GH_AW_GITHUB_TOKEN }} + SECRET_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Append agent step summary + if: always() + run: bash /opt/gh-aw/actions/append_agent_step_summary.sh + - name: Upload Safe Outputs + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: safe-output + path: ${{ env.GH_AW_SAFE_OUTPUTS }} + if-no-files-found: warn + - name: Ingest agent output + id: collect_output + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_SAFE_OUTPUTS: ${{ env.GH_AW_SAFE_OUTPUTS }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/collect_ndjson_output.cjs'); + await main(); + - name: Upload sanitized agent output + if: always() && env.GH_AW_AGENT_OUTPUT + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: agent-output + path: ${{ env.GH_AW_AGENT_OUTPUT }} + if-no-files-found: warn + - name: Upload engine output files + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: agent_outputs + path: | + /tmp/gh-aw/sandbox/agent/logs/ + /tmp/gh-aw/redacted-urls.log + if-no-files-found: ignore + - name: Parse agent logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: /tmp/gh-aw/sandbox/agent/logs/ + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_copilot_log.cjs'); + await main(); + - name: Parse MCP Gateway logs for step summary + if: always() + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_mcp_gateway_log.cjs'); + await main(); + - name: Print firewall logs + if: always() + continue-on-error: true + env: + AWF_LOGS_DIR: /tmp/gh-aw/sandbox/firewall/logs + run: | + # Fix permissions on firewall logs so they can be uploaded as artifacts + # AWF runs with sudo, creating files owned by root + sudo chmod -R a+r /tmp/gh-aw/sandbox/firewall/logs 2>/dev/null || true + # Only run awf logs summary if awf command exists (it may not be installed if workflow failed before install step) + if command -v awf &> /dev/null; then + awf logs summary | tee -a "$GITHUB_STEP_SUMMARY" + else + echo 'AWF binary not installed, skipping firewall log summary' + fi + - name: Upload agent artifacts + if: always() + continue-on-error: true + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: agent-artifacts + path: | + /tmp/gh-aw/aw-prompts/prompt.txt + /tmp/gh-aw/mcp-logs/ + /tmp/gh-aw/sandbox/firewall/logs/ + /tmp/gh-aw/agent-stdio.log + /tmp/gh-aw/agent/ + if-no-files-found: ignore + # --- Threat Detection (inline) --- + - name: Check if detection needed + id: detection_guard + if: always() + env: + OUTPUT_TYPES: ${{ steps.collect_output.outputs.output_types }} + HAS_PATCH: ${{ steps.collect_output.outputs.has_patch }} + run: | + if [[ -n "$OUTPUT_TYPES" || "$HAS_PATCH" == "true" ]]; then + echo "run_detection=true" >> "$GITHUB_OUTPUT" + echo "Detection will run: output_types=$OUTPUT_TYPES, has_patch=$HAS_PATCH" + else + echo "run_detection=false" >> "$GITHUB_OUTPUT" + echo "Detection skipped: no agent outputs or patches to analyze" + fi + - name: Clear MCP configuration for detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + rm -f /tmp/gh-aw/mcp-config/mcp-servers.json + rm -f /home/runner/.copilot/mcp-config.json + rm -f "$GITHUB_WORKSPACE/.gemini/settings.json" + - name: Prepare threat detection files + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection/aw-prompts + cp /tmp/gh-aw/aw-prompts/prompt.txt /tmp/gh-aw/threat-detection/aw-prompts/prompt.txt 2>/dev/null || true + cp /tmp/gh-aw/agent_output.json /tmp/gh-aw/threat-detection/agent_output.json 2>/dev/null || true + for f in /tmp/gh-aw/aw-*.patch; do + [ -f "$f" ] && cp "$f" /tmp/gh-aw/threat-detection/ 2>/dev/null || true + done + echo "Prepared threat detection files:" + ls -la /tmp/gh-aw/threat-detection/ 2>/dev/null || true + - name: Setup threat detection + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + WORKFLOW_NAME: "Weekly Repository Report" + WORKFLOW_DESCRIPTION: "Generates a weekly read-only report covering PRs, Issues, and Branch Hygiene for the previous calendar week (Asia/Shanghai time) and posts it to GitHub Discussions." + HAS_PATCH: ${{ steps.collect_output.outputs.has_patch }} + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/setup_threat_detection.cjs'); + await main(); + - name: Ensure threat-detection directory and log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + run: | + mkdir -p /tmp/gh-aw/threat-detection + touch /tmp/gh-aw/threat-detection/detection.log + - name: Execute GitHub Copilot CLI + if: always() && steps.detection_guard.outputs.run_detection == 'true' + id: detection_agentic_execution + # Copilot CLI tool arguments (sorted): + # --allow-tool shell(cat) + # --allow-tool shell(grep) + # --allow-tool shell(head) + # --allow-tool shell(jq) + # --allow-tool shell(ls) + # --allow-tool shell(tail) + # --allow-tool shell(wc) + timeout-minutes: 20 + run: | + set -o pipefail + touch /tmp/gh-aw/agent-step-summary.md + # shellcheck disable=SC1003 + sudo -E awf --env-all --container-workdir "${GITHUB_WORKSPACE}" --allow-domains "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,github.com,host.docker.internal,raw.githubusercontent.com,registry.npmjs.org,telemetry.enterprise.githubcopilot.com" --log-level info --proxy-logs-dir /tmp/gh-aw/sandbox/firewall/logs --enable-host-access --image-tag 0.23.0 --skip-pull --enable-api-proxy \ + -- /bin/bash -c '/usr/local/bin/copilot --add-dir /tmp/gh-aw/ --log-level all --log-dir /tmp/gh-aw/sandbox/agent/logs/ --add-dir "${GITHUB_WORKSPACE}" --disable-builtin-mcps --allow-tool '\''shell(cat)'\'' --allow-tool '\''shell(grep)'\'' --allow-tool '\''shell(head)'\'' --allow-tool '\''shell(jq)'\'' --allow-tool '\''shell(ls)'\'' --allow-tool '\''shell(tail)'\'' --allow-tool '\''shell(wc)'\'' --prompt "$(cat /tmp/gh-aw/aw-prompts/prompt.txt)"' 2>&1 | tee -a /tmp/gh-aw/threat-detection/detection.log + env: + COPILOT_AGENT_RUNNER_TYPE: STANDALONE + COPILOT_GITHUB_TOKEN: ${{ secrets.COPILOT_GITHUB_TOKEN }} + COPILOT_MODEL: ${{ vars.GH_AW_MODEL_DETECTION_COPILOT || '' }} + GH_AW_PROMPT: /tmp/gh-aw/aw-prompts/prompt.txt + GITHUB_API_URL: ${{ github.api_url }} + GITHUB_HEAD_REF: ${{ github.head_ref }} + GITHUB_REF_NAME: ${{ github.ref_name }} + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_STEP_SUMMARY: /tmp/gh-aw/agent-step-summary.md + GITHUB_WORKSPACE: ${{ github.workspace }} + XDG_CONFIG_HOME: /home/runner + - name: Parse threat detection results + id: parse_detection_results + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + with: + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/parse_threat_detection_results.cjs'); + await main(); + - name: Upload threat detection log + if: always() && steps.detection_guard.outputs.run_detection == 'true' + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: threat-detection.log + path: /tmp/gh-aw/threat-detection/detection.log + if-no-files-found: ignore + - name: Set detection conclusion + id: detection_conclusion + if: always() + env: + RUN_DETECTION: ${{ steps.detection_guard.outputs.run_detection }} + DETECTION_SUCCESS: ${{ steps.parse_detection_results.outputs.success }} + run: | + if [[ "$RUN_DETECTION" != "true" ]]; then + echo "conclusion=skipped" >> "$GITHUB_OUTPUT" + echo "success=true" >> "$GITHUB_OUTPUT" + echo "Detection was not needed, marking as skipped" + elif [[ "$DETECTION_SUCCESS" == "true" ]]; then + echo "conclusion=success" >> "$GITHUB_OUTPUT" + echo "success=true" >> "$GITHUB_OUTPUT" + echo "Detection passed successfully" + else + echo "conclusion=failure" >> "$GITHUB_OUTPUT" + echo "success=false" >> "$GITHUB_OUTPUT" + echo "Detection found issues" + fi + + conclusion: + needs: + - activation + - agent + - safe_outputs + if: (always()) && (needs.agent.result != 'skipped') + runs-on: ubuntu-slim + permissions: + contents: read + discussions: write + issues: write + concurrency: + group: "gh-aw-conclusion-weekly-report" + cancel-in-progress: false + outputs: + noop_message: ${{ steps.noop.outputs.noop_message }} + tools_reported: ${{ steps.missing_tool.outputs.tools_reported }} + total_count: ${{ steps.missing_tool.outputs.total_count }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@046e81c42fe2a9d91f47596660fcc69f48f5c70a # v0.56.0 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Process No-Op Messages + id: noop + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_NOOP_MAX: "1" + GH_AW_WORKFLOW_NAME: "Weekly Repository Report" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/noop.cjs'); + await main(); + - name: Record Missing Tool + id: missing_tool + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Weekly Repository Report" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/missing_tool.cjs'); + await main(); + - name: Handle Agent Failure + id: handle_agent_failure + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Weekly Repository Report" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_WORKFLOW_ID: "weekly-report" + GH_AW_SECRET_VERIFICATION_RESULT: ${{ needs.activation.outputs.secret_verification_result }} + GH_AW_CHECKOUT_PR_SUCCESS: ${{ needs.agent.outputs.checkout_pr_success }} + GH_AW_INFERENCE_ACCESS_ERROR: ${{ needs.agent.outputs.inference_access_error }} + GH_AW_CREATE_DISCUSSION_ERRORS: ${{ needs.safe_outputs.outputs.create_discussion_errors }} + GH_AW_CREATE_DISCUSSION_ERROR_COUNT: ${{ needs.safe_outputs.outputs.create_discussion_error_count }} + GH_AW_GROUP_REPORTS: "false" + GH_AW_TIMEOUT_MINUTES: "20" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_agent_failure.cjs'); + await main(); + - name: Handle No-Op Message + id: handle_noop_message + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_WORKFLOW_NAME: "Weekly Repository Report" + GH_AW_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + GH_AW_AGENT_CONCLUSION: ${{ needs.agent.result }} + GH_AW_NOOP_MESSAGE: ${{ steps.noop.outputs.noop_message }} + GH_AW_NOOP_REPORT_AS_ISSUE: "true" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/handle_noop_message.cjs'); + await main(); + + safe_outputs: + needs: agent + if: ((!cancelled()) && (needs.agent.result != 'skipped')) && (needs.agent.outputs.detection_success == 'true') + runs-on: ubuntu-slim + permissions: + contents: read + discussions: write + issues: write + timeout-minutes: 15 + env: + GH_AW_CALLER_WORKFLOW_ID: "${{ github.repository }}/weekly-report" + GH_AW_ENGINE_ID: "copilot" + GH_AW_WORKFLOW_ID: "weekly-report" + GH_AW_WORKFLOW_NAME: "Weekly Repository Report" + outputs: + code_push_failure_count: ${{ steps.process_safe_outputs.outputs.code_push_failure_count }} + code_push_failure_errors: ${{ steps.process_safe_outputs.outputs.code_push_failure_errors }} + create_discussion_error_count: ${{ steps.process_safe_outputs.outputs.create_discussion_error_count }} + create_discussion_errors: ${{ steps.process_safe_outputs.outputs.create_discussion_errors }} + process_safe_outputs_processed_count: ${{ steps.process_safe_outputs.outputs.processed_count }} + process_safe_outputs_temporary_id_map: ${{ steps.process_safe_outputs.outputs.temporary_id_map }} + steps: + - name: Setup Scripts + uses: github/gh-aw/actions/setup@046e81c42fe2a9d91f47596660fcc69f48f5c70a # v0.56.0 + with: + destination: /opt/gh-aw/actions + - name: Download agent output artifact + id: download-agent-output + continue-on-error: true + uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8 + with: + name: agent-output + path: /tmp/gh-aw/safeoutputs/ + - name: Setup agent output environment variable + if: steps.download-agent-output.outcome == 'success' + run: | + mkdir -p /tmp/gh-aw/safeoutputs/ + find "/tmp/gh-aw/safeoutputs/" -type f -print + echo "GH_AW_AGENT_OUTPUT=/tmp/gh-aw/safeoutputs/agent_output.json" >> "$GITHUB_ENV" + - name: Process Safe Outputs + id: process_safe_outputs + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + GH_AW_AGENT_OUTPUT: ${{ env.GH_AW_AGENT_OUTPUT }} + GH_AW_ALLOWED_DOMAINS: "api.business.githubcopilot.com,api.enterprise.githubcopilot.com,api.github.com,api.githubcopilot.com,api.individual.githubcopilot.com,api.snapcraft.io,archive.ubuntu.com,azure.archive.ubuntu.com,crl.geotrust.com,crl.globalsign.com,crl.identrust.com,crl.sectigo.com,crl.thawte.com,crl.usertrust.com,crl.verisign.com,crl3.digicert.com,crl4.digicert.com,crls.ssl.com,github.com,host.docker.internal,json-schema.org,json.schemastore.org,keyserver.ubuntu.com,ocsp.digicert.com,ocsp.geotrust.com,ocsp.globalsign.com,ocsp.identrust.com,ocsp.sectigo.com,ocsp.ssl.com,ocsp.thawte.com,ocsp.usertrust.com,ocsp.verisign.com,packagecloud.io,packages.cloud.google.com,packages.microsoft.com,ppa.launchpad.net,raw.githubusercontent.com,registry.npmjs.org,s.symcb.com,s.symcd.com,security.ubuntu.com,telemetry.enterprise.githubcopilot.com,ts-crl.ws.symantec.com,ts-ocsp.ws.symantec.com" + GITHUB_SERVER_URL: ${{ github.server_url }} + GITHUB_API_URL: ${{ github.api_url }} + GH_AW_SAFE_OUTPUTS_HANDLER_CONFIG: "{\"create_discussion\":{\"expires\":168,\"fallback_to_issue\":true,\"max\":1},\"missing_data\":{},\"missing_tool\":{}}" + with: + github-token: ${{ secrets.GH_AW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }} + script: | + const { setupGlobals } = require('/opt/gh-aw/actions/setup_globals.cjs'); + setupGlobals(core, github, context, exec, io); + const { main } = require('/opt/gh-aw/actions/safe_output_handler_manager.cjs'); + await main(); + - name: Upload safe output items manifest + if: always() + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7 + with: + name: safe-output-items + path: /tmp/safe-output-items.jsonl + if-no-files-found: warn + diff --git a/.github/workflows/weekly-report.md b/.github/workflows/weekly-report.md new file mode 100644 index 000000000000..ee7ad595a089 --- /dev/null +++ b/.github/workflows/weekly-report.md @@ -0,0 +1,249 @@ +--- +name: Weekly Repository Report +description: > + Generates a weekly read-only report covering PRs, Issues, and Branch Hygiene + for the previous calendar week (Asia/Shanghai time) and posts it to GitHub Discussions. +on: + schedule: + - cron: "0 22 * * 0" +permissions: + contents: read + pull-requests: read + issues: read + discussions: read +tools: + github: + mode: remote + toolsets: [default, repos, issues, pull_requests] + lockdown: false +safe-outputs: + create-discussion: + max: 1 +--- + +# Weekly Repository Report Generator + +You are a read-only reporting agent. Your ONLY write operation is posting a single GitHub Discussion with the weekly report. You must NOT create, update, comment on, label, assign, close, reopen, merge, or otherwise modify Pull Requests, Issues, or Branches. + +## Task + +Generate a comprehensive weekly report for the **taosdata/TDengine** repository covering the **previous calendar week in Asia/Shanghai (UTC+8) time**. + +### Date Calculation + +The reporting window is: + +- Start: Last Monday 00:00:00 Asia/Shanghai (i.e., the Monday that started 7 days before today) +- End: Last Sunday 23:59:59 Asia/Shanghai + +To calculate the exact UTC date range for API queries: + +- Asia/Shanghai is UTC+8 +- Start UTC = Last Monday 00:00:00 +08:00 = Last Sunday 16:00:00 UTC (the day before) +- End UTC = Last Sunday 23:59:59 +08:00 = Last Sunday 15:59:59 UTC + +Use bash to compute the ISO 8601 date boundaries in UTC for use in API queries: + +```bash +# Compute Asia/Shanghai previous-week boundaries in UTC +python3 -c " +from datetime import datetime, timedelta, timezone + +UTC8 = timezone(timedelta(hours=8)) +now_cst = datetime.now(UTC8) +# Monday of current week in CST +current_monday_cst = now_cst - timedelta(days=now_cst.weekday()) +current_monday_cst = current_monday_cst.replace(hour=0, minute=0, second=0, microsecond=0) +# Previous week +start_cst = current_monday_cst - timedelta(weeks=1) +end_cst = current_monday_cst - timedelta(seconds=1) + +start_utc = start_cst.astimezone(timezone.utc) +end_utc = end_cst.astimezone(timezone.utc) + +start_display = start_cst.strftime('%Y-%m-%d') +end_display = end_cst.strftime('%Y-%m-%d') + +print(f'START_UTC={start_utc.strftime(\"%Y-%m-%dT%H:%M:%SZ\")}') +print(f'END_UTC={end_utc.strftime(\"%Y-%m-%dT%H:%M:%SZ\")}') +print(f'WEEK_START_DISPLAY={start_display}') +print(f'WEEK_END_DISPLAY={end_display}') +" +``` + +### Data Collection Steps + +Using GitHub tools, collect the following data. For each query, handle errors gracefully — if a request fails, note the failure and continue with partial data. + +#### 1. Pull Requests + +Query PRs in `taosdata/TDengine` that were created, merged, closed, or still open within the reporting window: + +- **Opened** during the window: `repo:taosdata/TDengine is:pr created:START..END` +- **Merged** during the window: `repo:taosdata/TDengine is:pr is:merged merged:START..END` +- **Closed without merge** (terminated) during the window: `repo:taosdata/TDengine is:pr is:closed is:unmerged closed:START..END` +- **Still open** as of now: From the PRs opened during the window (`created:START..END`), select those whose state is still `open` as of END + +For each PR, collect: + +- Number, title, author (login) +- State (open / merged / closed) +- Created date, merged date or closed date +- HTML URL +- Head branch name (source branch) + +#### 2. Issues + +Query issues in `taosdata/TDengine` within the reporting window: + +- **Created** during the window: `repo:taosdata/TDengine is:issue created:START..END` +- **Closed** during the window: `repo:taosdata/TDengine is:issue is:closed closed:START..END` +- **Still open** created during window: filter from created list where state is open + +For each issue, collect: + +- Number, title, author (login) +- State (open / closed) +- Created date, closed date (if applicable) +- Labels, milestone +- HTML URL + +#### 3. Branch Hygiene + +- **Branches list**: Retrieve all branches in `taosdata/TDengine` (use pagination as needed). Exclude any branch matching `release/*` — do not list, count, or mention them. +- For **leftover branches**: Cross-reference branches that still exist with PRs that were **closed without merge** during the window. If the PR's head branch still exists in the repository, flag it. +- For **stale branches**: Identify non-`release/*` branches with no commits for ≥ 90 days. + +### Report Format + +After collecting data, compose the report in English using the following structure. Use markdown headings, bullet lists, and tables where appropriate. All dates should be displayed in Asia/Shanghai (CST) time in the format `YYYY-MM-DD`. + +--- + +**Title**: `Weekly Report (WEEK_START_DISPLAY ~ WEEK_END_DISPLAY)` + +--- + +```markdown +# Weekly Report (WEEK_START_DISPLAY ~ WEEK_END_DISPLAY) + +## 1. Highlights + +> _Top-level summary of the week._ + +- **Merged PRs**: X PRs merged (highlight the most significant by title if any stand out) +- **New Issues**: Y issues opened; Z closed +- **Risks / Blockers**: Note any PRs with "blocker", "critical", or "bug" labels still open; note if key PRs have been open with no activity for > 5 days +- **Branch Hygiene Warning**: N leftover branch(es) from terminated PRs detected (see Section 4) + +--- + +## 2. Pull Requests + +### Summary + +| Category | Count | +|---|---| +| Opened | X | +| Merged | X | +| Closed without merge | X | +| Still open | X | + +### Opened PRs + +| # | Title | Author | Status | Created | Closed/Merged | Branch | Link | +|---|---|---|---|---|---|---|---| +| ... | ... | ... | ... | ... | ... | ... | ... | + +### Merged PRs + +_(Same table columns as above)_ + +### Closed without Merge (Terminated) + +_(Same table columns as above)_ + +### Still Open + +_(Same table columns as above — include PRs from the window still open)_ + +--- + +## 3. Issues + +### Summary + +| Category | Count | +|---|---| +| Created | X | +| Closed | X | +| Still open | X | + +### Created + +| # | Title | Author | Status | Created | Closed | Labels | Milestone | Link | +|---|---|---|---|---|---|---|---|---| +| ... | ... | ... | ... | ... | ... | ... | ... | ... | + +### Closed + +_(Same table columns as above)_ + +--- + +## 4. Branch Hygiene + +> ⚠️ This section is read-only reporting. **No cleanup actions are triggered or recommended as automated steps.** +> Branches matching `release/*` are excluded from all subsections below. + +### 4A. Leftover Branches from Terminated PRs ⚠️ Critical + +Branches that still exist in the repository but whose PR was closed without merge: + +| PR | PR Title | Branch | Last Commit | Warning | +|---|---|---|---|---| +| #N | ... | branch-name | YYYY-MM-DD | ⚠️ Uncleaned branch — recommend maintainer review | + +_(If none: "✅ No leftover branches from terminated PRs detected.")_ + +### 4B. New Branches + +_(List branches created during the reporting window if creation time is reliably available. +If not reliably determinable from the API, state: "Branch creation timestamps are not directly available via the GitHub Branches API; this subsection is skipped.")_ + +### 4C. Stale / Inactive Branches (Suggested Review) + +> This is a best-effort inventory. No automated deletion is triggered. + +Branches with no commit activity for ≥ 90 days (excluding `release/*`): + +| Branch | Last Commit Date | Days Inactive | +|---|---|---| +| ... | ... | ... | + +_(If none: "✅ No stale branches detected beyond the 90-day threshold.")_ + +--- + +## 5. Errors / Missing Data + +_(List any API call failures, rate-limit hits, or data that could not be fetched. If all data was retrieved successfully, write "✅ All data retrieved successfully.")_ +``` + +--- + +### Posting the Report + +After composing the report: + +1. Set the discussion title to: `Weekly Report (WEEK_START_DISPLAY ~ WEEK_END_DISPLAY)` +2. Post the report as a new GitHub Discussion in the repository using the `create-discussion` safe output. +3. Use the **"General"** category for the discussion (or the closest available category; do not create a new category — use whichever category exists that is most appropriate for announcements or general updates). + +### Strict Read-Only Constraints + +- Do **NOT** comment on, edit, close, label, assign, reopen, or merge any PR or Issue. +- Do **NOT** create, rename, or delete any branch. +- Do **NOT** create any new Issue. +- The only write operation you may perform is posting exactly one GitHub Discussion. +- If you encounter any limitation or error, document it in the "Errors / Missing Data" section and still post the discussion with partial data. diff --git a/cmake/external.cmake b/cmake/external.cmake index 91d86206cf78..b61ce441b016 100644 --- a/cmake/external.cmake +++ b/cmake/external.cmake @@ -134,6 +134,13 @@ macro(INIT_EXT name) # { target_link_directories(${tgt} PUBLIC "${BREW_PREFIX}/lib") endif() endif() + + if(${TD_WINDOWS}) + if("z${name}" STREQUAL "zext_curl") + target_link_libraries(${tgt} PRIVATE crypt32 wldap32 normaliz secur32 bcrypt) + endif() + endif() + add_definitions(-D_${name}) endmacro() # } endmacro() # } @@ -867,21 +874,53 @@ if(NOT ${TD_WINDOWS}) # { endif(NOT ${TD_WINDOWS}) # } # libcurl -if(NOT ${TD_WINDOWS}) # { - if(${TD_LINUX}) - set(ext_curl_static libcurl.a) - set(_c_flags_list -fPIC) - elseif(${TD_DARWIN}) - set(ext_curl_static libcurl.a) - set(_c_flags_list) - endif() - INIT_EXT(ext_curl - INC_DIR include - LIB lib/${ext_curl_static} - # currently: tqStreamNotify.c uses curl_ws_send, but CURL4_OPENSSL exports curl_easy_send - # libcurl4-openssl-dev on ubuntu 22.04 is too old - # CHK_NAME CURL4_OPENSSL +if(${TD_LINUX}) + set(ext_curl_static libcurl.a) + set(_c_flags_list -fPIC) +elseif(${TD_DARWIN}) + set(ext_curl_static libcurl.a) + set(_c_flags_list) +elseif(${TD_WINDOWS}) + set(ext_curl_static libcurl$<$:-d>.lib) + set(_c_flags_list) +endif() + +INIT_EXT(ext_curl + INC_DIR include + LIB lib/${ext_curl_static} + # currently: tqStreamNotify.c uses curl_ws_send, but CURL4_OPENSSL exports curl_easy_send + # libcurl4-openssl-dev on ubuntu 22.04 is too old + # CHK_NAME CURL4_OPENSSL +) + +if(${TD_WINDOWS}) + # URL https://github.com/curl/curl/releases/download/curl-8_2_1/curl-8.2.1.tar.gz + # URL_HASH MD5=b25588a43556068be05e1624e0e74d41 + get_from_local_if_exists("https://github.com/curl/curl/releases/download/curl-8_2_1/curl-8.2.1.tar.gz") + ExternalProject_Add(ext_curl + URL ${_url} + URL_HASH MD5=b25588a43556068be05e1624e0e74d41 + PREFIX "${_base}" + CMAKE_ARGS -DCMAKE_BUILD_TYPE:STRING=${TD_CONFIG_NAME} + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:STRING=${_ins} + CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR:PATH=lib + CMAKE_ARGS -DBUILD_SHARED_LIBS:BOOL=OFF + CMAKE_ARGS -DBUILD_TESTING:BOOL=OFF + CMAKE_ARGS -DBUILD_CURL_EXE:BOOL=OFF + CMAKE_ARGS -DENABLE_WEBSOCKETS:BOOL=ON + CMAKE_ARGS -DCURL_USE_SCHANNEL:BOOL=ON + CMAKE_ARGS -DCURL_USE_OPENSSL:BOOL=OFF + CMAKE_ARGS -DCURL_ZLIB:BOOL=OFF + CMAKE_ARGS -DCURL_DISABLE_LDAP:BOOL=ON + CMAKE_ARGS -DCURL_DISABLE_LDAPS:BOOL=ON + BUILD_COMMAND + COMMAND "${CMAKE_COMMAND}" --build . --config "${TD_CONFIG_NAME}" + INSTALL_COMMAND + COMMAND "${CMAKE_COMMAND}" --install . --config "${TD_CONFIG_NAME}" --prefix "${_ins}" + EXCLUDE_FROM_ALL TRUE + VERBATIM ) +else() string(JOIN " " _c_flags ${_c_flags_list}) # URL https://github.com/curl/curl/releases/download/curl-8_2_1/curl-8.2.1.tar.gz # URL_HASH MD5=b25588a43556068be05e1624e0e74d41 @@ -911,8 +950,8 @@ if(NOT ${TD_WINDOWS}) # { EXCLUDE_FROM_ALL TRUE VERBATIM ) - add_dependencies(build_externals ext_curl) # this is for github workflow in cache-miss step. -endif(NOT ${TD_WINDOWS}) # } +endif() +add_dependencies(build_externals ext_curl) # this is for github workflow in cache-miss step. # geos if(${BUILD_GEOS}) # { diff --git a/docs/en/06-advanced/05-data-in/22-pspace.md b/docs/en/06-advanced/05-data-in/22-pspace.md new file mode 100644 index 000000000000..12710b10cb70 --- /dev/null +++ b/docs/en/06-advanced/05-data-in/22-pspace.md @@ -0,0 +1,101 @@ +--- +title: "pSpace" +sidebar_label: "pSpace" +--- + +This section describes how to create data migration/data synchronization tasks through the Explorer UI to migrate/synchronize data from pSpace to the current TDengine TSDB cluster. + +## Feature Overview + +TDengine TSDB can efficiently read data from pSpace and write it to TDengine TSDB for historical data migration or real-time data synchronization. + +## Create a Task + +### 1. Add a New Data Source + +On the Data In page, click **+ Add Data Source** to enter the Add Data Source page. + +![pspace-en-01.png](../../assets/Common-en00-EnterDataSourcePage.png) + +### 2. Configure Basic Information + +In **Name**, enter a task name, for example: "test_pspace". + +In the **Type** drop-down list, select **pSpace**. + +**Proxy** is optional. If needed, select a proxy from the drop-down list, or click **+ Create New Proxy** on the right first. + +In the **Target Database** drop-down list, select a target database, or click **+ Create Database** on the right first. + +![pspace-en-02.png](../../assets/pspace-02.png) + +### 3. Configure Connection and Authentication Information + +In the **Connection Configuration** section, fill in **Server Address** and **Server Port**. + +In the **Authentication** section, fill in **Username** and **Password**. + +Click **Connectivity Check** to verify whether the data source is available. + +![pspace-en-03.png](../../assets/pspace-03.png) + +### 4. Configure Data Points + +#### 4.1. Select Data Points + +In **Data Points**, configure the following items: + +1. **Root Node (root)**: The root node to start traversal from. Enter the LongName of the root node. For example: `\Beijing\Chaoyang\Wangjing` means traversal starts from `\Beijing\Chaoyang\Wangjing` and proceeds downward. By default, traversal starts from the root node. +2. **Data Point Name (point_name_pattern)**: Supports filtering by the LongName of data points. For example: `\Beijing\Chaoyang\Wangjing\temperature-*` means all data points under `\Beijing\Chaoyang\Wangjing` whose names start with "temperature-". +3. **Super Table Name (super_table_expression)**: Specifies the super table name for writing data points. Supports the `{type}` placeholder. Example: `pspace_{type}`. +4. **Table Name (child_table_expression)**: Specifies the subtable name for writing data points. Supports the `{point_id}` placeholder. Example: `t_{point_id}`. +5. **Timestamp Column (table_primary_key)**: Selects the source of the primary timestamp in the target table. Available values are `original_ts`, `request_ts`, and `received_ts`. +6. **Timestamp Column Name (table_primary_key_alias)**: Specifies the timestamp column name in the target table. Default is `ts`. +7. **Value Column Name (value_col)**: Specifies the column name for collected values in the target table. Default is `val`. +8. **Value Transform (value_transform)**: Applies an expression transform to values before writing. Example: `(val-32)/1.8`. +9. **Quality Column Name (quality_col)**: Specifies the data quality column name in the target table. Default is `quality`. +10. **Custom Tags (custom_tags)**: Configures tag mappings written to subtables. Supports static values and dynamic extraction from point attributes (for example, `{LongName}`). + +After configuring **Root Node** and **Data Point Name**, click **View Data Point List** to view matching data points, then continue configuring the remaining mapping rules. + +![pspace-en-04.png](../../assets/pspace-04.png) + +#### 4.2. Upload CSV Configuration File + +In **Upload CSV Configuration File**, click **Download Data Points**, select the required **Root Node** and **Data Point Name**, and a CSV configuration file will be generated and downloaded locally. Modify the generated CSV file as needed and upload it again. + +![pspace-en-05.png](../../assets/pspace-05.png) + +### 5. Configure Collection + +In the **Collection Configuration** section, fill in collection-related parameters. + +pSpace supports three collection modes: Historical Query, Real-time Subscription, and Query Sync. + +- Historical Query: Batch query historical data within a time range. The task ends after the query completes. +- Real-time Subscription: Subscribes to real-time changes of data points and keeps running until canceled. +- Query Sync: Completes historical data migration first, then continuously polls new data at a fixed interval. + +#### 5.1. Historical Query + +Select **Historical Query** mode and configure: Start Time, End Time, and Query Window. + +![pspace-en-06.png](../../assets/pspace-06.png) + +#### 5.2. Real-time Subscription + +Select **Real-time Subscription** mode. No additional parameters are required. + +#### 5.3. Query Sync + +Select **Query Sync** mode and configure: Start Time, Query Window, Out-of-order Tolerance, and Query Interval. + +![pspace-en-07.png](../../assets/pspace-07.png) + +### 6. Configure Advanced Options + +In the **Advanced Options** section, configure other parameters as needed. + +### 7. Complete Creation + +Click **Submit** to complete task creation. After submitting, return to the **Data In** page to view the task status. diff --git a/docs/en/14-reference/02-tools/07-taos-cli.md b/docs/en/14-reference/02-tools/07-taos-cli.md index c3a9e6c1bc48..7ee45448f35d 100644 --- a/docs/en/14-reference/02-tools/07-taos-cli.md +++ b/docs/en/14-reference/02-tools/07-taos-cli.md @@ -46,7 +46,7 @@ You can change the behavior of the TDengine CLI by configuring command line para - If the `-p` parameter is not followed by a password string, the user will be prompted to enter the password, e.g., `taos -u root -p`. - If the `-p` parameter is directly followed by a password string, that password will be used for connection, e.g., `taos -u root -ptaosdata`. - -?, --help: Prints out all command line parameters. -- -s COMMAND: SQL command executed in non-interactive mode. +- -s COMMAND: SQL command executed in non-interactive mode. Use the `-s` parameter to execute SQL non interactively, and exit after execution. This mode is suitable for use in automated scripts. For example, connect to the server h1.taos.com with the following command, and execute the SQL specified by `-s`: @@ -54,7 +54,7 @@ You can change the behavior of the TDengine CLI by configuring command line para taos -h my-server -s "use db; show tables;" ``` -- -c CONFIGDIR: Specify the configuration file directory. +- -c CONFIGDIR: Specify the configuration file directory. In Linux, the default is `/etc/tao`. The default name of the configuration file in this directory is `taos.cfg`. Use the `-c` parameter to change the location where the `taosc` client loads the configuration file. For client configuration parameters, refer to [Client Configuration](../../components/taosc). The following command specifies the `taos.cfg` configuration file under `/root/cfg/` loaded by the `taosc` client. @@ -80,7 +80,6 @@ You can change the behavior of the TDengine CLI by configuring command line para - If the `-q` parameter is not followed by a token string, the user will be prompted to enter the token, e.g., `taos -u root -q`. - If the `-q` parameter is directly followed by a token string, that token will be used for connection, e.g., `taos -u root -q`. - -r: Convert time columns to unsigned 64-bit integer type output (i.e., uint64_t in C language). -- -R: Connect to the server using RESTful mode. - -t: Test the startup status of the server, status same as -k. - -w DISPLAYWIDTH: Client column display width. - -z TIMEZONE: Specifies the timezone, default is the local timezone. diff --git a/docs/en/14-reference/02-tools/09-taosdump.md b/docs/en/14-reference/02-tools/09-taosdump.md index 414a701d020c..5148b652cf68 100644 --- a/docs/en/14-reference/02-tools/09-taosdump.md +++ b/docs/en/14-reference/02-tools/09-taosdump.md @@ -91,7 +91,6 @@ Usage: taosdump [OPTION...] dbname [tbname ...] -C, --cloud=CLOUD_DSN Alias for the -X/--dsn option. -k, --retry-count=VALUE Set the number of retry attempts for connection or query failures. - -R, --restful Use RESTful interface to connect server. -t, --timeout=SECONDS The timeout seconds for websocket to interact. -X, --dsn=DSN The dsn to connect the cloud service. -z, --retry-sleep-ms=VALUE Sleep interval between retries, in milliseconds. diff --git a/docs/en/14-reference/05-connector/30-python.md b/docs/en/14-reference/05-connector/30-python.md index 3cf2a6e785a4..8e9405a9226d 100644 --- a/docs/en/14-reference/05-connector/30-python.md +++ b/docs/en/14-reference/05-connector/30-python.md @@ -445,7 +445,7 @@ The interface for binding parameters of the standard Stmt. #### Establishing Connection - `def connect(*args, **kwargs):` - - **Interface Description**: Establish a connection to taosAdapter. + - **Interface Description**: Creates and returns a database connection object. - **Parameter Description**: - `kwargs`: Provided in the form of a Python dictionary, can be used to set - `user`: Username for the database diff --git a/docs/en/assets/pspace-02.png b/docs/en/assets/pspace-02.png new file mode 100644 index 000000000000..c58206c27083 Binary files /dev/null and b/docs/en/assets/pspace-02.png differ diff --git a/docs/en/assets/pspace-03.png b/docs/en/assets/pspace-03.png new file mode 100644 index 000000000000..2c4c9766ec1f Binary files /dev/null and b/docs/en/assets/pspace-03.png differ diff --git a/docs/en/assets/pspace-04.png b/docs/en/assets/pspace-04.png new file mode 100644 index 000000000000..a25ad0b15c87 Binary files /dev/null and b/docs/en/assets/pspace-04.png differ diff --git a/docs/en/assets/pspace-05.png b/docs/en/assets/pspace-05.png new file mode 100644 index 000000000000..7187a8c4b144 Binary files /dev/null and b/docs/en/assets/pspace-05.png differ diff --git a/docs/en/assets/pspace-06.png b/docs/en/assets/pspace-06.png new file mode 100644 index 000000000000..22b133e1923d Binary files /dev/null and b/docs/en/assets/pspace-06.png differ diff --git a/docs/en/assets/pspace-07.png b/docs/en/assets/pspace-07.png new file mode 100644 index 000000000000..b212cbc4f9b0 Binary files /dev/null and b/docs/en/assets/pspace-07.png differ diff --git a/docs/zh/04-get-started/_03-docker.md b/docs/zh/04-get-started/_03-docker.md index 8abb6ff7c4e3..a2bb21755105 100644 --- a/docs/zh/04-get-started/_03-docker.md +++ b/docs/zh/04-get-started/_03-docker.md @@ -62,4 +62,5 @@ docker exec -it 1. 从列表中下载获得 tar.gz 安装包： - + 2. 进入到安装包所在目录，使用 `tar` 解压安装包； ```bash - tar -zxvf tdengine-tsdb-oss-3.3.8.8-linux-x64.tar.gz + tar -zxvf tdengine-tsdb-oss-3.4.0.9-linux-x64.tar.gz ``` 3. 进入到安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本。 ```bash @@ -58,19 +58,19 @@ install.sh 安装脚本在执行过程中，会通过命令行交互界面询问 1. 从列表中下载获得 deb 安装包： - + 2. 进入到安装包所在目录，执行如下的安装命令： ```bash - sudo dpkg -i tdengine-tsdb-oss-3.3.8.8-linux-x64.deb + sudo dpkg -i tdengine-tsdb-oss-3.4.0.9-linux-x64.deb ``` 1. 从列表中下载获得 rpm 安装包： - + 2. 进入到安装包所在目录，执行如下的安装命令： ```bash - sudo rpm -ivh tdengine-tsdb-oss-3.3.8.8-linux-x64.rpm + sudo rpm -ivh tdengine-tsdb-oss-3.4.0.9-linux-x64.rpm ``` @@ -106,7 +106,7 @@ apt-get 方式只适用于 Debian 或 Ubuntu 系统。 1. 从列表中下载获得最新 Windows 安装程序： - + 2. 运行可执行程序来安装 TDengine。 :::note @@ -122,7 +122,7 @@ apt-get 方式只适用于 Debian 或 Ubuntu 系统。 1. 从列表中下载获得最新 macOS 安装包： - + 2. 运行可执行程序来安装 TDengine。如果安装被阻止，可以右键或者按 Ctrl 点击安装包，选择 `打开`。 diff --git a/docs/zh/14-reference/02-tools/07-taos-cli.md b/docs/zh/14-reference/02-tools/07-taos-cli.md index 09a600dcb218..f7bdd8010ef9 100644 --- a/docs/zh/14-reference/02-tools/07-taos-cli.md +++ b/docs/zh/14-reference/02-tools/07-taos-cli.md @@ -48,7 +48,7 @@ taos> quit - -?, --help：打印出所有命令行参数。 - -s COMMAND：以非交互模式执行的 SQL 命令。 - 使用 `-s` 参数可进行非交互式执行 SQL，执行完成后退出，此模式适合在自动化脚本中使用。 + 使用 `-s` 参数可进行非交互式执行 SQL，执行完成后退出，此模式适合在自动化脚本中使用。 如以下命令连接到服务器 h1.taos.com, 执行 -s 指定的 SQL： ```bash @@ -58,7 +58,7 @@ taos> quit - -c CONFIGDIR：指定配置文件目录。 Linux 环境下默认为 `/etc/taos`，该目录下的配置文件默认名称为 `taos.cfg`。 - 使用 `-c` 参数改变 `taosc` 客户端加载配置文件的位置，客户端配置参数参考 [客户端配置](../../components/taosc)。 + 使用 `-c` 参数改变 `taosc` 客户端加载配置文件的位置，客户端配置参数参考 [客户端配置](../../components/taosc)。 以下命令指定了 `taosc` 客户端加载 `/root/cfg/` 下的 `taos.cfg` 配置文件。 ```bash @@ -84,7 +84,6 @@ taos> quit - 如果 `-q` 参数后面不跟 Token 字符串，则会提示用户输入 Token，如：`taos -u root -q`。 - 如果 `-q` 参数后面直接跟 Token 字符串，则使用该 Token 进行连接，如：`taos -u root -q`。 - -r：将时间列转化为无符号 64 位整数类型输出 (即 C 语言中 uint64_t)。 -- -R：使用 RESTful 模式连接服务端。 - -t：测试服务端启动状态，状态同 -k。 - -w DISPLAYWIDTH：客户端列显示宽度。 - -z TIMEZONE：指定时区，默认为本地时区。 @@ -119,7 +118,7 @@ taos> source ; ### 设置字符列显示宽度 -可以在 TDengine TSDB CLI 里使用如下命令调整字符串类型字段列显示宽度，默认显示宽度为 30 个字符。 +可以在 TDengine TSDB CLI 里使用如下命令调整字符串类型字段列显示宽度，默认显示宽度为 30 个字符。 以下命令设置了显示宽度为 120 个字符： ```sql diff --git a/docs/zh/14-reference/02-tools/09-taosdump.md b/docs/zh/14-reference/02-tools/09-taosdump.md index 8bc3f9796f74..5865d54a5bd8 100644 --- a/docs/zh/14-reference/02-tools/09-taosdump.md +++ b/docs/zh/14-reference/02-tools/09-taosdump.md @@ -89,7 +89,6 @@ Usage: taosdump [OPTION...] dbname [tbname ...] -C, --cloud=CLOUD_DSN Alias for the -X/--dsn option. -k, --retry-count=VALUE Set the number of retry attempts for connection or query failures. - -R, --restful Use RESTful interface to connect server. -t, --timeout=SECONDS The timeout seconds for websocket to interact. -X, --dsn=DSN The dsn to connect the cloud service. -z, --retry-sleep-ms=VALUE Sleep interval between retries, in milliseconds. diff --git a/docs/zh/14-reference/05-connector/30-python.mdx b/docs/zh/14-reference/05-connector/30-python.mdx index 975dc108d46f..20ff44cdf13c 100644 --- a/docs/zh/14-reference/05-connector/30-python.mdx +++ b/docs/zh/14-reference/05-connector/30-python.mdx @@ -445,7 +445,7 @@ TaosResult 对象可以通过循环遍历获取查询到的数据。 #### 建立连接 - `def connect(*args, **kwargs):` - - **接口说明**：建立 taosAdapter 连接。 + - **接口说明**：创建并返回数据库连接对象。 - **参数说明**： - `kwargs`：以 Python 字典的形式提供，可用于设置 - `user`：数据库的用户名 diff --git a/docs/zh/14-reference/05-connector/_linux_install.mdx b/docs/zh/14-reference/05-connector/_linux_install.mdx index 61d06135e4b8..9782539ab9a7 100644 --- a/docs/zh/14-reference/05-connector/_linux_install.mdx +++ b/docs/zh/14-reference/05-connector/_linux_install.mdx @@ -2,11 +2,11 @@ import PkgListV37 from "/components/PkgListV37"; 1. 下载客户端安装包 - + 2. 解压缩软件包 - 将软件包放置在当前用户可读写的任意目录下，然后执行下面的命令：`tar -xzvf tdengine-tsdb-oss-client-3.3.8.8-linux-x64.tar.gz` + 将软件包放置在当前用户可读写的任意目录下，然后执行下面的命令：`tar -xzvf tdengine-tsdb-oss-client-3.4.0.9-linux-x64.tar.gz` 其中 VERSION 需要替换为实际版本的字符串。 3. 执行安装脚本 diff --git a/docs/zh/14-reference/05-connector/_macos_install.mdx b/docs/zh/14-reference/05-connector/_macos_install.mdx index de1f2f829e37..76d4579cac6e 100644 --- a/docs/zh/14-reference/05-connector/_macos_install.mdx +++ b/docs/zh/14-reference/05-connector/_macos_install.mdx @@ -2,7 +2,7 @@ import PkgListV37 from "/components/PkgListV37"; 1. 下载客户端安装包 - + 2. 执行安装程序，按提示选择默认值，完成安装。如果安装被阻止，可以右键或者按 Ctrl 点击安装包，选择 `打开`。 3. 配置 taos.cfg diff --git a/docs/zh/14-reference/05-connector/_windows_install.mdx b/docs/zh/14-reference/05-connector/_windows_install.mdx index 75c796cb4a5a..a5bfe91508ee 100644 --- a/docs/zh/14-reference/05-connector/_windows_install.mdx +++ b/docs/zh/14-reference/05-connector/_windows_install.mdx @@ -2,7 +2,7 @@ import PkgListV37 from "/components/PkgListV37"; 1. 下载客户端安装包 - + 2. 执行安装程序，按提示选择默认值，完成安装 3. 安装路径 diff --git a/docs/zh/27-train-faq/01-faq.md b/docs/zh/27-train-faq/01-faq.md index 7a1ee0e8ea9a..39e1ac7b00bd 100644 --- a/docs/zh/27-train-faq/01-faq.md +++ b/docs/zh/27-train-faq/01-faq.md @@ -390,3 +390,11 @@ TDengine TSDB 3.3.5.0 及以上的版本，有些用户可能会遇到一个问 这种情况通常不会产生乱序，首先我们来解释下 TDengine TSDB 中乱序是指什么？TDengine TSDB 中的乱序是指从时间戳为 0 开始按数据库设置的 Duration 参数（默认是 10 天）切割成时间窗口，在每个时间窗口中写入的数据不按顺序时间写入导致的现象为乱序现象，只要保证同一窗口是顺序写入的，即使窗口之间写入并非顺序，也不会产生乱序。 再看上面场景，补旧数据和新数据同时写入，新旧数据之间一般会存在较大距离，不会落在同一窗口中，只要保证新老数据都是顺序写的，即不会产生乱序现象。 + +### 40 如果从 Docker Hub 拉取 TDengine 镜像失败，应该如何解决？{#docker-hub-failure} + +如果无法正常访问 Docker Hub 官方仓库（hub.docker.com），可以尝试以下方法解决： + +- 检查网络连接是否正常。 +- 从 [TDengine 下载中心](https://www.taosdata.com/download-center) 下载镜像文件，然后使用 `docker load` 命令加载镜像，用法详见下载页面的使用说明。 +- 尝试使用其他镜像源，例如：[CNIX Internal Container Registry Mirror](https://m.ixdev.cn/)，该镜像源与涛思数据无关，如果失效，请尝试更换网络上的其它镜像源，用法详见镜像源的使用说明。 diff --git a/include/libs/new-stream/dataSink.h b/include/libs/new-stream/dataSink.h index 4fe80a764f95..07a58767dc19 100644 --- a/include/libs/new-stream/dataSink.h +++ b/include/libs/new-stream/dataSink.h @@ -12,34 +12,15 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -// 一个 group 无法避免有多个数据块在同一个文件中 -// 需要有标记来记录已经 get 完成 但是没有开始 put 的 group， 这些是优先要淘汰的内存 -// 上述内存淘汰完之后，内存还不足，代表这需要淘汰正在写的和正在读的的内存块了。 - -// 1. 非 sliding, 读取后立即释放内存，一个单独 list 存放 buff 信息，每个 buff 10 M，当前 buff 写满开始下一个 -// buff，总大小超过则按时间升序将部分 buff 块写入文件，释放内存； 每次读取数据，会同时读取文件和 buff -// 中的数据，读取完成后会全部释放。内存和文件中数据会有一个 groupid 标签，目前非 sliding 模式同时只会存在一个 groupid -// 的数据，如果发现 有多个 groupid 数据在尝试同时写入，报错。 非 slidng -// 模式，内存中需保存管理信息(同时保存的group信息理论上只有一个)： groupid，usedMemSize blocksInMem: address, capacity, -// size + (windows: startTime, endTime, dataLen) + dataBlock serialized data blocksInFile: offset, capacity, size -// 文件中保存信息：(windows: startTime, endTime, dataLen) + dataBlock serialized data -// 2. sliding 模式：每次写入数据，写入独立的内存段，内存不足时触发淘汰机制，将某些 group 的数据从内存淘汰，写入文件 -// 内存不足时，先淘汰占用内存最多并且当前不活跃的 group ，淘汰后内存仍不足时，淘汰当前 group -// 的数据（理论上同一时间只有一个活跃的 group ） 当前不活跃的 group: 定义为 getdata 完成，没有进行新一轮的 putdata 的 -// group 每个 task 首次需要写入文件时，计算要写入的 group data 的长度（考虑 +20% 做缓冲，最多 + 1M ），作为文件中每个 -// group block 的大小，后续写入的 group block 均使用改大小，当大小不足时，申请新的 block 淘汰内存时，linux 下使用 writev -// 来写入文件, 可以将分散的内存连续一次写入，其他平台不支持，可以先逐个写入内存后再写入文件，后续优化 从 group -// 读取数据时，文件/内存中可能均有数据，先读文件后读取内存； -// 每次读取数据后，检查使用内存情况，如果需要触发淘汰机制，在一个新线程中进行内存淘汰。读取数据时的内存淘汰触发条件要比写入时内存淘汰更敏感，设置一个略小的值（例如比写入时限制小 -// 10 M），尽量提前触发，避免在读取时需要阻塞读取进行释放 sliding 模式，内存中需保存的管理信息（多个 -// group）：groupid，usedMemSize blocksInMem: address, capacity, size + (windows: startTime, endTime, dataLen) + -// dataBlock serialized data blocksInFile: groupOffset, dataStartOffset, dataLen， capacity(same in a task) -// 文件中保存信息：(windows: startTime, endTime, dataLen) + dataBlock serialized data -// 每个块写入的 window 保存：list, endtime, len +// Notes: +// - A group may map to multiple data blocks in one file. +// - Memory blocks are spilled to file when memory pressure is high. +// - For sliding mode, both in-memory and on-disk blocks may coexist and are read in order. #ifndef TDENGINE_DATA_SINK_H #define TDENGINE_DATA_SINK_H +#include #include #include "tarray.h" #include "tcommon.h" @@ -113,14 +94,14 @@ typedef struct SAlignBlocksInMem { int64_t capacity; int64_t dataLen; int32_t nWindow; - // void* address; // 后续地址存放的内容为 SSlidingWindowInMem 数组序列化后的内容 + // trailing bytes store serialized SSlidingWindowInMem items } SAlignBlocksInMem; typedef struct SBlocksInfoFile { int64_t groupOffset; // offset in file int64_t dataLen; int64_t capacity; // size in file - // SSlidingWindowInMem *windowDataInFile; // array SSlidingWindowInMem 实际数据，反序列化保存至文件 + // serialized SSlidingWindowInMem payload is stored in file at groupOffset } SBlocksInfoFile; typedef struct STaskDSMgr { @@ -143,7 +124,7 @@ typedef struct SSlidingTaskDSMgr { int64_t taskId; int64_t sessionId; // sessionId is used to distinguish different sessions in the same task int32_t tsSlotId; - int64_t capacity; // group 在文件中的每个 block 块大小 + int64_t capacity; // per-group block size in file SHashObj* pSlidingGrpList; // hash SDataSinkFileMgr* pFileMgr; } SSlidingTaskDSMgr; @@ -209,77 +190,39 @@ typedef struct SSlidingGrpMemList { } SSlidingGrpMemList; extern SSlidingGrpMemList g_slidigGrpMemList; -//----------------- ************************************** -----------------// -//----------------- 以下函数 DataSink 对外提供接口 -----------------// -//----------------- ************************************** -----------------// +// ----------------- External DataSink APIs ----------------- -// @brief 创建一个数据缓存 -// @param cleanMode 清理模式，具体含义如下: -// 1. 一行数据只会被读取一次，所以读取结束后可以立刻被清理 -// 2. 一行数据可能被读取多次，所以等到下次读取时，才清理时间范围之前的数据 +// Create a data sink cache. int32_t initStreamDataCache(int64_t streamId, int64_t taskId, int64_t sessionId, int32_t cleanMode, int32_t tsSlotId, void** ppCache); -// @brief 清理数据缓存，包括缓存的数据文件和内存 +// Destroy a data sink cache and associated resources. void destroyStreamDataCache(void* pCache); -// @brief 向数据缓存中添加数据 -// @param pCache 数据缓存,使用 StreamDataCacheInit 创建 -// @param wstart 当前数据集的起始时间戳 -// @param wend 当前数据集的结束时间戳 -// @param pBlock 数据块 -// @param startIndex 数据块的起始索引 -// @param endIndex 数据块的结束索引 -// @note -// 1. 起始索引和结束索引是数据块数据的索引范围,从0开始计数 -// 2. 可能会对同一个 {groupId, tableId, wstart} 进行多次调用,添加多个数据块,调用者保证这些数据是严格时间有序的 +// Append data to cache. int32_t putStreamDataCache(void* pCache, int64_t groupId, TSKEY wstart, TSKEY wend, SSDataBlock* pBlock, int32_t startIndex, int32_t endIndex); -// @brief 向数据缓存中添加数据 -// @note 和 putStreamDataCache 区别是： -// 1. 会移交 pBlock 的所有权 -// 2. 如果返回 success，pBlock 的内存释放由 Cache Sink 负责； -// 3. 如果返回 error，pBlock 的内存释放由调用者负责； +// Append data by transferring ownership of pBlock to cache on success. int32_t moveStreamDataCache(void* pCache, int64_t groupId, TSKEY wstart, TSKEY wend, SSDataBlock* pBlock); -// @brief 从数据缓存中读取数据 -// @param pCache 数据缓存,使用 StreamDataCacheInit 创建 -// @param groupId 数据的分组ID,实际上是 "__" 格式的字符串 -// @param start 读取数据的起始时间戳 -// @param end 读取数据的结束时间戳 -// @param pIter 迭代器,用于遍历数据块 -// @note -// 1. 没有数据时，把 pIter 置为 NULL -// 2. 符合筛选条件的数据可能包含多个数据块,由 pIter 负责迭代遍历 -// 3. 这里没有区分 tableId,后续对 pIter 遍历的结果应该是按照 tableId 有序,内部再以时间戳有序 -// 4. start, end 一定是对齐到数据集边界的，即 [start, end] 包含若干个数据集，但不会包含任意数据集的一部分 +// Get an iterator for reading cached blocks by group and time range. int32_t getStreamDataCache(void* pCache, int64_t groupId, TSKEY start, TSKEY end, void** pIter); -// @brief 遍历获取所有符合条件的数据块 -// @param pIter 迭代器,用于遍历数据块 -// @param ppBlock 用于指向结果数据块，调用者不会释放指向的内存 -// @note -// 1. 需要把 pIter 指向迭代器的下一位，如果没有数据了，返回 NULL +// Read next block from iterator. int32_t getNextStreamDataCache(void** pIter, SSDataBlock** ppBlock); -// @brief 清理数据缓存中的数据 -// @param pCache 数据缓存,使用 StreamDataCacheInit 创建 +// Clean cache data for a group. int32_t cleanStreamDataCache(void* pCache, int64_t groupId); -// @brief 取消对读取结果的遍历 -// @note -// 1. 调用者在使用 pIter 遍历数据时，可以用这个接口提前结束遍历，通常用于异常情况 -// 2. 取消数据遍历意味着读取操作结束，会触发底层 Cache Sink 的数据清理 +// Cancel iterator traversal early. void cancelStreamDataCacheIterate(void** pIter); -// @brief 释放 DataSink 相关所有资源 +// Destroy all DataSink manager resources. void destroyDataSinkMgr(); void setDataSinkMaxMemSize(int64_t maxMemSize); -//----------------- ************************************** -----------------// -//----------------- 以下函数 DataSink 内部调用，不提供于其他模块 -----------------// -//----------------- ************************************** -----------------// +// ----------------- Internal DataSink APIs ----------------- int32_t initDataSinkFileDir(); int32_t initStreamDataSink(); int32_t checkAndMoveMemCache(bool forWrite); @@ -300,12 +243,12 @@ int32_t buildAlignWindowInMemBlock(SAlignGrpMgr* pAlignGrpMgr, SSDataBlock* pBlo int32_t buildMoveAlignWindowInMem(SAlignGrpMgr* pAlignGrpMgr, SSDataBlock* pBlock, int32_t tsColSlotId, TSKEY wstart, TSKEY wend); -// @brief 读取数据从内存 +// Read data from in-memory cache. int32_t readDataFromMem(SResultIter* pResult, SSDataBlock** ppBlock, bool* finished); int32_t readDataFromFile(SResultIter* pResult, SSDataBlock** ppBlock, int32_t tsColSlotId); -// @brief 从内存查找下一组数据位置 -// return true: 需要继续查看文件, false: 不需要继续查看文件 +// Find next iterator position from memory cache. +// return true: continue to file side, false: no file-side iteration needed bool setNextIteratorFromMem(SResultIter** ppResult); bool setNextIteratorFromFile(SResultIter** ppResult); int32_t createDataResult(void** ppResult); diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 8b7149224052..6789d29e7012 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -102,7 +102,8 @@ typedef struct SColumnNode { uint8_t flags; struct { uint8_t hasMask : 1; - uint8_t reserve : 7; + uint8_t appendByPrivCond : 1; + uint8_t reserve : 6; }; }; char refDbName[TSDB_DB_NAME_LEN]; diff --git a/include/util/tcurl.h b/include/util/tcurl.h index 55c4d6987ff9..adeae5266342 100644 --- a/include/util/tcurl.h +++ b/include/util/tcurl.h @@ -19,7 +19,10 @@ #include "os.h" #include "taos.h" -#ifndef WINDOWS +#if defined(WINDOWS) +#define CURL_STATICLIB +#endif + #include "curl/curl.h" typedef struct SCurl { @@ -33,8 +36,6 @@ int32_t tcurlGetConnection(const char* url, SCURL** pConn); int32_t tcurlSend(SCURL* curl, const void* buffer, size_t buflen, size_t* sent, curl_off_t fragsize, unsigned int flags); -#endif - void closeThreadNotificationConn(); #endif diff --git a/include/util/tpriv.h b/include/util/tpriv.h index 477a640875d7..16942f80fdb6 100644 --- a/include/util/tpriv.h +++ b/include/util/tpriv.h @@ -52,7 +52,7 @@ extern "C" { #define TSDB_WORD_VARIABLES "variables" #define TSDB_WORD_INFORMATION "information" -#define PRIV_INFO_TABLE_VERSION 3 +#define PRIV_INFO_TABLE_VERSION 4 // N.B. increase this version for any update of privInfoTable typedef enum { PRIV_TYPE_UNKNOWN = -1, // ==================== Common Privilege ==================== diff --git a/packaging/delete_ref_lock.py b/packaging/delete_ref_lock.py index 7200246a8a92..e4cd742a6500 100644 --- a/packaging/delete_ref_lock.py +++ b/packaging/delete_ref_lock.py @@ -1,80 +1,142 @@ import subprocess import re +import os +from typing import Optional +from abc import ABC, abstractmethod -def git_fetch(): - result = subprocess.run(['git', 'fetch'], capture_output=True, text=True) - return result -def git_prune(): - # git remote prune origin - print("git remote prune origin") - result = subprocess.run(['git', 'remote', 'prune', 'origin'], capture_output=True, text=True) - return result - -def parse_branch_name_type1(error_output): +class RefLockErrorHandler(ABC): + """抽象基类，定义处理接口""" + + @abstractmethod + def match(self, error_output: str) -> bool: + pass + + @abstractmethod + def parse_branch(self, error_output: str) -> Optional[str]: + pass + + def handle(self, error_output: str): + branch = self.parse_branch(error_output) + if branch: + print(f"Detected error, attempting to delete ref for branch: {branch}") + self.delete_ref(branch) + else: + print("Error parsing branch name.") + + def delete_ref(self, branch_name: str): + try: + subprocess.run(["git", "update-ref", "-d", branch_name], check=True) + except subprocess.CalledProcessError as e: + print(f"git update-ref failed: {e}") + lock_files = [f".git/{branch_name}.lock", f".git/logs/{branch_name}.lock"] + for lock_file in lock_files: + if os.path.exists(lock_file): + try: + os.remove(lock_file) + print(f"Removed lock file: {lock_file}") + except Exception as ex: + print(f"Failed to remove lock file {lock_file}: {ex}") + # retry + try: + subprocess.run(["git", "update-ref", "-d", branch_name], check=True) + except subprocess.CalledProcessError as e2: + print(f"Still failed to delete ref after removing lock: {e2}") + + +class Type1Handler(RefLockErrorHandler): # error: cannot lock ref 'refs/remotes/origin/fix/3.0/TD-32817': is at 7af5 but expected eaba # match the branch name before ‘is at’ with a regular expression - match = re.search(r"error: cannot lock ref '(refs/remotes/origin/[^']+)': is at", error_output) - if match: - return match.group(1) - return None + def match(self, error_output: str) -> bool: + return "is at" in error_output and "but expected" in error_output + + def parse_branch(self, error_output: str) -> str: + # 匹配 cannot lock ref 部分，兼容中英文 + match = re.search( + r"cannot lock ref '(refs/remotes/origin/[^']+)': is at", error_output + ) + return match.group(1) if match else None -def parse_branch_name_type2(error_output): + +class Type2Handler(RefLockErrorHandler): # match the branch name before ‘exists; cannot create’ with a regular expression - match = re.search(r"'(refs/remotes/origin/[^']+)' exists;", error_output) - if match: - return match.group(1) - return None + def match(self, error_output: str) -> bool: + return "exists; cannot create" in error_output + + def parse_branch(self, error_output: str) -> str: + match = re.search(r"'(refs/remotes/origin/[^']+)' exists;", error_output) + return match.group(1) if match else None -# parse branch name from error output of git remote prune origin -def parse_branch_name_type3(error_output): + +class Type3Handler(RefLockErrorHandler): # match the branch name before the first single quote before 'Unable to' with a regular expression # git error: could not delete references: cannot lock ref 'refs/remotes/origin/test/3.0/TS-4893': Unable to create 'D:/workspace/main/TDinternal/community/.git/refs/remotes/origin/test/3.0/TS-4893.lock': File exists - match = re.search(r"references: cannot lock ref '(refs/remotes/origin/[^']+)': Unable to", error_output) - if match: - return match.group(1) - return None + def match(self, error_output: str) -> bool: + return "Unable to create" in error_output and "File exists" in error_output + + def parse_branch(self, error_output: str) -> str: + match = re.search( + r"(?:error|references): cannot lock ref '(refs/remotes/origin/[^']+)': Unable to", + error_output, + ) + return match.group(1) if match else None + + +class RefLockErrorHandlerFactory: + """工厂类，返回合适的处理器""" + handlers = [Type1Handler(), Type2Handler(), Type3Handler()] + + @classmethod + def get_handler(cls, error_output: str): + for handler in cls.handlers: + if handler.match(error_output): + return handler + return None -# execute git update-ref -d to delete the ref -def git_update_ref(branch_name): - if branch_name: - subprocess.run(['git', 'update-ref', '-d', f'{branch_name}'], check=True) -# parse error type and execute corresponding repair operation def handle_error(error_output): - error_types = [ - ("is at", "but expected", parse_branch_name_type1, "type 1"), - ("exists; cannot create", None, parse_branch_name_type2, "type 2"), - ("Unable to create", "File exists", parse_branch_name_type3, "type 3") - ] - - for error_type in error_types: - if error_type[0] in error_output and (error_type[1] is None or error_type[1] in error_output): - branch_name = error_type[2](error_output) - if branch_name: - print(f"Detected error {error_type[3]}, attempting to delete ref for branch: {branch_name}") - git_update_ref(branch_name) - else: - print(f"Error parsing branch name for {error_type[3]}.") - break + handler = RefLockErrorHandlerFactory.get_handler(error_output) + if handler: + handler.handle(error_output) + else: + print("No handler found for this error.") + + +def git_fetch(): + print("Running: git fetch") + result = subprocess.run(["git", "fetch"], capture_output=True, text=True) + if result.returncode != 0: + print("git fetch failed:") + print(result.stderr) + else: + print("git fetch successful.") + return result + + +def git_prune(): + print("Running: git remote prune origin") + result = subprocess.run( + ["git", "remote", "prune", "origin"], capture_output=True, text=True + ) + if result.returncode != 0: + print("git remote prune origin failed:") + print(result.stderr) + else: + print("git remote prune origin successful.") + return result + def main(): fetch_result = git_fetch() - if fetch_result.returncode != 0: - error_output = fetch_result.stderr - handle_error(error_output) - else: - print("Git fetch successful.") + if fetch_result.returncode != 0: + handle_error(fetch_result.stderr) + return prune_result = git_prune() - print(prune_result.returncode) - if prune_result.returncode != 0: - error_output = prune_result.stderr - print(error_output) - handle_error(error_output) - else: - print("Git prune successful.") + if prune_result.returncode != 0: + handle_error(prune_result.stderr) + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 8a9f6d49257b..1074c13e03f6 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -69,7 +69,7 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC (void)atomic_val_compare_exchange_8(&pAppHbMgr->connHbFlag, 1, 2); _return: - taosArrayDestroy(batchRsp.pArray); + tFreeSUserAuthBatchRsp(&batchRsp); return code; } diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 47b3238cdb78..970f745b2408 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -83,7 +83,7 @@ if(${TD_WINDOWS}) PUBLIC crypt INTERFACE api ) - + DEP_ext_curl(common) else() target_link_libraries( common diff --git a/source/common/src/tpriv.c b/source/common/src/tpriv.c index 7adb090f1eb9..6095dbc68289 100644 --- a/source/common/src/tpriv.c +++ b/source/common/src/tpriv.c @@ -37,6 +37,7 @@ static const SPrivObjInfo __privObjInfo[] = { */ #define SYS_ADMIN_BASIC_ROLES (T_ROLE_SYSDBA | T_ROLE_SYSSEC | T_ROLE_SYSAUDIT) +#define SYS_ADMIN_CORE_ROLES (T_ROLE_SYSDBA | T_ROLE_SYSSEC) #define SYS_ADMIN_INFO1_ROLES (SYS_ADMIN_BASIC_ROLES | T_ROLE_SYSINFO_1) #define SYS_ADMIN_INFO_ROLES (SYS_ADMIN_BASIC_ROLES | T_ROLE_SYSINFO_0 | T_ROLE_SYSINFO_1) #define SYS_ADMIN_EXT_ROLES (T_ROLE_SYSINFO_0 | T_ROLE_SYSINFO_1 | T_ROLE_SYSAUDIT_LOG) @@ -82,8 +83,8 @@ static SPrivInfo privInfoTable[] = { {PRIV_USER_SET_AUDIT, PRIV_CATEGORY_SYSTEM, 0, 0, T_ROLE_SYSDBA | T_ROLE_SYSAUDIT, 0, "", "SET USER AUDIT INFORMATION"}, {PRIV_USER_SET_BASIC, PRIV_CATEGORY_SYSTEM, 0, 0, T_ROLE_SYSDBA, 0, "", "SET USER BASIC INFORMATION"}, - {PRIV_USER_UNLOCK, PRIV_CATEGORY_SYSTEM, 0, 0, T_ROLE_SYSDBA, 0, "", "UNLOCK USER"}, - {PRIV_USER_LOCK, PRIV_CATEGORY_SYSTEM, 0, 0, T_ROLE_SYSDBA, 0, "", "LOCK USER"}, + {PRIV_USER_UNLOCK, PRIV_CATEGORY_SYSTEM, 0, 0, SYS_ADMIN_CORE_ROLES, 0, "", "UNLOCK USER"}, + {PRIV_USER_LOCK, PRIV_CATEGORY_SYSTEM, 0, 0, SYS_ADMIN_CORE_ROLES, 0, "", "LOCK USER"}, {PRIV_USER_SHOW, PRIV_CATEGORY_SYSTEM, 0, 0, SYS_ADMIN_INFO1_ROLES, 0, "", "SHOW USERS"}, {PRIV_USER_ALTER, PRIV_CATEGORY_SYSTEM, 0, 0, SYS_ADMIN_INFO_ROLES, 0, "", "ALTER USER"}, {PRIV_USER_SHOW_SECURITY, PRIV_CATEGORY_SYSTEM, 0, 0, SYS_ADMIN_BASIC_ROLES, 0, "", "SHOW USERS SECURITY INFORMATION"}, @@ -91,6 +92,8 @@ static SPrivInfo privInfoTable[] = { // Role Management {PRIV_ROLE_CREATE, PRIV_CATEGORY_SYSTEM, 0, 0, T_ROLE_SYSDBA, 0, "", "CREATE ROLE"}, {PRIV_ROLE_DROP, PRIV_CATEGORY_SYSTEM, 0, 0, T_ROLE_SYSDBA, 0, "", "DROP ROLE"}, + {PRIV_ROLE_UNLOCK, PRIV_CATEGORY_SYSTEM, 0, 0, SYS_ADMIN_CORE_ROLES, 0, "", "UNLOCK ROLE"}, + {PRIV_ROLE_LOCK, PRIV_CATEGORY_SYSTEM, 0, 0, SYS_ADMIN_CORE_ROLES, 0, "", "LOCK ROLE"}, {PRIV_ROLE_SHOW, PRIV_CATEGORY_SYSTEM, 0, 0, SYS_ADMIN_INFO1_ROLES, 0, "", "SHOW ROLES"}, // Token Privileges diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index bb1dc324cf84..a471402a12c4 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -1193,6 +1193,7 @@ static int32_t mndRetrieveConsumer(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock * END: sdbRelease(pSdb, pConsumer); sdbCancelFetch(pSdb, pShow->pIter); + mndReleaseUser(pMnode, pOperUser); if (code != 0) { mError("show consumer failed, code:%d", code); return code; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 4387764c5fa8..4abc365db78e 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -3126,6 +3126,8 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, precVstr, false), &lino, _OVER); } else if (i == 15) { TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, statusVstr, false), &lino, _OVER); + } else if (i == 28) { + TAOS_CHECK_GOTO(colDataSetVal(pColInfo, rows, (const char *)&pDb->cfg.keepTimeOffset, false), &lino, _OVER); } else { colDataSetNULL(pColInfo, rows); } diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index f6a459289941..caa5ee215226 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -875,7 +875,7 @@ int32_t vnodeSetElectBaseline(SVnode* pVnode, int32_t ms){ } void vnodeSyncPreClose(SVnode *pVnode) { - vInfo("vgId:%d, sync pre close", pVnode->config.vgId); + vInfo("vgId:%d, vnode sync pre close", pVnode->config.vgId); int32_t code = syncLeaderTransfer(pVnode->sync); if (code) { vError("vgId:%d, failed to transfer leader since %s", pVnode->config.vgId, tstrerror(code)); @@ -894,12 +894,12 @@ void vnodeSyncPreClose(SVnode *pVnode) { } void vnodeSyncPostClose(SVnode *pVnode) { - vInfo("vgId:%d, sync post close", pVnode->config.vgId); + vInfo("vgId:%d, vnode sync post close", pVnode->config.vgId); syncPostStop(pVnode->sync); } void vnodeSyncClose(SVnode *pVnode) { - vInfo("vgId:%d, close sync", pVnode->config.vgId); + vInfo("vgId:%d, vnode close sync", pVnode->config.vgId); syncStop(pVnode->sync); } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index e244df276476..518fab8e28fe 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1863,7 +1863,8 @@ static int32_t ctgGetUserAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const cha } /** - * @brief shallow copy + * @brief shallow copy, only applicable to use non-pointer fields, while pointer fields may become invalid after return, + * caller should make deep copy if needed later. */ int32_t catalogGetUserAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, SGetUserAuthRsp* pRes) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 7a5969a5720a..3d5109d01160 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -972,7 +972,7 @@ void ctgDequeue(SCtgCacheOperation **op) { *op = node->op; } -int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { +int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation, bool *enqueued) { int32_t code = TSDB_CODE_SUCCESS; SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == node) { @@ -994,8 +994,7 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { CTG_RET(code); } } - - + CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); if (gCtgMgmt.queue.stopQueue) { @@ -1010,6 +1009,9 @@ int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { gCtgMgmt.queue.stopQueue = operation->stopQueue; CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); + if (enqueued) { + *enqueued = true; // the ownership of operation is transferred to queue + } ctgDebug("%sync action [%s] added into queue", syncOp ? "S": "As", opName); @@ -1071,7 +1073,7 @@ int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1108,7 +1110,7 @@ int32_t ctgDropDbVgroupEnqueue(SCatalog *pCtg, const char *dbFName, bool syncOp) op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1148,7 +1150,7 @@ int32_t ctgDropStbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1182,7 +1184,7 @@ int32_t ctgDropTbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1230,7 +1232,7 @@ int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1272,7 +1274,7 @@ int32_t ctgUpdateDbCfgEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1312,7 +1314,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog *pCtg, STableMetaOutput *output, bool sy op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1351,7 +1353,7 @@ int32_t ctgUpdateVgEpsetEnqueue(SCatalog *pCtg, char *dbFName, int32_t vgId, SEp op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1362,10 +1364,11 @@ int32_t ctgUpdateVgEpsetEnqueue(SCatalog *pCtg, char *dbFName, int32_t vgId, SEp int32_t ctgUpdateUserEnqueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncOp) { int32_t code = 0; + bool enqueued = false; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); if (NULL == op) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgCacheOperation)); - CTG_ERR_RET(terrno); + CTG_ERR_JRET(terrno); } op->opId = CTG_OP_UPDATE_USER; @@ -1383,13 +1386,21 @@ int32_t ctgUpdateUserEnqueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncOp op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); - - return TSDB_CODE_SUCCESS; + code = ctgEnqueue(pCtg, op, &enqueued); + // Clear source pointers to transfer ownership after successful enqueue + if (enqueued) { + pAuth->objPrivs = NULL; + pAuth->selectTbs = NULL; + pAuth->insertTbs = NULL; + pAuth->deleteTbs = NULL; + pAuth->tokens = NULL; + pAuth->ownedDbs = NULL; + } _return: - - tFreeSGetUserAuthRsp(pAuth); + if (!enqueued) { + tFreeSGetUserAuthRsp(pAuth); + } CTG_RET(code); } @@ -1417,7 +1428,7 @@ int32_t ctgUpdateTbIndexEnqueue(SCatalog *pCtg, STableIndex **pIndex, bool syncO op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); *pIndex = NULL; return TSDB_CODE_SUCCESS; @@ -1454,7 +1465,7 @@ int32_t ctgDropTbIndexEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1488,7 +1499,7 @@ int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool clearMeta, bool freeCtg, bool msg->freeCtg = freeCtg; op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1526,7 +1537,7 @@ int32_t ctgUpdateViewMetaEnqueue(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncOp op->data = msg; - CTG_ERR_RET(ctgEnqueue(pCtg, op)); + CTG_ERR_RET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1567,7 +1578,7 @@ int32_t ctgDropViewMetaEnqueue(SCatalog *pCtg, const char *dbFName, uint64_t dbI op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1601,7 +1612,7 @@ int32_t ctgUpdateTbTSMAEnqueue(SCatalog *pCtg, STSMACache **pTsma, int32_t tsmaV op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); *pTsma = NULL; return TSDB_CODE_SUCCESS; @@ -1638,7 +1649,7 @@ int32_t ctgDropTbTSMAEnqueue(SCatalog* pCtg, const STSMACache* pTsma, bool sync tstrncpy(msg->tsmaName, pTsma->name, TSDB_TABLE_NAME_LEN); op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; @@ -1721,7 +1732,7 @@ int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { CTG_ERR_JRET(code); - CTG_ERR_JRET(ctgEnqueue(pCtg, pOp)); + CTG_ERR_JRET(ctgEnqueue(pCtg, pOp, NULL)); return TSDB_CODE_SUCCESS; @@ -1767,7 +1778,7 @@ int32_t ctgUpdateDbTsmaVersionEnqueue(SCatalog* pCtg, int32_t tsmaVersion, const op->data = msg; - CTG_ERR_JRET(ctgEnqueue(pCtg, op)); + CTG_ERR_JRET(ctgEnqueue(pCtg, op, NULL)); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/groupcacheoperator.c b/source/libs/executor/src/groupcacheoperator.c index 49ef2a6a4345..5e117b7d0279 100644 --- a/source/libs/executor/src/groupcacheoperator.c +++ b/source/libs/executor/src/groupcacheoperator.c @@ -1524,6 +1524,7 @@ static void resetGroupCacheBlockCache(SGcBlkCacheInfo* pCache) { } static int32_t resetGroupCacheDownstreamCtx(SOperatorInfo* pOper) { + int32_t code = 0, lino = 0; SGroupCacheOperatorInfo* pInfo = pOper->info; if (NULL == pInfo->pDownstreams) { return TSDB_CODE_SUCCESS; @@ -1539,7 +1540,7 @@ static int32_t resetGroupCacheDownstreamCtx(SOperatorInfo* pOper) { int32_t defaultVg = 0; SGcVgroupCtx vgCtx = {0}; initGcVgroupCtx(pOper, &vgCtx, pCtx->id, defaultVg, NULL); - tSimpleHashPut(pCtx->pVgTbHash, &defaultVg, sizeof(defaultVg), &vgCtx, sizeof(vgCtx)); + TAOS_CHECK_EXIT(tSimpleHashPut(pCtx->pVgTbHash, &defaultVg, sizeof(defaultVg), &vgCtx, sizeof(vgCtx))); } taosArrayClearEx(pCtx->pFreeBlock, freeGcBlockInList); @@ -1555,7 +1556,13 @@ static int32_t resetGroupCacheDownstreamCtx(SOperatorInfo* pOper) { pCtx->fetchDone = false; } - return TSDB_CODE_SUCCESS; +_exit: + + if (code) { + qError("%s failed at line %d since %s", __func__, lino, tstrerror(code)); + } + + return code; } static int32_t resetGroupCacheOperState(SOperatorInfo* pOper) { @@ -1568,7 +1575,7 @@ static int32_t resetGroupCacheOperState(SOperatorInfo* pOper) { taosHashClear(pInfo->pGrpHash); - resetGroupCacheDownstreamCtx(pOper); + TAOS_CHECK_EXIT(resetGroupCacheDownstreamCtx(pOper)); memset(pInfo->execInfo.pDownstreamBlkNum, 0, pOper->numOfDownstream * sizeof(int64_t)); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 3ec2222b8938..dfd2cc67e8e7 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -435,9 +435,16 @@ static int32_t sifInitOperParams(SIFParam **params, SOperatorNode *node, SIFCtx indexError("invalid operation node, left: %p, rigth: %p", node->pLeft, node->pRight); SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } + + if (nParam == 2 && node->pRight != NULL && (nodeType(node->pRight)) != QUERY_NODE_VALUE) { + indexError("right node should be value, node:%p, type:%d", node->pRight, nodeType(node->pRight)); + SIF_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + if (node->opType == OP_TYPE_JSON_GET_VALUE) { return code; } + if ((node->pLeft != NULL && nodeType(node->pLeft) == QUERY_NODE_COLUMN) && (node->pRight != NULL && nodeType(node->pRight) == QUERY_NODE_VALUE)) { SColumnNode *cn = (SColumnNode *)(node->pLeft); @@ -721,6 +728,11 @@ static int8_t sifShouldUseIndexBasedOnType(SIFParam *left, SIFParam *right) { // not compress if (left->colValType == TSDB_DATA_TYPE_FLOAT) return 0; + // Column-to-column comparison cannot use index filter (e.g., tag1=tag2) + // right->condValue is NULL when right operand is a column reference, not a value + if (right->condValue == NULL) return 0; + + if (left->colValType == TSDB_DATA_TYPE_GEOMETRY || right->colValType == TSDB_DATA_TYPE_GEOMETRY || left->colValType == TSDB_DATA_TYPE_JSON || right->colValType == TSDB_DATA_TYPE_JSON) { return 0; diff --git a/source/libs/new-stream/src/dataSinkCache.c b/source/libs/new-stream/src/dataSinkCache.c index 6dd30f93a6e5..f757fb3221cb 100644 --- a/source/libs/new-stream/src/dataSinkCache.c +++ b/source/libs/new-stream/src/dataSinkCache.c @@ -231,7 +231,7 @@ bool setNextIteratorFromMem(SResultIter** ppResult) { return true; } } else { - // 在读取数据时已完成指针移动 + // pointer movement is completed while reading data SAlignGrpMgr* pAlignGrpMgr = (SAlignGrpMgr*)pResult->groupData; return pAlignGrpMgr->blocksInMem->size == 0; } diff --git a/source/libs/new-stream/src/dataSinkFile.c b/source/libs/new-stream/src/dataSinkFile.c index 977e70e05f5b..86332eaea5c0 100644 --- a/source/libs/new-stream/src/dataSinkFile.c +++ b/source/libs/new-stream/src/dataSinkFile.c @@ -186,7 +186,7 @@ bool setNextIteratorFromFile(SResultIter** ppResult) { return true; } } else { - // 在读取数据时已完成指针移动 + // pointer movement is completed while reading data SAlignGrpMgr* pAlignGrpMgr = (SAlignGrpMgr*)pResult->groupData; // todo return pAlignGrpMgr->blocksInMem->size == 0; @@ -265,7 +265,7 @@ static int32_t readFileDataToSlidingWindows(SResultIter* pResult, SSlidingGrpMgr } start += sizeof(SSlidingWindowInMem) + pWindowData->dataLen; if (start >= buf + pBlockInfo->dataLen) { - break; // 已经读取到数据末尾 + break; // end of current data buffer } } _exit: @@ -416,9 +416,9 @@ int32_t moveSlidingGrpMemCache(SSlidingTaskDSMgr* pSlidingTaskMgr, SSlidingGrpMg pSlidingGrp->groupId, moveWinCount, needSize, fileBlockInfo.groupOffset, fileBlockInfo.capacity, fileBlockInfo.dataLen); - if (false) { // 续写时， 可以不进行 taosLSeekFile, todo + if (false) { // append path may skip taosLSeekFile (todo) - } else { // 第一次写入 + } else { // first write int64_t ret = taosLSeekFile(pFileMgr->writeFilePtr, fileBlockInfo.groupOffset, SEEK_SET); if (ret < 0) { code = terrno; diff --git a/source/libs/new-stream/src/dataSinkMgr.c b/source/libs/new-stream/src/dataSinkMgr.c index fdc5bdc22dcf..c0f041ffc6a8 100644 --- a/source/libs/new-stream/src/dataSinkMgr.c +++ b/source/libs/new-stream/src/dataSinkMgr.c @@ -54,7 +54,7 @@ int32_t initStreamDataSink() { taosHashSetFreeFp(g_pDataSinkManager.dsStreamTaskList, destroySStreamDSTaskMgr); stInfo("data sink manager init success, max mem size: %" PRId64, tsStreamBufferSizeBytes); return TSDB_CODE_SUCCESS; -}; +} static bool isManagerReady() { if (g_pDataSinkManager.dsStreamTaskList != NULL) { @@ -246,7 +246,7 @@ static int32_t createSlidingTaskMgr(int64_t streamId, int64_t taskId, int64_t se return TSDB_CODE_SUCCESS; } -// @brief 初始化数据缓存 +// initialize data cache int32_t initStreamDataCache(int64_t streamId, int64_t taskId, int64_t sessionId, int32_t cleanMode, int32_t tsSlotId, void** ppCache) { int32_t code = 0; @@ -280,7 +280,7 @@ int32_t initStreamDataCache(int64_t streamId, int64_t taskId, int64_t sessionId, return code; } -// @brief 销毁数据缓存 +// destroy data cache void destroyStreamDataCache(void* pCache) { int32_t code = 0; if (pCache == NULL) { diff --git a/source/libs/new-stream/src/streamTriggerTask.c b/source/libs/new-stream/src/streamTriggerTask.c index 7d524c584cbe..11b7bce64971 100644 --- a/source/libs/new-stream/src/streamTriggerTask.c +++ b/source/libs/new-stream/src/streamTriggerTask.c @@ -2967,10 +2967,10 @@ static int32_t stRealtimeContextCalcExpr(SSTriggerRealtimeContext *pContext, SSD pResCol->info.precision = 0; int32_t nrows = blockDataGetNumOfRows(pDataBlock); - code = colInfoDataEnsureCapacity(pResCol, nrows, false); + code = colInfoDataEnsureCapacity(pResCol, pDataBlock->info.capacity, false); QUERY_CHECK_CODE(code, lino, _end); - TAOS_MEMSET(pResCol->nullbitmap, 0, BitmapLen(nrows)); - TAOS_MEMSET(pResCol->pData, 0, nrows); + TAOS_MEMSET(pResCol->nullbitmap, 0, BitmapLen(pDataBlock->info.capacity)); + TAOS_MEMSET(pResCol->pData, 0, pDataBlock->info.capacity); uint8_t idx = 1; SNode *pNode = NULL; @@ -3006,9 +3006,24 @@ static int32_t stRealtimeContextCalcExpr(SSTriggerRealtimeContext *pContext, SSD pResCol->info.scale = pType->scale; pResCol->info.precision = pType->precision; - SScalarParam output = {.columnData = pResCol}; + if (pTmpCol == NULL) { + pTmpCol = taosMemoryCalloc(1, sizeof(SColumnInfoData)); + QUERY_CHECK_NULL(pTmpCol, code, lino, _end, terrno); + } + pTmpCol->info.type = pType->type; + pTmpCol->info.bytes = pType->bytes; + pTmpCol->info.scale = pType->scale; + pTmpCol->info.precision = pType->precision; + + SScalarParam output = {.columnData = pTmpCol}; code = scalarCalculate(pExpr, pList, &output, NULL); QUERY_CHECK_CODE(code, lino, _end); + int32_t nrows = blockDataGetNumOfRows(pDataBlock); + QUERY_CHECK_CONDITION(output.numOfRows == nrows, code, lino, _end, TSDB_CODE_INTERNAL_ERROR); + code = colInfoDataEnsureCapacity(pResCol, pDataBlock->info.capacity, false); + QUERY_CHECK_CODE(code, lino, _end); + code = colDataAssign(pResCol, pTmpCol, nrows, NULL); + QUERY_CHECK_CODE(code, lino, _end); } _end: @@ -5931,10 +5946,10 @@ static int32_t stHistoryContextCalcExpr(SSTriggerHistoryContext *pContext, SSDat pResCol->info.precision = 0; int32_t nrows = blockDataGetNumOfRows(pDataBlock); - code = colInfoDataEnsureCapacity(pResCol, nrows, false); + code = colInfoDataEnsureCapacity(pResCol, pDataBlock->info.capacity, false); QUERY_CHECK_CODE(code, lino, _end); - TAOS_MEMSET(pResCol->nullbitmap, 0, BitmapLen(nrows)); - TAOS_MEMSET(pResCol->pData, 0, nrows); + TAOS_MEMSET(pResCol->nullbitmap, 0, BitmapLen(pDataBlock->info.capacity)); + TAOS_MEMSET(pResCol->pData, 0, pDataBlock->info.capacity); uint8_t idx = 1; SNode *pNode = NULL; @@ -5970,9 +5985,24 @@ static int32_t stHistoryContextCalcExpr(SSTriggerHistoryContext *pContext, SSDat pResCol->info.scale = pType->scale; pResCol->info.precision = pType->precision; - SScalarParam output = {.columnData = pResCol}; + if (pTmpCol == NULL) { + pTmpCol = taosMemoryCalloc(1, sizeof(SColumnInfoData)); + QUERY_CHECK_NULL(pTmpCol, code, lino, _end, terrno); + } + pTmpCol->info.type = pType->type; + pTmpCol->info.bytes = pType->bytes; + pTmpCol->info.scale = pType->scale; + pTmpCol->info.precision = pType->precision; + + SScalarParam output = {.columnData = pTmpCol}; code = scalarCalculate(pExpr, pList, &output, NULL); QUERY_CHECK_CODE(code, lino, _end); + int32_t nrows = blockDataGetNumOfRows(pDataBlock); + QUERY_CHECK_CONDITION(output.numOfRows == nrows, code, lino, _end, TSDB_CODE_INTERNAL_ERROR); + code = colInfoDataEnsureCapacity(pResCol, pDataBlock->info.capacity, false); + QUERY_CHECK_CODE(code, lino, _end); + code = colDataAssign(pResCol, pTmpCol, nrows, NULL); + QUERY_CHECK_CODE(code, lino, _end); } _end: diff --git a/source/libs/new-stream/src/streamUtil.c b/source/libs/new-stream/src/streamUtil.c index 4bf78caf6fd7..43acf3a0be38 100755 --- a/source/libs/new-stream/src/streamUtil.c +++ b/source/libs/new-stream/src/streamUtil.c @@ -23,10 +23,6 @@ #include "decimal.h" #include "cmdnodes.h" -#ifndef WINDOWS -#include "curl/curl.h" -#endif - int32_t streamGetThreadIdx(int32_t threadNum, int64_t streamGId) { return threadNum ? (streamGId % threadNum) : 0; } int32_t stmAddFetchStreamGid(void) { @@ -766,8 +762,6 @@ static int32_t streamAppendNotifyContent(int32_t triggerType, int64_t groupId, c return code; } -#ifndef WINDOWS - #define STREAM_EVENT_NOTIFY_RETRY_MS 50 // 50 ms int32_t streamSendNotifyContent(SStreamTask* pTask, const char* streamName, const char* tableName, int32_t triggerType, @@ -884,14 +878,6 @@ int32_t streamSendNotifyContent(SStreamTask* pTask, const char* streamName, cons } return code; } -#else -int32_t streamSendNotifyContent(SStreamTask* pTask, const char* streamName, const char* tableName, int32_t triggerType, - int64_t groupId, const SArray* pNotifyAddrUrls, int32_t errorHandle, - const SSTriggerCalcParam* pParams, int32_t nParam) { - ST_TASK_ELOG("stream notify events is not supported on windows, streamName:%s", streamName); - return TSDB_CODE_NOT_SUPPORTTED_IN_WINDOWS; -} -#endif int32_t readStreamDataCache(int64_t streamId, int64_t taskId, int64_t sessionId, int64_t groupId, TSKEY start, TSKEY end, void*** pppIter) { diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 316af87c03d1..602dc8d1867c 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -135,6 +135,7 @@ static int32_t columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { COPY_SCALAR_FIELD(resIdx); COPY_SCALAR_FIELD(hasDep); COPY_SCALAR_FIELD(hasRef); + COPY_SCALAR_FIELD(flags); COPY_CHAR_ARRAY_FIELD(refDbName); COPY_CHAR_ARRAY_FIELD(refTableName); COPY_CHAR_ARRAY_FIELD(refColName); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index b2851af9aa83..605f75cb24ef 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -3317,15 +3317,23 @@ static EDealRes doCollect(SCollectColumnsCxt* pCxt, SColumnNode* pCol, SNode* pN } SNode** pNodeFound = taosHashGet(pCxt->pColHash, name, len); if (pNodeFound == NULL) { - pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, &pNode, POINTER_BYTES); + SNode* pNew = NULL; + pCxt->errCode = nodesCloneNode(pNode, &pNew); if (TSDB_CODE_SUCCESS == pCxt->errCode) { - SNode* pNew = NULL; - pCxt->errCode = nodesCloneNode(pNode, &pNew); - if (TSDB_CODE_SUCCESS == pCxt->errCode) { - pCxt->errCode = nodesListStrictAppend(pCxt->pCols, pNew); - } + pCxt->errCode = nodesListStrictAppend(pCxt->pCols, pNew); + } + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + // Store cloned node pointer in hash so we can update its flags later + pCxt->errCode = taosHashPut(pCxt->pColHash, name, len, &pNew, POINTER_BYTES); + } else { + nodesDestroyNode(pNew); } return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); + } else if (0 == pCol->appendByPrivCond) { + SColumnNode* pExistCol = (SColumnNode*)(*pNodeFound); + if (pExistCol->appendByPrivCond == 1) { + pExistCol->appendByPrivCond = 0; + } } return DEAL_RES_CONTINUE; } diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 920c050d519f..dad60fbe8b2c 100755 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -539,6 +539,8 @@ priv_type(A) ::= ALTER PASS. priv_type(A) ::= CREATE ROLE. { A = PRIV_SET_TYPE(PRIV_ROLE_CREATE); } priv_type(A) ::= DROP ROLE. { A = PRIV_SET_TYPE(PRIV_ROLE_DROP); } +priv_type(A) ::= UNLOCK ROLE. { A = PRIV_SET_TYPE(PRIV_ROLE_UNLOCK); } +priv_type(A) ::= LOCK ROLE. { A = PRIV_SET_TYPE(PRIV_ROLE_LOCK); } priv_type(A) ::= SHOW ROLES. { A = PRIV_SET_TYPE(PRIV_ROLE_SHOW); } priv_type(A) ::= CREATE USER. { A = PRIV_SET_TYPE(PRIV_USER_CREATE); } diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index 7d3b1409cc55..df9405e7d6f2 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -233,6 +233,7 @@ EDealRes rewriteAuthTable(SNode* pNode, void* pContext) { SAuthRewriteCxt* pCxt = (SAuthRewriteCxt*)pContext; tstrncpy(pCol->tableName, pCxt->pTarget->tableName, TSDB_TABLE_NAME_LEN); tstrncpy(pCol->tableAlias, pCxt->pTarget->tableAlias, TSDB_TABLE_NAME_LEN); + pCol->appendByPrivCond = 1; } return DEAL_RES_CONTINUE; diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index a2f6daf17976..88ae44b2e00d 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -4488,6 +4488,14 @@ static void clearCatalogReq(SCatalogReq* pCatalogReq) { static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SVnodeModifyOpStmt* pStmt) { clearCatalogReq(pCatalogReq); + if (pStmt->pPrivCols) { + taosArrayDestroy(pStmt->pPrivCols); + pStmt->pPrivCols = NULL; + } + if (pStmt->pTagCond) { + nodesDestroyNode(pStmt->pTagCond); + pStmt->pTagCond = NULL; + } int32_t code = checkAuthFromMetaData(pCxt, pMetaData, pStmt); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5d58b159a2a0..56e2464c6cec 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1791,8 +1791,10 @@ static int32_t findAndSetRealTableColumn(STranslateContext* pCxt, SColumnNode** int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; for (int32_t i = 0; i < nums; ++i) { - if (0 == strcmp(pCol->colName, pMeta->schema[i].name) && - !invisibleColumn(true, pMeta->tableType, pMeta->schema[i].flags)) { // pCxt->pParseCxt->enableSysInfo, only control the output columns + if (0 == strcmp(pCol->colName, pMeta->schema[i].name)) { + if (invisibleColumn(pCxt->pParseCxt->enableSysInfo, pMeta->tableType, pMeta->schema[i].flags)) { + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_COL_PERMISSION_DENIED, pCol->colName); + } SSchemaExt* pSchemaExt = pMeta->schemaExt ? (i >= pMeta->tableInfo.numOfColumns ? NULL : (pMeta->schemaExt + i)) : NULL; setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i - pMeta->tableInfo.numOfColumns), pCol, @@ -7170,6 +7172,9 @@ static int32_t translateCheckPrivCols(STranslateContext* pCxt, SSelectStmt* pSel FOREACH(pNode, pRetrievedCols) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SColumnNode* pCol = (SColumnNode*)pNode; + if (pCol->appendByPrivCond) { + continue; + } SColIdNameKV colIdNameKV = {.colId = pCol->colId}; snprintf(colIdNameKV.colName, TSDB_COL_NAME_LEN, "%s", pCol->colName); STableCols* pTblCols = tSimpleHashGet(pTblColHash, (const void*)&pCol->tableId, sizeof(pCol->tableId)); @@ -17679,7 +17684,7 @@ static int32_t fillPrivSetRowCols(STranslateContext* pCxt, SArray** ppReqCols, S return code; } - int32_t columnNum = pTableMeta->tableInfo.numOfColumns; + int32_t columnNum = pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; const SSchema* pSchemaCols = pTableMeta->schema; SNode* pNode = NULL; SColumnNode* pCol = NULL; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index b9553f57c0f7..9017d437a85c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1684,7 +1684,7 @@ void syncHbTimerDataFree(SSyncHbTimerData* pData) { taosMemoryFree(pData); } void syncNodeClose(SSyncNode* pSyncNode) { int32_t code = 0; if (pSyncNode == NULL) return; - sNInfo(pSyncNode, "sync close, node:%p", pSyncNode); + sNInfo(pSyncNode, "sync node close, node:%p", pSyncNode); syncRespCleanRsp(pSyncNode->pSyncRespMgr); @@ -1774,7 +1774,11 @@ int32_t syncNodeStopPingTimer(SSyncNode* pSyncNode) { int32_t code = 0; (void)atomic_add_fetch_64(&pSyncNode->pingTimerLogicClockUser, 1); bool stop = taosTmrStop(pSyncNode->pPingTimer); - sDebug("vgId:%d, stop ping timer, stop:%d", pSyncNode->vgId, stop); + if (!stop) { + sWarn("vgId:%d, failed to stop ping timer, maybe it's already stopped, stop:%d", pSyncNode->vgId, stop); + } else { + sDebug("vgId:%d, stop ping timer, stop:%d", pSyncNode->vgId, stop); + } pSyncNode->pPingTimer = NULL; return code; } diff --git a/source/util/src/tcurl.c b/source/util/src/tcurl.c index 2a2d9373ec11..664bfecaab26 100644 --- a/source/util/src/tcurl.c +++ b/source/util/src/tcurl.c @@ -21,10 +21,6 @@ #include "tcurl.h" #include "tutil.h" -#ifndef WINDOWS - -#include "curl/curl.h" - static threadlocal SHashObj* tNotificationConnHash = NULL; // key: url, value: CURL* static threadlocal bool tInitialized = false; @@ -206,10 +202,3 @@ void closeThreadNotificationConn() { tNotificationConnHash = NULL; tInitialized = false; } - -#else -void closeThreadNotificationConn() { - // no-op on Windows -} - -#endif diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c index 6f7d73962591..865b1e2990df 100644 --- a/source/util/src/tpagedbuf.c +++ b/source/util/src/tpagedbuf.c @@ -67,6 +67,13 @@ static int32_t createDiskFile(SDiskbasedBuf* pBuf) { return terrno; } + int64_t realSize = -1; + if (taosFStatFile(pBuf->pFile, &realSize, NULL) != TSDB_CODE_SUCCESS) { + realSize = -1; + } + uDebug("paged buffer file opened, path:%s, realSize:%" PRId64 ", nextPos:%" PRIu64 ", fileSize:%" PRIu64 ", %s", + pBuf->path, realSize, pBuf->nextPos, pBuf->fileSize, pBuf->id); + return TSDB_CODE_SUCCESS; } @@ -625,7 +632,8 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) { int32_t code = taosCloseFile(&pBuf->pFile); if (TSDB_CODE_SUCCESS != code) { - uDebug("WARNING tPage failed to close file when destroy disk basebuf: %s", pBuf->path); + uError("failed to close paged buffer file when destroying, path:%s, closeCode:%d, err:%s, %s", pBuf->path, code, + tstrerror(code), pBuf->id); } } else { uDebug("Paged buffer closed, total:%.2f Kb, no file created, %s", pBuf->totalBufSize / 1024.0, pBuf->id); @@ -756,6 +764,22 @@ void dBufPrintStatis(const SDiskbasedBuf* pBuf) { } void clearDiskbasedBuf(SDiskbasedBuf* pBuf) { + if (pBuf == NULL) { + return; + } + + int64_t realSizeBefore = -1; + if (pBuf->pFile != NULL && taosFStatFile(pBuf->pFile, &realSizeBefore, NULL) != TSDB_CODE_SUCCESS) { + realSizeBefore = -1; + } + + const SDiskbasedBufStatis* ps = &pBuf->statis; + uDebug( + "clear paged buffer begin, pages:%d, inMemPages:%d, fileSize:%" PRIu64 ", nextPos:%" PRIu64 + ", realSize:%" PRId64 ", get/release:%d/%d, flush/load:%d/%d, %s", + pBuf->numOfPages, listNEles(pBuf->lruList), pBuf->fileSize, pBuf->nextPos, realSizeBefore, ps->getPages, + ps->releasePages, ps->flushPages, ps->loadPages, pBuf->id); + size_t n = taosArrayGetSize(pBuf->pIdList); for (int32_t i = 0; i < n; ++i) { SPageInfo* pi = taosArrayGetP(pBuf->pIdList, i); @@ -777,4 +801,21 @@ void clearDiskbasedBuf(SDiskbasedBuf* pBuf) { pBuf->totalBufSize = 0; pBuf->allocateId = -1; pBuf->fileSize = 0; + pBuf->nextPos = 0; + + if (pBuf->pFile != NULL) { + int32_t code = taosFtruncateFile(pBuf->pFile, 0); + if (code != TSDB_CODE_SUCCESS) { + uWarn("failed to truncate paged buffer file, path:%s, code:%s, %s", pBuf->path, tstrerror(code), pBuf->id); + } + } + + int64_t realSizeAfter = -1; + if (pBuf->pFile != NULL && taosFStatFile(pBuf->pFile, &realSizeAfter, NULL) != TSDB_CODE_SUCCESS) { + realSizeAfter = -1; + } + + uDebug("clear paged buffer end, pages:%d, inMemPages:%d, fileSize:%" PRIu64 ", nextPos:%" PRIu64 + ", realSize:%" PRId64 ", %s", + pBuf->numOfPages, listNEles(pBuf->lruList), pBuf->fileSize, pBuf->nextPos, realSizeAfter, pBuf->id); } diff --git a/test/README.md b/test/README.md index 065972d3a9d8..d7fba552b978 100644 --- a/test/README.md +++ b/test/README.md @@ -2,10 +2,10 @@ 1. [Introduction](#1-introduction) 1. [Prerequisites](#2-prerequisites) -1. [Project Structure](#3-Project-Structure) -1. [Run Test Cases](#4-Run-Test-Cases) -1. [Add New Case](#5-Add-New-Case) -1. [Add New Case to CI](#6-Add-New-Case-to-CI) +1. [Project Structure](#3-project-structure) +1. [Run Test Cases](#4-run-test-cases) +1. [Add New Case](#5-add-new-case) +1. [Add New Case to CI](#6-add-new-case-to-ci) 1. [Workflows](#7-workflows) 1. [Test Report](#8-test-report) @@ -26,6 +26,7 @@ This is the new end-to-end testing framework for TDengine. It offers several adv 6. **Integration with Github Action**: Including workflow to run the test cases in the new test framework, workflow to validate the docstring of test cases and workflow to publish case docs to Github Pages. > [!NOTE] +> > - The commands and scripts below are verified on Linux (Ubuntu 18.04/20.04/22.04). > - [taos-connector-python](https://github.com/taosdata/taos-connector-python) is used by tests written in Python, which requires Python 3.8+. @@ -34,8 +35,8 @@ This is the new end-to-end testing framework for TDengine. It offers several adv - Install Python3 ```bash -apt install python3 -apt install python3-pip +apt install python3 -y +apt install python3-pip -y ``` - Install Python dependencies @@ -45,6 +46,12 @@ cd test pip3 install -r requirements.txt ``` +- Install screen + +```bash +apt install screen -y +``` + - Building (Optional) Tests can be run against TDengine either by installation or by build. When building TDengine, please make sure the building options `-DBUILD_TOOLS=true -DBUILD_TEST=true -DBUILD_CONTRIB=true` has been used: @@ -58,7 +65,8 @@ make && make install # 3. Project Structure Outline the main directories and their purposes: -``` + +```text test/ │ ├── cases/ # cases directory @@ -96,7 +104,8 @@ pytest [options] [test_file_path] ``` Notes: -- options: described below + +- options: described below - test_file_path: optional, if provided, only the test case in the file or path will be run; if not provided, all test cases (files start with test_) will be run ## 4.3 Run tests by command line arguments @@ -141,9 +150,10 @@ Options: ## 4.4 Run tests by configuration file -- `--yaml_file `: TDengine deploy configuration yaml file (default directory is `env`, no need to specify the `env` path) +- `--yaml_file `: TDengine deploy configuration yaml file (default directory is `env`, no need to specify the `env` path) **Mutually Exclusive Options**: + - The `--yaml_file` option is mutually exclusive with the following options: - `-N` - `-R` @@ -156,6 +166,7 @@ Options: ## 4.5 Common Options Some useful pytest common options: + - `-s`: disable output capturing - `--log-level`: set log level - `--alluredir`: generate allure report directory @@ -190,6 +201,7 @@ pytest --clean --only_deploy To run test cases in batch: Run with default test list file: + ```bash ./start_run_test_list.sh #run with default test list file(test_list.txt) ./start_run_test_list.sh path/to/case_list_file #run with custom test list file @@ -198,11 +210,13 @@ Run with default test list file: For the format of the test list file, please refer to `test_list.txt`. To stop the test execution: + ```bash ./stop_run_test_list.sh #the script will stop after completing the current test case. ``` Batch test results can be found in the `test_logs` directory: + - `test_logs/case_result.txt`: Case execution results - `test_logs/run_tests.log`: All cases outputs - `test_logs/xxx.log`: Failed case outputs @@ -248,15 +262,19 @@ To add a new test case to the CI pipeline, include the case run command in the ` Every time new code is submitted, the corresponding GitHub workflows are triggered as follows: ## 7.1 CI Test + A CI test is triggered whenever a pull request (PR) is submitted. ## 7.2 Docstring Check + A Docstring check is triggered for any PR submitted to the `test/cases` directory, ensuring the completeness of the case descriptions. ## 7.3 Cases Doc Publish + A cases documentation publish is triggered when a PR is merged into the `test/cases` directory, updating the case description documentation page. Note: + - Please referto [Deploy Case Docs](https://github.com/taosdata/TDengine/actions/workflows/deploy-case-docs.yml) for details. - Published cases doc can be found at [TDengine Case List](https://taosdata.github.io/TDengine/main/). diff --git a/test/cases/15-TagIndices/test_index_tag_basic.py b/test/cases/15-TagIndices/test_index_tag_basic.py index 97316b82de86..7521458c2c4a 100644 --- a/test/cases/15-TagIndices/test_index_tag_basic.py +++ b/test/cases/15-TagIndices/test_index_tag_basic.py @@ -233,7 +233,7 @@ def longname_idx(self, stbname): # ------------------- 2 ---------------- # def prepareData(self): - self.dbname = 'db' + self.dbname = 'db_ts4403' self.stbname = 'st' # db tdSql.execute("create database {};".format(self.dbname)) @@ -284,6 +284,51 @@ def do_ts4403(self): # add index for multiple tags(TD-28078) tdSql.error("create index tt on {} (t2, t3);".format(self.stbname)) tdLog.debug("Verify add index for multiple tags successfully") + + def do_tag_column_comparison(self): + """Test that tag-to-tag comparison does not crash taosd + + Bug: When executing query like 'select * from st where tag1=tag2', + the index filter sifSetFltParam would crash because right->condValue is NULL. + The fix skips the index filter for column-to-column comparisons, + allowing the query to execute via fallback path without crash. + """ + tdSql.execute("create database if not exists test_tag_col;") + tdSql.execute("use test_tag_col;") + tdSql.execute("create stable st(ts timestamp, val int) tags(tag1 int, tag2 int);") + tdSql.execute("create table ct1 using st tags(1, 2);") + tdSql.execute("create table ct2 using st tags(3, 3);") + tdSql.execute("create table ct3 using st tags(5, 6);") + tdSql.execute("insert into ct1 values(now, 1);") + tdSql.execute("insert into ct2 values(now, 2);") + tdSql.execute("insert into ct3 values(now, 3);") + + # tag1=tag2 should work without crash (fallback to non-index filter path) + tdSql.query("select * from st where tag1=tag2;") + tdSql.checkRows(1) # ct2 where tag1=3, tag2=3 + tdLog.info("tag1=tag2 query works correctly (no crash)") + + # tag1tag2 should also work + tdSql.query("select * from st where tag1>tag2;") + tdSql.checkRows(0) # no matching rows + tdLog.info("tag1>tag2 query works correctly (no crash)") + # Normal tag=value queries should still work + tdSql.query("select * from st where tag1=1;") + tdSql.checkRows(1) + tdLog.info("tag1=1 query works correctly") + + tdSql.query("select * from st where tag2=3;") + tdSql.checkRows(1) + tdLog.info("tag2=3 query works correctly") + + # Cleanup + tdSql.execute("drop database test_tag_col;") + tdLog.info("Tag-to-tag comparison test passed") # @@ -304,6 +349,7 @@ def test_index_tag_basic(self): 8. Drop all tag indexes 9. Attempt to create tag index with excessively long name and verify error 10. bug TS-4403: Create/drop tag index on supertable and verify behavior + 11. Verify tag-to-tag comparison does not crash taosd Since: v3.0.0.0 @@ -334,6 +380,7 @@ def test_index_tag_basic(self): self.longname_idx(stable) self.do_ts4403() + self.do_tag_column_comparison() diff --git a/test/cases/24-Users/test_user_privilege_sysinfo.py b/test/cases/24-Users/test_user_privilege_sysinfo.py index 5e5a34984dd8..15afbc1943b9 100644 --- a/test/cases/24-Users/test_user_privilege_sysinfo.py +++ b/test/cases/24-Users/test_user_privilege_sysinfo.py @@ -96,8 +96,7 @@ def do_user_privilege_sysinfo(self): tdSql.error(f"use db") tdSql.error(f"alter database db replica 1;") tdSql.error(f"alter database db keep 21") - tdSql.execute(f"show db.vgroups") - tdSql.checkRows(0) + tdSql.error(f"show db.vgroups") tdSql.error(f"create table db.stb1 (ts timestamp, i int) tags (t int)") tdSql.error(f"create table db.ctb1 using db.stb1 tags (1)") @@ -138,16 +137,12 @@ def loop_check_sysinfo_0(): tdSql.error(f"show qnodes") tdSql.error(f"show mnodes") tdLog.info(f"=============== check show of sysinfo 0 - sleep") - tdSql.execute(f"show db.vgroups") - tdSql.checkRows(0) + tdSql.error(f"show db.vgroups") tdSql.error(f"show db.stables") tdSql.error(f"show db.tables") tdSql.error(f"show indexes from stb from db") tdSql.query(f" show databases") - tdSql.query(f"show d2.vgroups") - tdSql.checkRows(2) - tdSql.checkData(0, 0, 'leader') # sysinfo0 can only see columns with sysinfo false - tdSql.checkData(1, 0, 'leader') + tdSql.error(f"show d2.vgroups") tdSql.query(f" show d2.stables") tdSql.query(f" show d2.tables") tdSql.query(f" show indexes from stb2 from d2") @@ -204,6 +199,7 @@ def loop_check_sysinfo_0(): tdSql.error(f"select * from information_schema.ins_vgroups") tdSql.query(f"select * from information_schema.ins_configs") tdSql.error(f"select * from information_schema.ins_dnode_variables") + tdSql.error(f"select `precision`, `keep` from information_schema.ins_databases", expectErrInfo="Permission denied for column: keep", fullMatched=False) tdLog.info(f"=============== check performance_schema of sysinfo 0") tdSql.execute(f"use performance_schema;") diff --git a/test/cases/25-Privileges/test_priv_control.py b/test/cases/25-Privileges/test_priv_control.py index 694f2825f06e..90c5adda5d3c 100644 --- a/test/cases/25-Privileges/test_priv_control.py +++ b/test/cases/25-Privileges/test_priv_control.py @@ -1469,7 +1469,7 @@ def do_column_row_combined_privilege(self): self.query_expect_rows(f"SELECT temperature, humidity FROM {db_name}.st1", 1) # Only ct1 ''' self.exec_sql_failed(f"SELECT status FROM {db_name}.st1", TSDB_CODE_PAR_COL_PERMISSION_DENIED) # Column not authorized - self.exec_sql_failed(f"SELECT temperature FROM {db_name}.ct2", TSDB_CODE_PAR_COL_PERMISSION_DENIED) # Row not authorized + self.query_expect_rows(f"SELECT temperature FROM {db_name}.ct2", 0) # Row not authorized, reuse super table privilege # Test 2: Column INSERT with row condition self.login() diff --git a/test/cases/25-Privileges/test_priv_rbac.py b/test/cases/25-Privileges/test_priv_rbac.py index 5fd5ff7c3e7b..2c8b5fa74dd0 100644 --- a/test/cases/25-Privileges/test_priv_rbac.py +++ b/test/cases/25-Privileges/test_priv_rbac.py @@ -1,5 +1,6 @@ from new_test_framework.utils import tdLog, tdSql, tdDnodes, etool, TDSetSql from new_test_framework.utils.sqlset import TDSetSql +from taos.tmq import Consumer from itertools import product import os import time @@ -48,6 +49,7 @@ def do_basic_user_privileges(self): tdSql.execute("grant create table on database d0 to u1") tdSql.execute("grant use database on database d0 to u1") tdSql.execute("grant use on database d0 to u1") + tdSql.execute("grant lock role,unlock role,lock user,unlock user to u1") tdSql.execute("grant select(c0,c1),insert(ts,c0),delete on table d0.stb0 with t1=0 and ts=0 to u1") def do_basic_role_privileges(self): @@ -67,9 +69,61 @@ def do_basic_role_privileges(self): tdSql.execute("revoke all on table d0.stb0 from r1") tdSql.execute("grant select(c0,c1),insert(ts,c0),delete on table d0.stb0 with t1=0 and ts=0 to r1") + def do_check_column_privileges(self): + """Test column privileges""" + + tdSql.execute(f"create user u_col_2 pass '{self.test_pass}'") + tdSql.execute(f"grant use on database d0 to u_col_2") + tdSql.execute(f"grant select(c0),insert(ts,c0) on table d0.stb0 with t1=0 to u_col_2") + tdSql.connect("u_col_2", self.test_pass) + tdSql.error("select * from d0.stb0 where t1=0", expectErrInfo="Permission denied for column: ts", fullMatched=False) + tdSql.error("select c0,c1 from d0.stb0", expectErrInfo="Permission denied for column: c1", fullMatched=False) + tdSql.error("select c0,t1 from d0.stb0", expectErrInfo="Permission denied for column: t1", fullMatched=False) + tdSql.error("select c0 from d0.stb0 where t1=0 and ts=0", expectErrInfo="Permission denied for column: ts", fullMatched=False) + tdSql.error("select c0,t1 from d0.ctb0", expectErrInfo="Permission denied for column: t1", fullMatched=False) + tdSql.query("select c0 from d0.stb0") + tdSql.checkRows(2) + tdSql.query("select c0 from d0.ctb0") + tdSql.checkRows(2) + tdSql.error("select c1 from d0.stb0 where t1=0", expectErrInfo="Permission denied for column: c1", fullMatched=False) + for i in range(10): + tdSql.execute("insert into d0.ctb0 (ts,c0) values(now+%ds,%d)" % (i, i)) + tdSql.error("insert into d0.ctb0 (ts,c1) values(now+%ds,%d)" % (i, i), expectErrInfo="Permission denied for column: c1", fullMatched=False) + + def subscribe_topic(self, user, password, group_id, topic_name): + attr = { + 'group.id': group_id, + 'td.connect.user': user, + 'td.connect.pass': password, + 'auto.offset.reset': 'earliest' + } + consumer = Consumer(attr) + consumer.subscribe([topic_name]) + + def do_check_topic_privileges(self): + """Test topic privileges""" + tdSql.connect("root", "taosdata") + tdSql.execute(f"create user u_topic pass '{self.test_pass}'") + tdSql.execute(f"create user u_consumer pass '{self.test_pass}'") + tdSql.execute(f"grant use on database d0 to u_topic") + tdSql.execute(f"grant create topic on database d0 to u_topic") + tdSql.execute(f"grant select on d0.stb0 to u_topic") + tdSql.connect("u_topic", self.test_pass) + time.sleep(5) # wait for privileges to take effect + tdSql.query("select * from d0.stb0") + tdSql.execute(f"create topic topic1 as select * from d0.stb0") + tdSql.error(f"create topic topic2 as select * from d0.stb1", expectErrInfo="Permission denied", fullMatched=False) + self.subscribe_topic("u_topic", self.test_pass, "g1", "topic1") + tdSql.connect("root", "taosdata") + tdSql.execute(f"grant use on database d0 to u_consumer") + tdSql.execute(f"grant subscribe on topic d0.topic1 to u_consumer") + tdSql.connect("u_consumer", self.test_pass) + time.sleep(5) # wait for privileges to take effect + self.subscribe_topic("u_consumer", self.test_pass, "g1", "topic1") + def do_check_role_privileges(self): """Test role privileges""" - + tdSql.connect("root", "taosdata") tdSql.execute(f"create user ur1 pass '{self.test_pass}'") tdSql.execute(f"grant role `SYSDBA` to ur1") tdSql.error("grant role `SYSSEC` to ur1", expectErrInfo=f"Conflicts with existing role", fullMatched=False) @@ -146,9 +200,10 @@ def test_priv_basic(self): # self.do_check_db_privileges() # self.do_check_table_privileges() # self.do_check_row_privileges() - # self.do_check_column_privileges() + self.do_check_column_privileges() # self.do_check_grant_privileges() # self.do_check_view_privileges() + self.do_check_topic_privileges() # self.do_check_audit_privileges() # self.do_check_user_privileges() self.do_check_role_privileges() diff --git a/test/ci/cases.task b/test/ci/cases.task index e37c5f0b5c71..ae441cd2226c 100644 --- a/test/ci/cases.task +++ b/test/ci/cases.task @@ -534,6 +534,7 @@ ,,y,.,./ci/pytest.sh pytest cases/15-TagIndices/test_index_create_drop.py ,,y,.,./ci/pytest.sh pytest cases/15-TagIndices/test_index_overflow.py ,,y,.,./ci/pytest.sh pytest cases/15-TagIndices/test_index_perf.py +,,y,.,./ci/pytest.sh pytest cases/15-TagIndices/test_index_tag_basic.py # 16-Views ,,y,.,./ci/pytest.sh pytest cases/16-Views/test_view_basic.py @@ -1003,7 +1004,7 @@ ,,y,.,./ci/pytest.sh pytest cases/81-Tools/02-Taos/test_tool_taos_shell_error.py ,,y,.,./ci/pytest.sh pytest cases/81-Tools/02-Taos/test_tool_taos_shell_net_chk.py ## 03-Benchmark -,,n,pytest cases/81-Tools/03-Benchmark/test_benchmark_basic.py +,,n,.,pytest cases/81-Tools/03-Benchmark/test_benchmark_basic.py ,,y,.,./ci/pytest.sh pytest cases/81-Tools/03-Benchmark/test_benchmark_bugs.py -B ,,y,.,./ci/pytest.sh pytest cases/81-Tools/03-Benchmark/test_benchmark_commandline.py ,,y,.,./ci/pytest.sh pytest cases/81-Tools/03-Benchmark/test_benchmark_conn_mode.py -B diff --git a/test/ci/run.sh b/test/ci/run.sh index c2e8f0d6c554..083a19badd80 100755 --- a/test/ci/run.sh +++ b/test/ci/run.sh @@ -32,6 +32,7 @@ while getopts "m:t:b:l:o:w:eh" opt; do ;; o) timeout_param="-o $OPTARG" + timeout_val=$OPTARG ;; w) web_server=$OPTARG @@ -141,21 +142,73 @@ function is_local_host() { return 1 } +function get_timeout_val() { + echo "${timeout_val:-1260}" +} + +function get_local_workdir() { + local i=0 + while [ $i -lt ${#hosts[*]} ]; do + if is_local_host "${hosts[i]}"; then + echo "${workdirs[i]}" + return 0 + fi + i=$((i + 1)) + done + return 1 +} + function get_remote_ssh_command() { local index=$1 + local cmd_timeout + cmd_timeout=$(get_timeout_val) if [ -z "${passwords[index]}" ]; then - echo "ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" + echo "timeout ${cmd_timeout} ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=3 ${usernames[index]}@${hosts[index]}" else - echo "sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no ${usernames[index]}@${hosts[index]}" + echo "timeout ${cmd_timeout} sshpass -p ${passwords[index]} ssh -o StrictHostKeyChecking=no -o ServerAliveInterval=60 -o ServerAliveCountMax=3 ${usernames[index]}@${hosts[index]}" fi } function get_remote_scp_command() { local index=$1 if [ -z "${passwords[index]}" ]; then - echo "scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" + echo "scp -o StrictHostKeyChecking=no -r" + else + echo "sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r" + fi +} + +function save_build_artifacts() { + local build_dir=$1 + local case_info=$2 + local local_work_dir + local local_build_dir + local local_unit_test_log_dir + + local_work_dir=$(get_local_workdir) + if [ -z "$local_work_dir" ]; then + return 0 + fi + + local_build_dir="${local_work_dir}/${DEBUGPATH}/build" + local_unit_test_log_dir="${local_work_dir}/${DEBUGPATH}/Testing/Temporary/" + + if [ ! -d "$local_build_dir" ]; then + return 0 + fi + + mkdir -p "$build_dir" >/dev/null 2>&1 || true + if [[ "$case_info" == *"UnitTest/test.sh"* ]]; then + cp -rf "${local_build_dir}/"* "$build_dir/" >/dev/null 2>&1 || true else - echo "sshpass -p ${passwords[index]} scp -o StrictHostKeyChecking=no -r ${usernames[index]}@${hosts[index]}" + ( + cd "$local_build_dir" || exit 1 + find . -type f ! -path "./bin/*est*" -exec cp --parents {} "$build_dir/" \; + ) >/dev/null 2>&1 || true + fi + + if [ -d "$local_unit_test_log_dir" ] && [ -n "$(ls -A "$local_unit_test_log_dir" 2>/dev/null)" ]; then + cp -rf "${local_unit_test_log_dir}"* "$build_dir/" >/dev/null 2>&1 || true fi } @@ -174,7 +227,7 @@ function transfer_debug_dirs() { i=$((i + 1)) done - cd "$local_work_dir" + cd "$local_work_dir" || return 1 rm -rf debug.tar.gz tar -czf debug.tar.gz \ debugSan/build/bin/taos* \ @@ -206,11 +259,8 @@ function transfer_debug_dirs() { bash -c "${remote_cmd} rm -rf '${workdirs[index]}/debugSan'" bash -c "${remote_cmd} rm -rf '${workdirs[index]}/debugNoSan'" # transfer debug.tar.gz to remote - if [ -n "${passwords[index]}" ]; then - sshpass -p "${passwords[index]}" scp -o StrictHostKeyChecking=no -r debug.tar.gz "${usernames[index]}@${hosts[index]}:${workdirs[index]}/debug.tar.gz" - else - scp -o StrictHostKeyChecking=no -r debug.tar.gz "${usernames[index]}@${hosts[index]}:${workdirs[index]}/debug.tar.gz" - fi + scp_prefix=$(get_remote_scp_command "$index") + timeout "$(get_timeout_val)" $scp_prefix debug.tar.gz "${usernames[index]}@${hosts[index]}:${workdirs[index]}/debug.tar.gz" # untar debug.tar.gz to remote bash -c "${remote_cmd} \"tar -xzf '${workdirs[index]}/debug.tar.gz' -C '${workdirs[index]}' && rm -rf '${workdirs[index]}/debug.tar.gz'\"" fi @@ -355,8 +405,10 @@ function run_thread() { real_start_time=$(date +%s) # echo "cmd:${cmd}" if ! is_local_host "${hosts[index]}"; then + # 远程：cmd 已包含 timeout ssh，直接执行 $cmd >>"$case_log_file" 2>&1 else + # 本地：用 bash -c 执行以正确处理引号 bash -c "$cmd" >>"$case_log_file" 2>&1 fi ret=$? @@ -409,15 +461,13 @@ function run_thread() { echo "${hosts[index]} total time: ${total_time}s" >>"$case_log_file" # echo "$thread_no ${line} DONE" - local scpcmd="" + local scp_prefix="" local allure_report_results="${workdirs[index]}/tmp/thread_volume/$thread_no/allure-results" if ! is_local_host "${hosts[index]}"; then - scpcmd=$(get_remote_scp_command "$index") - cmd="$scpcmd:${allure_report_results}/* $log_dir/allure-results/" - bash -c "$cmd" >/dev/null 2>&1 || true + scp_prefix=$(get_remote_scp_command "$index") + timeout "$(get_timeout_val)" $scp_prefix "${usernames[index]}@${hosts[index]}:${allure_report_results}/*" "$log_dir/allure-results/" >/dev/null 2>&1 || true else - cmd="cp -rf ${allure_report_results}/* $log_dir/allure-results/" - bash -c "$cmd" >/dev/null 2>&1 || true + cp -rf ${allure_report_results}/* "$log_dir/allure-results/" >/dev/null 2>&1 || true fi echo "Save allure report results to $log_dir/allure-results/ from ${allure_report_results} with cmd: $cmd" if [ $ret -eq 0 ]; then @@ -433,11 +483,10 @@ function run_thread() { if [ "$(ls -A ${remote_coredump_dir} 2>/dev/null)" ]; then mkdir -p "${log_dir}"/"${case_file}".coredump if ! is_local_host "${hosts[index]}"; then - cmd="$scpcmd:${remote_coredump_dir}/* $log_dir/${case_file}.coredump/" + timeout "$(get_timeout_val)" $scp_prefix "${usernames[index]}@${hosts[index]}:${remote_coredump_dir}/*" "$log_dir/${case_file}.coredump/" >/dev/null 2>&1 || true else - cmd="cp -rf ${remote_coredump_dir}/* $log_dir/${case_file}.coredump/" + cp -rf ${remote_coredump_dir}/* "$log_dir/${case_file}.coredump/" >/dev/null 2>&1 || true fi - bash -c "$cmd" >/dev/null 2>&1 || true fi echo -e "$case_index \e[34m DONE <<<<< \e[0m ${case_info} \e[34m[${total_time}s]\e[0m \e[31m failed\e[0m" @@ -455,20 +504,7 @@ function run_thread() { echo -e "\e[34m corefiles: $corefile \e[0m" fi local build_dir=$log_dir/build_${hosts[index]} - local remote_build_dir="${workdirs[index]}/${DEBUGPATH}/build" - local remote_unit_test_log_dir="${workdirs[index]}/${DEBUGPATH}/Testing/Temporary/" - - if is_local_host "${hosts[index]}"; then - mkdir "$build_dir" >/dev/null - cmd="cp -rf ${remote_build_dir}/* ${build_dir}/" - echo "$cmd" - bash -c "$cmd" >/dev/null 2>&1 || true - if [ -d "${remote_unit_test_log_dir}" ] && [ "$(ls -A "${remote_unit_test_log_dir}" 2>/dev/null)" ]; then - cmd="cp -rf ${remote_unit_test_log_dir}/* ${build_dir}/" - echo "$cmd" - bash -c "$cmd" >/dev/null 2>&1 || true - fi - fi + save_build_artifacts "$build_dir" "$case_info" local remote_sim_dir="${workdirs[index]}/tmp/thread_volume/$thread_no" if ! is_local_host "${hosts[index]}"; then cmd="$runcase_script \"cd $remote_sim_dir; tar -czf sim.tar.gz sim\"" @@ -480,12 +516,10 @@ function run_thread() { local remote_sim_tar="${workdirs[index]}/tmp/thread_volume/$thread_no/sim.tar.gz" local remote_case_sql_file="${workdirs[index]}/tmp/thread_volume/$thread_no/${case_sql_file}" if ! is_local_host "${hosts[index]}"; then - cmd="$scpcmd:${remote_sim_tar} $log_dir/${case_file}.sim.tar.gz" - echo "scp sim.tar.gz cmd: $cmd" - bash -c "$cmd" >/dev/null 2>&1 || true + timeout "$(get_timeout_val)" $scp_prefix "${usernames[index]}@${hosts[index]}:${remote_sim_tar}" "$log_dir/${case_file}.sim.tar.gz" >/dev/null 2>&1 || true + echo "scp sim.tar.gz done" if [ "$(ls -A "$remote_case_sql_file" 2>/dev/null)" ];then - cmd="$scpcmd:${remote_case_sql_file} $log_dir/${case_file}.sql" - bash -c "$cmd" >/dev/null 2>&1 || true + timeout "$(get_timeout_val)" $scp_prefix "${usernames[index]}@${hosts[index]}:${remote_case_sql_file}" "$log_dir/${case_file}.sql" >/dev/null 2>&1 || true fi else cmd="cp -f ${remote_sim_tar} $log_dir/${case_file}.sim.tar.gz" diff --git a/test/ci/run_win_cases.py b/test/ci/run_win_cases.py index 2622ac63be49..30327002567c 100644 --- a/test/ci/run_win_cases.py +++ b/test/ci/run_win_cases.py @@ -7,6 +7,7 @@ import psutil import sys import zipfile +import signal from datetime import datetime logging.basicConfig(level=logging.INFO, @@ -14,6 +15,23 @@ datefmt='%Y-%m-%d %H:%M:%S') logger = logging.getLogger(__name__) failed_cases = 0 # 全局变量，记录失败用例数量 +exit_flag = False # 全局退出标志 +current_process = None # 当前运行的子进程 + +def signal_handler(signum, frame): + """处理 Ctrl+C 信号""" + global exit_flag, current_process + logger.info("接收到中断信号，正在退出...") + exit_flag = True + # 终止当前子进程 + if current_process is not None: + try: + current_process.terminate() + except: + pass + +# 注册信号处理 +signal.signal(signal.SIGINT, signal_handler) def get_git_commit_id(): try: @@ -55,12 +73,25 @@ def clean_taos_process(keywords=None): :param keywords: List[str]，用于匹配进程命令行的关键字列表。如果为 None，则默认匹配 'taos'。 """ if keywords is None: - keywords = ['taos'] # 默认关键字为 'taos' + keywords = ["taos", "taosd", "taosadapter", "taoskeeper", "taos-explorer", "taosx", "tmq_sim", "taosdump", "taosBenchmark" ] + + current_pid = os.getpid() for proc in psutil.process_iter(['pid', 'name', 'cmdline']): try: - # 检查进程命令行是否包含指定关键字 - if proc.info['cmdline'] and any(keyword in ' '.join(proc.info['cmdline']) for keyword in keywords): + if proc.info["pid"] == current_pid: + continue + + cmdline_parts = proc.info.get("cmdline") or [] + cmdline = " ".join(cmdline_parts).lower() + proc_name = (proc.info.get("name") or "").lower() + + # 跳过 Jenkins agent，避免断开 remoting channel 导致当前用例被系统回收 + if "agent.jar" in cmdline or "jenkins" in cmdline: + continue + + # 清理 taos 相关进程，同时避免误杀 Jenkins/agent + if any(keyword in proc_name for keyword in keywords) or proc_name.startswith("taos"): logger.debug(f"Found matching process: {proc.info}") proc.terminate() # 优雅终止进程 try: @@ -84,13 +115,17 @@ def zip_dir(dir_path, zip_path): rel_path = os.path.relpath(abs_path, dir_path) zipf.write(abs_path, rel_path) -def safe_rmtree(path, retries=5, delay=1): +def safe_rmtree(path, retries=10, delay=2): """ 安全删除目录，支持重试机制。 :param path: 要删除的目录路径。 :param retries: 重试次数。 :param delay: 每次重试之间的延迟时间（秒）。 """ + import sys + # Windows 上进程终止后句柄可能未立即释放，先等待一下 + if sys.platform == "win32": + time.sleep(1) for i in range(retries): try: if os.path.exists(path): @@ -101,29 +136,9 @@ def safe_rmtree(path, retries=5, delay=1): time.sleep(delay) raise Exception(f"Failed to remove {path} after {retries} retries.") -def safe_rmtree(path, retries=5, delay=1): - """ - 安全删除目录，支持重试机制。 - - :param path: 要删除的目录路径。 - :param retries: 重试次数。 - :param delay: 每次重试之间的延迟时间（秒）。 - """ - for i in range(retries): - try: - if os.path.exists(path): - shutil.rmtree(path) - break - except Exception as e: - logger.warning(f"Retry {i + 1}/{retries}: Failed to remove {path}. Error: {e}") - time.sleep(delay) - else: - raise Exception(f"Failed to remove {path} after {retries} retries.") - - def process_pytest_file(input_file, log_path="C:\\CI_logs", exclusion_file=os.path.join(os.path.dirname(__file__), "win_ignore_cases")): - global failed_cases # 声明使用全局变量 + global failed_cases, exit_flag # 声明使用全局变量 # 初始化统计变量 total_cases = 0 success_cases = 0 @@ -144,25 +159,55 @@ def process_pytest_file(input_file, log_path="C:\\CI_logs", shutil.rmtree(log_dir) os.makedirs(log_dir, exist_ok=True) + # 定义清理函数（失败时抛出异常，终止测试） + def cleanup_environment(phase=""): + """清理进程和目录，失败则抛出异常终止测试""" + logger.info(f"Cleaning up environment ({phase})...") + + # 1. 结束残留进程 + clean_taos_process() + + # 2. 等待句柄释放 + time.sleep(1) + + # 3. 删除 sim 目录，失败则终止 + if os.path.exists(work_dir): + try: + shutil.rmtree(work_dir) + logger.info(f"Removed {work_dir}") + except Exception as e: + logger.error(f"CRITICAL: Failed to remove {work_dir}: {e}") + raise RuntimeError(f"Cleanup failed: cannot remove work_dir") from e + with open(input_file, 'r', encoding="utf-8", errors="ignore") as f: for line in f: line = line.strip() - # 跳过空行和注释行 if not line or line.startswith('#'): continue - # 解析pytest命令 + # 解析 pytest 命令 - 格式: priority,rerunTimes,sanitizer(y/n),path,command + # 检查格式: 第三列是 y/n，第五列决定使用 ./ci/pytest.sh 还是 pytest + parts = line.split(',') + if len(parts) < 5: + logger.error(f"格式错误: 列数不足(应有5列,实际{len(parts)}列): {line}") + continue + + sanitizer = parts[2].strip() # 第三列: y 或 n + if sanitizer not in ('y', 'n'): + logger.error(f"格式错误: 第三列必须是 y 或 n, 实际为 '{sanitizer}': {line}") + continue + + # 根据原逻辑解析 if "ci/pytest.sh " in line: pytest_cmd = line.split("ci/pytest.sh ")[1] else: pytest_cmd = line.split(",,n,.,")[1] - # 确保是pytest命令 if not pytest_cmd.startswith("pytest"): logger.warning(f"异常pytest命令: {pytest_cmd}") continue - case_base_name = pytest_cmd.split(" ")[1] # 获取用例无参数名称用于与排除用例列表比对 + case_base_name = pytest_cmd.split(" ")[1] if case_base_name and len(exclusion_list) > 0 and case_base_name in exclusion_list: skipped_cases += 1 logger.info(f"Case {case_base_name} not support runnning on Windows. Skip test.") @@ -171,65 +216,130 @@ def process_pytest_file(input_file, log_path="C:\\CI_logs", rf.write(result_str) continue - case_name = pytest_cmd.split("/")[-1].replace(" ", "_") # 获取用例名称 + case_name = pytest_cmd.split("/")[-1].replace(" ", "_") total_cases += 1 log_file = os.path.join(log_dir, f"{case_name}.log") logger.info(f"Running case {pytest_cmd}") case_start = time.time() + return_code = None + # ========== 用例前清理（失败则终止） ========== try: - # 清理环境，kill残留进程，删除sim目录 - clean_taos_process() - if os.path.exists(work_dir): - safe_rmtree(work_dir) - # 执行pytest命令，设置超时为300秒（5分钟） - with open(log_file, 'w') as log: - process = subprocess.Popen( - pytest_cmd, + cleanup_environment("pre-case") + except RuntimeError as e: + logger.error(f"Pre-case cleanup failed, stopping test suite: {e}") + result_str = f"ERROR\t{pytest_cmd}\t\t\t0.00s\tPre-case cleanup failed: {e}\n" + with open(result_file, "a", encoding="utf-8") as rf: + rf.write(result_str) + break # 清理失败，终止整个测试 + + # ========== 执行用例 ========== + try: + if exit_flag: + logger.info("检测到退出标志，终止测试") + break + + global current_process + pytest_cmd_clean = f"{pytest_cmd} --clean" + + with open(log_file, 'w', buffering=1) as log: + current_process = subprocess.Popen( + pytest_cmd_clean, shell=True, stdout=log, stderr=subprocess.STDOUT, text=True ) - try: - return_code = process.wait(timeout=3000) - except subprocess.TimeoutExpired: - process.kill() + # 轮询等待 + waited = 0 + while waited < 1200: + ret = current_process.poll() + if ret is not None: + return_code = ret + break + if exit_flag: + logger.info("测试被中断") + current_process.kill() + current_process.wait() + sys.exit(130) + time.sleep(0.5) + waited += 0.5 + else: + # 超时 + current_process.kill() return_code = -1 logger.info(f"Case {pytest_cmd} running timeout, killed process.") + current_process = None + + # 确保日志刷盘 + if os.path.exists(log_file): + with open(log_file, 'a') as f: + f.flush() + try: + os.fsync(f.fileno()) + except: + pass + + except KeyboardInterrupt: + exit_flag = True + if current_process: + current_process.kill() + sys.exit(130) + except Exception as e: + return_code = -2 # 执行异常 + logger.error(f"Case execution error: {e}") - case_end = time.time() - execution_time = case_end - case_start - - if return_code == 0: - success_cases += 1 - os.remove(log_file) - logger.info(f"Case {pytest_cmd}: Success. Time cost: {execution_time:.2f}s") - result_str = f"SUCCESS\t{pytest_cmd}\t\t\t{execution_time:.2f}s\n" + # ========== 用例后清理（失败则终止） ========== + try: + cleanup_environment("post-case") + except RuntimeError as e: + logger.error(f"Post-case cleanup failed, stopping test suite: {e}") + # 记录当前用例结果（如果执行过） + if return_code is not None: + if return_code == 0: + result_str = f"SUCCESS\t{pytest_cmd}\t\t\t{time.time() - case_start:.2f}s\n" + else: + result_str = f"FAILED\t{pytest_cmd}\t\t\t{time.time() - case_start:.2f}s\n" else: - failed_cases += 1 - failed_case_list.append(pytest_cmd) - logger.info(f"Case {pytest_cmd} Failed. Time cost: {execution_time:.2f}s") - result_str = f"FAILED\t{pytest_cmd}\t\t\t{execution_time:.2f}s\n" - - except Exception as e: - case_end = time.time() - execution_time = case_end - case_start + result_str = f"ERROR\t{pytest_cmd}\t\t\t{time.time() - case_start:.2f}s\tExecution error\n" + with open(result_file, "a", encoding="utf-8") as rf: + rf.write(result_str) + break # 清理失败，终止整个测试 + + # ========== 记录用例结果 ========== + case_end = time.time() + execution_time = case_end - case_start + + if return_code == 0: + success_cases += 1 + os.remove(log_file) + logger.info(f"Case {pytest_cmd}: Success. Time cost: {execution_time:.2f}s") + result_str = f"SUCCESS\t{pytest_cmd}\t\t\t{execution_time:.2f}s\n" + elif return_code == -1: + failed_cases += 1 + failed_case_list.append(pytest_cmd) + logger.info(f"Case {pytest_cmd} Failed (timeout). Time cost: {execution_time:.2f}s") + result_str = f"FAILED\t{pytest_cmd}\t\t\t{execution_time:.2f}s\ttimeout\n" + elif return_code == -2: + failed_cases += 1 + failed_case_list.append(pytest_cmd) + logger.info(f"Case {pytest_cmd} Exception. Time cost: {execution_time:.2f}s") + result_str = f"ERROR\t{pytest_cmd}\t\t\t{execution_time:.2f}s\texecution error\n" + else: failed_cases += 1 failed_case_list.append(pytest_cmd) - logger.info(f"Case {total_cases} Exception: {str(e)}. Time cost: {execution_time:.2f}s") - result_str = f"ERROR\t{pytest_cmd}\t\t\t{execution_time:.2f}s\t{str(e)}\n" - # 每条用例执行完都写入结果文件 + logger.info(f"Case {pytest_cmd} Failed. Time cost: {execution_time:.2f}s") + result_str = f"FAILED\t{pytest_cmd}\t\t\t{execution_time:.2f}s\n" + with open(result_file, "a", encoding="utf-8") as rf: rf.write(result_str) - # 计算总执行时间 + # ========== 收尾 ========== end_time = time.time() total_execution_time = end_time - start_time - # 输出统计信息 logger.info("All cases run finished:") logger.info(f"Total cost time: {total_execution_time:.2f}s") logger.info(f"Total cases: {total_cases}") @@ -242,8 +352,12 @@ def process_pytest_file(input_file, log_path="C:\\CI_logs", logger.info(cmd) with open(result_file, "a", encoding="utf-8") as rf: - rf.write( - f"\nAll cases run finished:\nTotal cost time: {total_execution_time:.2f}s\nTotal cases: {total_cases}\nSuccess cases: {success_cases}\nFailed cases: {failed_cases}\nWindows skip cases: {skipped_cases}\n") + rf.write(f"\nAll cases run finished:\n") + rf.write(f"Total cost time: {total_execution_time:.2f}s\n") + rf.write(f"Total cases: {total_cases}\n") + rf.write(f"Success cases: {success_cases}\n") + rf.write(f"Failed cases: {failed_cases}\n") + rf.write(f"Windows skip cases: {skipped_cases}\n") if failed_cases > 0: rf.write("\nFailed cases list:\n") for cmd in failed_case_list: @@ -255,10 +369,6 @@ def process_pytest_file(input_file, log_path="C:\\CI_logs", shutil.move(f"{log_prefix}.zip", os.path.join(log_path, f"{log_prefix}.zip")) shutil.move(result_file, os.path.join(log_path, result_file)) - # 如果没有失败用例，删除日志目录 - # if failed_cases == 0 and os.path.exists(log_dir): - # shutil.rmtree(log_dir) - if __name__ == "__main__": import sys diff --git a/test/ci/win.pipline b/test/ci/win.pipline new file mode 100644 index 000000000000..f3be15af71bf --- /dev/null +++ b/test/ci/win.pipline @@ -0,0 +1,158 @@ +def pre_test_win(){ + bat ''' + hostname + ipconfig + set + date /t + time /t + taskkill /f /t /im python.exe + taskkill /f /t /im bash.exe + taskkill /f /t /im taosd.exe + rd /s /Q %WIN_INTERNAL_ROOT%\\debug || echo "no debug folder" + echo "clean environment done" + exit 0 + ''' + bat ''' + cd %WIN_INTERNAL_ROOT% + git config --global --add safe.directory %WIN_INTERNAL_ROOT% + git reset --hard + git remote prune origin + git fetch || git fetch + ''' + bat ''' + cd %WIN_COMMUNITY_ROOT% + git config --global --add safe.directory %WIN_COMMUNITY_ROOT% + git reset --hard + git remote prune origin + git fetch || git fetch + ''' + bat ''' + cd %WIN_INTERNAL_ROOT% + git checkout %BRANCH_NAME% + git remote prune origin + git clean -f + ''' + bat ''' + cd %WIN_COMMUNITY_ROOT% + git checkout %BRANCH_NAME% + git remote prune origin + git clean -f + ''' + bat ''' + cd %WIN_INTERNAL_ROOT% + git pull origin %BRANCH_NAME% + ''' + bat ''' + cd %WIN_COMMUNITY_ROOT% + git pull origin %BRANCH_NAME% + ''' + bat ''' + cd %WIN_INTERNAL_ROOT% + git branch + git log -5 + ''' + bat ''' + cd %WIN_COMMUNITY_ROOT% + git branch + git log -5 + ''' +} +def pre_test_build_win(skip_build=false) { + if (skip_build) { + echo "SKIP_BUILD is set to true, skipping build stage" + // 检查必要的二进制文件是否存在（使用双引号确保路径正确） + bat """ + if not exist "${WIN_INTERNAL_ROOT}\\debug\\build\\bin\\taosd.exe" ( + echo ERROR: taosd.exe not found in ${WIN_INTERNAL_ROOT}\\debug\\build\\bin\\ + echo Please ensure binaries exist or disable SKIP_BUILD + exit 1 + ) + echo Using existing build in ${WIN_INTERNAL_ROOT}\\debug\\ + """ + // 仍然需要安装 Python 依赖和复制 DLL + bat """ + cd ${WIN_TEST_ROOT} + python3 -m pip install --upgrade pip + python3 -m pip uninstall taospy -y + python3 -m pip install taospy + xcopy /e/y/i/f ${WIN_INTERNAL_ROOT}\\debug\\build\\bin\\taos.dll C:\\Windows\\System32 || echo "taos.dll already exists" + xcopy /e/y/i/f ${WIN_INTERNAL_ROOT}\\debug\\build\\bin\\taosnative.dll C:\\Windows\\System32 || echo "taosnative.dll already exists" + """ + return 1 + } + + bat ''' + echo "building ..." + time /t + cd %WIN_INTERNAL_ROOT% + mkdir debug + cd debug + rm -rf %WIN_INTERNAL_ROOT%/debug/* + time /t + call "C:\\Program Files\\Microsoft Visual Studio\\2022\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64 + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true -DBUILD_TOOLS=true -DBUILD_HTTP=internal " + time /t + cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true -DBUILD_TOOLS=true -DBUILD_HTTP=internal || exit 7 + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 10 " + time /t + jom -j 10 || exit 8 + jom install || exit 8 + time /t + ''' + bat ''' + cd %WIN_TEST_ROOT% + python3 -m pip install --upgrade pip + python3 -m pip uninstall taospy -y + python3 -m pip install taospy + xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\bin\\taos.dll C:\\Windows\\System32 + xcopy /e/y/i/f %WIN_INTERNAL_ROOT%\\debug\\build\\bin\\taosnative.dll C:\\Windows\\System32 + ''' + return 1 +} +def run_win_test_python() { + bat ''' + echo "windows test ..." + ls -l C:\\Windows\\System32\\taos.dll + ''' + bat ''' + time /t + cd %WIN_TEST_ROOT% + echo "python testing ..." + python3 ci\\run_win_cases.py ci\\cases.task + time /t + ''' +} +pipeline { + agent none + stages { + stage('run test') { + parallel { + stage('windows test') { + agent {label " u1-83 "} + environment{ + // 支持通过参数指定路径，默认使用 D:\TDinternal + // 如果 Jenkins 在子目录 checkout，请设置 WIN_INTERNAL_ROOT_PARAM 参数 + WIN_INTERNAL_ROOT="${params.WIN_INTERNAL_ROOT_PARAM ?: 'D:\\TDinternal'}" + WIN_COMMUNITY_ROOT="${params.WIN_INTERNAL_ROOT_PARAM ? params.WIN_INTERNAL_ROOT_PARAM + '\\community' : 'D:\\TDinternal\\community'}" + WIN_TEST_ROOT="${params.WIN_INTERNAL_ROOT_PARAM ? params.WIN_INTERNAL_ROOT_PARAM + '\\community\\test' : 'D:\\TDinternal\\community\\test'}" + WIN_SYSTEM_TEST_ROOT="${params.WIN_INTERNAL_ROOT_PARAM ? params.WIN_INTERNAL_ROOT_PARAM + '\\community\\test' : 'D:\\TDinternal\\community\\test'}" + BRANCH_NAME="${params.TEST_BRANCH}" + } + steps { + timeout(time: 4800, unit: 'MINUTES') { + script { + pre_test_win() + // 根据 SKIP_BUILD 参数决定是否跳过编译 + def skipBuild = params.SKIP_BUILD ?: false + pre_test_build_win(skipBuild) + catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { + run_win_test_python() + } + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/test/ci/win_ignore_cases b/test/ci/win_ignore_cases index 2120d5686d40..4c9309f874a6 100644 --- a/test/ci/win_ignore_cases +++ b/test/ci/win_ignore_cases @@ -1,143 +1,29 @@ -cases/22-Show/test_show_table_distributed_null.py -cases/uncatalog/army/alter/test_alter_config.py -cases/uncatalog/army/cluster/test_snapshot.py -cases/uncatalog/army/cmdline/test_taos_cli.py -# TD-37673 -cases/22-Functions/01-Scalar/test_scalar_all.py -cases/uncatalog/army/query/function/test_function.py -# TD-37674 -cases/22-Functions/02-Aggregate/test_agg_all.py -cases/uncatalog/army/query/function/test_interval_diff_tz.py -cases/12-DataCompression/test_compress_alter_table.py -cases/uncatalog/army/storage/test_compress_basic.py -cases/uncatalog/army/tools/benchmark/basic/test_stmt_insert_alltypes_same_min_max.py -cases/uncatalog/army/tools/benchmark/basic/test_stmt_sample_csv_json_subtable.py -cases/uncatalog/army/tools/benchmark/basic/test_stmt_sample_csv_json.py -# TD-37683 -cases/uncatalog/army/tools/benchmark/basic/test_insert_tag_order_sml.py -cases/uncatalog/army/tools/benchmark/basic/test_insert_tag_order_sql.py -cases/uncatalog/army/tools/benchmark/basic/test_insert_tag_order_stmt.py -cases/uncatalog/army/tools/benchmark/basic/test_insert_tag_order_stmt2.py -cases/uncatalog/army/tools/benchmark/basic/test_telnet_tcp.py -cases/uncatalog/army/tools/taosdump/native/test_taosdump_test_loose_mode.py -cases/uncatalog/army/tools/taosdump/native/test_taosdump_commandline.py -cases/uncatalog/system-test/0-others/test_mounts.py - # udf not supported in Windows CI -cases/uncatalog/system-test/0-others/test_udf_test.py -cases/uncatalog/system-test/0-others/test_udf_create.py -cases/uncatalog/system-test/0-others/test_udf_restart_taosd.py -cases/uncatalog/system-test/0-others/test_udf_cfg1.py -cases/uncatalog/system-test/0-others/test_udf_cfg2.py +cases/12-UDFs/test_udf_test.py +cases/12-UDFs/test_udf_create.py +cases/12-UDFs/test_udf_restart_taosd.py +cases/12-UDFs/test_udf_cfg1.py +cases/12-UDFs/test_udf_cfg2.py cases/uncatalog/system-test/0-others/test_udfpy_main.py cases/uncatalog/system-test/7-tmq/test_tmqUdf.py cases/uncatalog/system-test/7-tmq/test_tmqUdf_multCtb_snapshot0.py cases/uncatalog/system-test/7-tmq/test_tmqUdf_multCtb_snapshot1.py -# Stream not supported in Windows CI -cases/18-StreamProcessing/01-Snode/test_snode_mgmt_basic.py -cases/18-StreamProcessing/01-Snode/test_snode_mgmt_replica3.py -cases/18-StreamProcessing/01-Snode/test_snode_mgmt_replicas.py -cases/18-StreamProcessing/01-Snode/test_snode_params_alter_value.py -cases/18-StreamProcessing/01-Snode/test_snode_params_check_default.py -cases/18-StreamProcessing/01-Snode/test_snode_params_check_maxvalue.py -cases/18-StreamProcessing/01-Snode/test_snode_params_check_minvalue.py -cases/18-StreamProcessing/01-Snode/test_snode_privileges_recalc.py -cases/18-StreamProcessing/01-Snode/test_snode_privileges_stream.py -cases/18-StreamProcessing/01-Snode/test_snode_privileges_systable.py -cases/18-StreamProcessing/01-Snode/test_snode_privileges_twodb.py -cases/18-StreamProcessing/02-Stream/test_stream_check_name.py -cases/18-StreamProcessing/02-Stream/test_stream_long_name.py -cases/18-StreamProcessing/02-Stream/test_stream_no_snode.py -cases/18-StreamProcessing/02-Stream/test_stream_same_name.py -cases/18-StreamProcessing/03-TriggerMode/test_count_new.py -cases/18-StreamProcessing/03-TriggerMode/test_count.py -cases/18-StreamProcessing/03-TriggerMode/test_event_new.py -cases/18-StreamProcessing/03-TriggerMode/test_event.py -cases/18-StreamProcessing/03-TriggerMode/test_fill_history.py -cases/18-StreamProcessing/03-TriggerMode/test_period_1.py -cases/18-StreamProcessing/03-TriggerMode/test_sliding.py -cases/18-StreamProcessing/03-TriggerMode/test_state_new.py -cases/18-StreamProcessing/03-TriggerMode/test_state_disorder_update_new.py -cases/18-StreamProcessing/03-TriggerMode/test_state_window_close.py -cases/18-StreamProcessing/03-TriggerMode/test_state.py -cases/18-StreamProcessing/04-Options/test_abnormal_data_table.py -cases/18-StreamProcessing/04-Options/test_abnormal_data_vtable.py -cases/18-StreamProcessing/04-Options/test_meta_change_table.py -cases/18-StreamProcessing/04-Options/test_meta_change_vtable.py -cases/18-StreamProcessing/04-Options/test_options_abnormal.py -cases/18-StreamProcessing/04-Options/test_options_basic.py -cases/18-StreamProcessing/04-Options/test_options_ns.py -cases/18-StreamProcessing/04-Options/test_options_us.py -cases/18-StreamProcessing/04-Options/test_options_vtable.py -cases/18-StreamProcessing/05-Notify/test_notify.py -cases/18-StreamProcessing/06-ResultSaved/test_result_saved_comprehensive.py -cases/18-StreamProcessing/07-SubQuery/test_subquery_basic.py -cases/18-StreamProcessing/07-SubQuery/test_subquery_count_1.py -cases/18-StreamProcessing/07-SubQuery/test_subquery_count_2.py -cases/18-StreamProcessing/07-SubQuery/test_subquery_event.py -cases/18-StreamProcessing/07-SubQuery/test_subquery_sliding.py -cases/18-StreamProcessing/07-SubQuery/test_subquery_session.py -cases/18-StreamProcessing/07-SubQuery/test_subquery_state.py -cases/18-StreamProcessing/07-SubQuery/test_subquery_usage_restrict.py -cases/18-StreamProcessing/07-SubQuery/test_subquery_vtable_change.py -cases/18-StreamProcessing/08-Recalc/test_recalc_combined_options.py -cases/18-StreamProcessing/08-Recalc/test_recalc_delete_recalc.py -cases/18-StreamProcessing/08-Recalc/test_recalc_expired_time.py -cases/18-StreamProcessing/08-Recalc/test_recalc_ignore_disorder.py -cases/18-StreamProcessing/08-Recalc/test_recalc_manual_basic.py -cases/18-StreamProcessing/08-Recalc/test_recalc_manual_with_options.py -cases/18-StreamProcessing/08-Recalc/test_recalc_watermark.py -cases/18-StreamProcessing/20-UseCase/test_idmp_manager.py -cases/18-StreamProcessing/20-UseCase/test_idmp_meters.py -cases/18-StreamProcessing/20-UseCase/test_idmp_public.py -cases/18-StreamProcessing/20-UseCase/test_idmp_pv.py -cases/18-StreamProcessing/20-UseCase/test_idmp_tobacco.py -cases/18-StreamProcessing/20-UseCase/test_idmp_vehicle.py -cases/18-StreamProcessing/20-UseCase/test_idmp_yuxi.py -cases/18-StreamProcessing/20-UseCase/test_nevados.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_case4.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_case5.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_phase1.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_second_case1.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_second_case2.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_second_case3.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_second_case4.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_second_case6.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_second_case17.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_second_case18.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_second_case19.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_second_case19_bug1.py -cases/18-StreamProcessing/20-UseCase/test_three_gorges_second_case22.py -cases/18-StreamProcessing/20-UseCase/test_yuxi_TS_7152.py -cases/18-StreamProcessing/23-Compatibility/test_compatibility_cross_version.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_at_once.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_backquote_check.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_checkpoint_info.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_drop.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_empty_identifier.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_math_func.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_snode_restart_with_checkpoint.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_state_window.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_stream_basic.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_stream_multi_agg.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_string_func.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_taosdShell.py -cases/18-StreamProcessing/30-OldPyCases/test_oldcase_window_true_for.py -cases/18-StreamProcessing/31-OldTsimCases/test_oldcase_basic1.py -cases/18-StreamProcessing/31-OldTsimCases/test_oldcase_basic2.py -cases/18-StreamProcessing/31-OldTsimCases/test_oldcase_check.py -cases/18-StreamProcessing/31-OldTsimCases/test_oldcase_checkpoint.py -cases/18-StreamProcessing/31-OldTsimCases/test_oldcase_concat.py -cases/18-StreamProcessing/31-OldTsimCases/test_oldcase_continuewindowclose.py -cases/18-StreamProcessing/31-OldTsimCases/test_oldcase_state.py -cases/18-StreamProcessing/31-OldTsimCases/test_oldcase_twa.py -cases/18-StreamProcessing/23-Compatibility/test_compatibility_rolling_upgrade_all.py -cases/18-StreamProcessing/23-Compatibility/test_compatibility_backward_forward.py -cases/18-StreamProcessing/23-Compatibility/test_compatibility_rolling_upgrade.py -cases/uncatalog/system-test/7-tmq/test_tmq_td37436.py # MQTT bnode not supported in Windows CI -cases/40-DataSubscription/03-MQTT/test_mqtt_smoking.py -cases/40-DataSubscription/03-MQTT/test_mqtt_bnodes.py -cases/40-DataSubscription/03-MQTT/test_mqtt_qos.py -cases/40-DataSubscription/03-MQTT/test_mqtt_special.py -cases/40-DataSubscription/03-MQTT/test_mqtt_rb.py +cases/17-DataSubscription/03-MQTT/test_mqtt_smoking.py +cases/17-DataSubscription/03-MQTT/test_mqtt_bnodes.py +cases/17-DataSubscription/03-MQTT/test_mqtt_qos.py +cases/17-DataSubscription/03-MQTT/test_mqtt_special.py +cases/17-DataSubscription/03-MQTT/test_mqtt_rb.py + +# Unit Test +82-UnitTest/test.sh + +#docs Test +83-DocTest/c.sh +83-DocTest/python.sh +83-DocTest/node.sh +83-DocTest/csharp.sh +83-DocTest/jdbc.sh +83-DocTest/rust.sh +83-DocTest/go.sh +83-DocTest/test_R.sh \ No newline at end of file diff --git a/test/new_test_framework/utils/common.py b/test/new_test_framework/utils/common.py index 56dc017ae168..1223dcc2c3e0 100644 --- a/test/new_test_framework/utils/common.py +++ b/test/new_test_framework/utils/common.py @@ -2974,7 +2974,7 @@ def generate_query_result_file(self, test_case, idx, sql): # print(f"taosCmd:{taosCmd}, currentPath:{os.getcwd()}") os.system(taosCmd) return self.query_result_file - + def run_sql(self, sql, db): tdsql = self.newTdSql() if db: @@ -2994,14 +2994,11 @@ def execute_query_file(self, inputfile, max_workers=8): tdLog.info(f"Executing query file: {inputfile}") - with open(inputfile, 'r') as f: + with open(inputfile, "r") as f: lines = [line.strip() for line in f if line.strip()] # 假设第一行是 use 语句 - db = lines[0].split()[1].rstrip(';') - sql_lines = [ - line.replace('\\G', '').rstrip(';') + ';' - for line in lines[1:] - ] + db = lines[0].split()[1].rstrip(";") + sql_lines = [line.replace("\\G", "").rstrip(";") + ";" for line in lines[1:]] with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: executor.map(lambda sql: self.run_sql(sql, db), sql_lines) diff --git a/tools/taos-tools/src/taosdump.c b/tools/taos-tools/src/taosdump.c index 92749ec1366a..87664bd981cb 100644 --- a/tools/taos-tools/src/taosdump.c +++ b/tools/taos-tools/src/taosdump.c @@ -152,7 +152,6 @@ static struct argp_option options[] = { {"inspect", 'I', 0, 0, "inspect avro file content and print on screen.", 10}, {"no-escape", 'n', 0, 0, "No escape char '`'. Default is using it.", 10}, - {"restful", 'R', 0, 0, "Use RESTful interface to connect server.", 11}, {"cloud", 'C', "CLOUD_DSN", 0, OLD_DSN_DESC, 11}, {"timeout", 't', "SECONDS", 0, "The timeout seconds for " "websocket to interact."}, @@ -863,9 +862,6 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { } g_args.thread_num = atoi((const char *)arg); break; - case 'R': - warnPrint("%s\n", "'-R' is not supported, ignore this options."); - break; case 'C': case 'X': if (arg) {