diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 06c026bd7..bd19f0905 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -22,7 +22,7 @@ RUN apt update && apt install -y \ # Install Node.js RUN npm install -g n && n 20.12.0 - + # Install Solidity compiler using pipx (isolated environment) RUN pipx install solc-select && \ pipx ensurepath && \ @@ -40,10 +40,10 @@ RUN pipx install solc-select && \ cp -r /root/.solc-select/* /home/vscode/.solc-select/ && \ chown -R vscode:vscode /home/vscode/.solc-select -# Install Hardhat and related tools as root -RUN npm install -g hardhat@2.22.16 @nomicfoundation/hardhat-ethers@3.0.8 ethers@6.13.4 && \ - ln -sf /usr/local/lib/node_modules/hardhat/internal/cli/cli.js /usr/local/bin/hardhat && \ - chmod +x /usr/local/bin/hardhat +RUN npm install -g ethers@6.13.4 + +# Install cloc for code analysis +RUN npm install -g cloc # Install Foundry for Anvil (as root for global installation) RUN curl -L https://foundry.paradigm.xyz | bash && \ diff --git a/.devcontainer/README.md b/.devcontainer/README.md index f0079a7b7..c27c20c33 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -18,57 +18,47 @@ The dev container provides a consistent development environment with caching to ## Cache System -The container uses a simple caching system: +The container uses a conservative caching approach to prevent cache corruption issues: -1. **Host Cache Directories**: Created on the host and mounted into the container - - `/cache/vscode-cache` → `/home/vscode/.cache` - - `/cache/vscode-config` → `/home/vscode/.config` - - `/cache/vscode-data` → `/home/vscode/.local/share` - - `/cache/vscode-bin` → `/home/vscode/.local/bin` - - `/cache/*` → Tool-specific cache directories +1. **Local Cache Directories**: Each container instance maintains its own cache directories -2. **Package Cache Symlinks**: Created inside the container by project-setup.sh - - Each package's cache directory is symlinked to a subdirectory in `/cache/hardhat` + - `vscode-cache` → `/home/vscode/.cache` (VS Code cache) + - `vscode-config` → `/home/vscode/.config` (VS Code configuration) + - `vscode-data` → `/home/vscode/.local/share` (VS Code data) + - `vscode-bin` → `/home/vscode/.local/bin` (User binaries) -## Setup Instructions - -### 1. Host Setup (One-time) - -Before starting the dev container for the first time, run the included host setup script to create the necessary cache directories on the host: - -```bash -sudo /git/graphprotocol/contracts/.devcontainer/host-setup.sh -``` +2. **Safe Caches Only**: Only caches that won't cause cross-branch issues are configured -This script creates all required cache directories on the host, including: + - GitHub CLI: `/home/vscode/.cache/github` + - Python packages: `/home/vscode/.cache/pip` -- Standard VS Code directories (for .cache, .config, etc.) -- Tool-specific cache directories (for npm, yarn, cargo, etc.) +3. **Intentionally Not Cached**: These tools use their default cache locations to avoid contamination + - NPM, Yarn (different dependency versions per branch) + - Foundry, Solidity (different compilation artifacts per branch) + - Hardhat (different build artifacts per branch) -The script is idempotent and can be run multiple times without issues. +## Setup Instructions -### 2. Start the Dev Container +### Start the Dev Container -After creating the cache directories, you can start the dev container: +To start the dev container: 1. Open VS Code 2. Use the "Remote-Containers: Open Folder in Container" command -3. Select the repository directory +3. Select the repository directory (for example `/git/graphprotocol/contracts`) When the container starts, the `project-setup.sh` script will automatically run and: -- Create package-specific cache directories -- Set up symlinks for package cache directories - Install project dependencies using yarn - Configure Git to use SSH signing with your forwarded SSH key -- Source shell customizations if available in PATH (currently depends on base image configuration) +- Source shell customizations if available in PATH ## Environment Variables Environment variables are defined in two places: 1. **docker-compose.yml**: Contains most of the environment variables for tools and caching -2. **Environment File**: Personal settings are stored in `/opt/configs/graphprotocol/contracts.env` +2. **Environment File**: Personal settings are stored in `/opt/configs/graphprotocol/contracts.env` on the host ### Git Configuration @@ -84,11 +74,13 @@ These environment variables are needed for Git commit signing to work properly. ## Troubleshooting -If you encounter permission denied errors when trying to access directories, make sure you've run the `host-setup.sh` script on the host before starting the container: +### Cache Issues -```bash -sudo .devcontainer/host-setup.sh -``` +If you encounter build or compilation issues that seem related to cached artifacts: + +1. **Rebuild the container**: This will start with fresh local caches +2. **Clean project caches**: Run `yarn clean` to clear project-specific build artifacts +3. **Clear node modules**: Delete `node_modules` and run `yarn install` again ### Git SSH Signing Issues diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8bb295680..8d7fb643d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,44 +1,42 @@ { - "name": "graph contracts", - "dockerComposeFile": [ - "docker-compose.yml" - ], - "service": "dev-graph-contracts", - "features": { - "ghcr.io/devcontainers/features/git:1": { - "configureGitHubCLI": true, - "gitCredentialHelper": "cache" - }, - "ghcr.io/devcontainers/features/github-cli:1": {}, - "ghcr.io/devcontainers/features/common-utils:2.5.3": {}, - "ghcr.io/devcontainers/features/node:1": { - "version": "20" - }, - "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {} - }, - "postCreateCommand": ".devcontainer/project-setup.sh", - "remoteUser": "vscode", - "workspaceFolder": "${localWorkspaceFolder}", - "customizations": { - "vscode": { - "extensions": [ - "rust-lang.rust-analyzer", - "tamasfe.even-better-toml", - "usernamehw.errorlens", - "yzhang.markdown-all-in-one", - "DavidAnson.vscode-markdownlint", - "shd101wyy.markdown-preview-enhanced", - "bierner.markdown-preview-github-styles", - "Gruntfuggly.todo-tree", - "ms-azuretools.vscode-docker", - "donjayamanne.githistory", - "eamodio.gitlens", - "fill-labs.dependi", - "streetsidesoftware.code-spell-checker", - "Augment.vscode-augment", - "NomicFoundation.hardhat-solidity", - "foundry-rs.foundry-vscode" - ] - } - } + "name": "graph contracts", + "dockerComposeFile": ["docker-compose.yml"], + "service": "dev-graph-contracts", + "features": { + "ghcr.io/devcontainers/features/git:1": { + "configureGitHubCLI": true, + "gitCredentialHelper": "cache" + }, + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers/features/common-utils:2.5.3": {}, + "ghcr.io/devcontainers/features/node:1": { + "version": "20" + }, + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {} + }, + "postCreateCommand": ".devcontainer/project-setup.sh", + "remoteUser": "vscode", + "workspaceFolder": "${localWorkspaceFolder}", + "customizations": { + "vscode": { + "extensions": [ + "rust-lang.rust-analyzer", + "tamasfe.even-better-toml", + "usernamehw.errorlens", + "yzhang.markdown-all-in-one", + "DavidAnson.vscode-markdownlint", + "shd101wyy.markdown-preview-enhanced", + "bierner.markdown-preview-github-styles", + "Gruntfuggly.todo-tree", + "ms-azuretools.vscode-docker", + "donjayamanne.githistory", + "eamodio.gitlens", + "fill-labs.dependi", + "streetsidesoftware.code-spell-checker", + "Augment.vscode-augment", + "NomicFoundation.hardhat-solidity", + "foundry-rs.foundry-vscode" + ] + } + } } diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 293c78c92..794054a87 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,62 +1,43 @@ services: dev-graph-contracts: - container_name: dev-graph-contracts build: context: . dockerfile: Dockerfile - x-bake: - cache-from: - - type=local,src=/cache/docker - cache-to: - - type=local,dest=/cache/docker,mode=max env_file: - /opt/configs/graphprotocol/contracts.env environment: - # Cache directories - - FOUNDRY_CACHE_DIR=/cache/foundry - - SOLC_CACHE=/cache/solidity + # Essential for large builds + - NODE_OPTIONS=--max-old-space-size=4096 + + # Clean development environment + - PYTHONDONTWRITEBYTECODE=1 - # GitHub - - GH_CONFIG_DIR=/cache/github + # Disable interactive prompts + - COREPACK_ENABLE_DOWNLOAD_PROMPT=0 - # XDG standard directories + # Standard user directories - XDG_CACHE_HOME=/home/vscode/.cache - XDG_CONFIG_HOME=/home/vscode/.config - XDG_DATA_HOME=/home/vscode/.local/share - # Tool-specific settings - - HARDHAT_CACHE_DIR=/cache/hardhat - - HARDHAT_ANALYTICS=true - - HARDHAT_NO_ANALYTICS_PROMPT=true - - # Node.js settings - - NPM_CONFIG_CACHE=/cache/npm - - YARN_CACHE_FOLDER=/cache/yarn - - NODE_OPTIONS=--max-old-space-size=4096 + # Safe caches (won't cause cross-branch issues) + - GH_CONFIG_DIR=/home/vscode/.cache/github + - PIP_CACHE_DIR=/home/vscode/.cache/pip - # Python settings - - PIP_CACHE_DIR=/cache/pip - - PYTHONPYCACHEPREFIX=/cache/pycache - - PYTHONDONTWRITEBYTECODE=1 - networks: - - shared + # Note: NPM, Yarn, Foundry, and Solidity caches are intentionally not set + # to avoid cross-branch contamination. Tools will use their default locations. volumes: - # Mount cache directory - - /cache:/cache - - # System-specific mounts that need specific locations - - /cache/apt:/var/cache/apt - - /cache/apt-lib:/var/lib/apt - # Git repo root - /git:/git - # Local cache directories for XDG standards - - /cache/vscode-cache:/home/vscode/.cache - - /cache/vscode-config:/home/vscode/.config - - /cache/vscode-data:/home/vscode/.local/share - - /cache/vscode-bin:/home/vscode/.local/bin - -networks: - shared: - external: true + # Local directories for user data (keep local to container) + - vscode-cache:/home/vscode/.cache + - vscode-config:/home/vscode/.config + - vscode-data:/home/vscode/.local/share + - vscode-bin:/home/vscode/.local/bin + +volumes: + vscode-cache: + vscode-config: + vscode-data: + vscode-bin: diff --git a/.devcontainer/project-setup.sh b/.devcontainer/project-setup.sh index 05b05dfb0..313cd25d8 100755 --- a/.devcontainer/project-setup.sh +++ b/.devcontainer/project-setup.sh @@ -11,73 +11,15 @@ REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" echo "Script directory: $SCRIPT_DIR" echo "Repository root: $REPO_ROOT" -# Check if cache directories exist -echo "Checking if cache directories exist..." +# Set up local user directories with proper permissions +echo "Setting up local user directories..." -# Required cache directories -REQUIRED_DIRS=( - "/cache/hardhat" - "/cache/npm" - "/cache/yarn" -) +# Ensure all user directories exist and have proper ownership +sudo mkdir -p /home/vscode/.cache /home/vscode/.config /home/vscode/.local/share /home/vscode/.local/bin +sudo chown -R vscode:vscode /home/vscode/.cache /home/vscode/.config /home/vscode/.local +sudo chmod -R 755 /home/vscode/.cache /home/vscode/.config /home/vscode/.local -# Check if required directories exist -missing_dirs=() -for dir in "${REQUIRED_DIRS[@]}"; do - if [ ! -d "$dir" ]; then - missing_dirs+=("$dir") - fi -done - -# If any required directories are missing, show a warning -# Note: With set -u, we need to ensure missing_dirs is always initialized -if [ "${#missing_dirs[@]}" -gt 0 ]; then - echo "WARNING: The following required cache directories are missing:" - for dir in "${missing_dirs[@]}"; do - echo " - $dir" - done - echo "Please run the host setup script before starting the container:" - echo " sudo .devcontainer/host-setup.sh" - echo "Continuing anyway, but you may encounter issues..." -fi - -# Set up cache symlinks -echo "Setting up cache symlinks..." - -# Function to create symlinks for package cache directories -setup_cache_symlink() { - # With set -u, we need to ensure all parameters are provided - if [ "$#" -ne 1 ]; then - echo "Error: setup_cache_symlink requires exactly 1 argument (package_name)" - return 1 - fi - - local package_name=$1 - local cache_path="$REPO_ROOT/packages/${package_name}/cache" - local cache_dest="/cache/hardhat/${package_name}" - - # Skip if the package directory doesn't exist - if [ ! -d "$REPO_ROOT/packages/${package_name}" ]; then - return - fi - - # Create the package-specific cache directory if it doesn't exist - if [ ! -d "$cache_dest" ]; then - echo "Creating package-specific cache directory: $cache_dest" - mkdir -p "$cache_dest" - chmod -R 777 "$cache_dest" - fi - - # Create the symlink (will replace existing symlink if it exists) - ln -sf "$cache_dest" "$cache_path" - echo "Created symlink for ${package_name} cache" -} - -# Set up cache symlinks for main packages -setup_cache_symlink "contracts" -setup_cache_symlink "horizon" -setup_cache_symlink "subgraph-service" -setup_cache_symlink "data-edge" +echo "User directories set up with proper permissions" # Install project dependencies echo "Installing project dependencies..." @@ -131,7 +73,6 @@ else fi # Set up Git SSH signing -echo "Setting up Git SSH signing..." if [ -f "$SCRIPT_DIR/setup-git-signing.sh" ]; then "$SCRIPT_DIR/setup-git-signing.sh" else diff --git a/.gitignore b/.gitignore index 45e325072..08bb97ade 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,7 @@ addresses-fork.json # Forge artifacts cache_forge +forge-artifacts/ # Graph client .graphclient diff --git a/scripts/count-changes b/scripts/count-changes new file mode 100755 index 000000000..db079f24d --- /dev/null +++ b/scripts/count-changes @@ -0,0 +1,76 @@ +#!/bin/bash + +# Count changes in Solidity files between two branches, categorizing by code, comments, and blank lines +set -euo pipefail + +# Check if cloc is installed +if ! command -v cloc &> /dev/null; then + echo "Error: cloc is not installed. Install with: sudo npm install -g cloc" + exit 1 +fi + +# Define the branches to compare +BASE_BRANCH="${1:-main}" # Default to 'main' if no argument provided +CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + +echo "Comparing Solidity changes between $BASE_BRANCH and $CURRENT_BRANCH..." +echo + +# Get all modified Solidity contract files +ALL_FILES=$(git diff --name-only "$BASE_BRANCH" -- 'packages/*/contracts/**/*.sol') + +# Get excluded files (mock and test files) +EXCLUDED_FILES=$(echo "$ALL_FILES" | grep -E '(/mocks?/|/tests?/|Mock[A-Z].*\.sol$|.*Test\.sol$|Test[A-Z].*\.sol$)' || true) + +# Get included files (non-mock, non-test files) +FILES=$(echo "$ALL_FILES" | grep -v -E '(/mocks?/|/tests?/|Mock[A-Z].*\.sol$|.*Test\.sol$|Test[A-Z].*\.sol$)' || true) + +# Check if there are any files to process +if [ -z "$FILES" ]; then + echo "No Solidity files changed between $BASE_BRANCH and $CURRENT_BRANCH." + exit 0 +fi + +echo "Found changed Solidity files (excluding mocks and tests):" +echo "$FILES" | sed 's/^/- /' +echo + +# Display excluded files if any +if [ -n "$EXCLUDED_FILES" ]; then + echo "Excluded mock and test files:" + echo "$EXCLUDED_FILES" | sed 's/^/- /' + echo +fi + +# Create directories for diff analysis +TEMP_DIR=$(mktemp -d) +trap 'rm -rf "$TEMP_DIR"' EXIT +mkdir -p "$TEMP_DIR/old" +mkdir -p "$TEMP_DIR/new" + +# Extract all changed files +echo "$FILES" | while IFS= read -r file; do + if [ -f "$file" ]; then + # Create directory structure + dir=$(dirname "$file") + mkdir -p "$TEMP_DIR/new/$dir" + mkdir -p "$TEMP_DIR/old/$dir" + + # Copy current version + cp "$file" "$TEMP_DIR/new/$file" + + # Get old version if it exists + if git show "$BASE_BRANCH:$file" &>/dev/null; then + git show "$BASE_BRANCH:$file" > "$TEMP_DIR/old/$file" + fi + fi +done + +# Run cloc diff on all files +echo "Summary of changes (excluding mock and test files):" +echo "================================================" +cloc --diff "$TEMP_DIR/old" "$TEMP_DIR/new" --include-lang=Solidity --quiet + +echo +echo "Note: This analysis only counts changes in Solidity files, excluding mock and test contracts." +echo "The 'same' category shows lines that were unchanged in modified files."