extractor.py

import re
import json
from filters import is_known_package, NODE_BUILTINS, CSS_PROPERTIES

# ── Regex patterns for extracting package names ──────────────────────────

# Pattern 1: node_modules paths (webpack bundles — highest value)
RE_NODE_MODULES = re.compile(
    r'node_modules/(@[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+|[a-zA-Z0-9._-]+)'
)

# Pattern 2: require() calls
RE_REQUIRE = re.compile(
    r'''require\s*\(\s*['"](@[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+|[a-zA-Z0-9][a-zA-Z0-9._-]*)['"]'''
)

# Pattern 3: ES import from statements (must have import before from)
# Matches: import x from "pkg", import {x} from "pkg", export {x} from "pkg"
# Does NOT match: "inherit properties from "gfeedfetcher" class" (comment text)
RE_IMPORT_FROM = re.compile(
    r'''(?:import|export)\s+.*?\s+from\s+['"](@[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+|[a-zA-Z0-9][a-zA-Z0-9._-]*)['"]'''
)

# Pattern 4: ES import statements (import "pkg" — side-effect import)
RE_IMPORT_DIRECT = re.compile(
    r'''import\s+['"](@[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+|[a-zA-Z0-9][a-zA-Z0-9._-]*)['"]'''
)

# Pattern 5: Dynamic imports
RE_DYNAMIC_IMPORT = re.compile(
    r'''import\s*\(\s*['"](@[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+|[a-zA-Z0-9][a-zA-Z0-9._-]*)['"]'''
)

# Pattern 6: Webpack chunk/module maps
RE_WEBPACK_MODULE = re.compile(
    r'''["']/?node_modules/(@[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+|[a-zA-Z0-9._-]+)'''
)

# Pattern 7: Dependency entry (ONLY used inside parsed dep blocks, never blind)
RE_DEP_ENTRY = re.compile(
    r'''"(@[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+|[a-zA-Z0-9][a-zA-Z0-9._-]*)"\s*:\s*"([\^~>=<*|]?\s*\d+\.\d+[\d.]*)'''
)

# Pattern 8: Python requirements.txt format
RE_PYTHON_REQ = re.compile(
    r'^([a-zA-Z0-9][a-zA-Z0-9._-]*)\s*(?:[=!<>~]=|[<>])', re.MULTILINE
)

# Pattern 9: Gemfile gem declarations
RE_RUBY_GEM = re.compile(
    r'''gem\s+['"]([a-zA-Z0-9][a-zA-Z0-9._-]*)['"]'''
)

# Pattern 10: setup.py install_requires
RE_SETUP_PY = re.compile(
    r'''['"]([a-zA-Z0-9][a-zA-Z0-9._-]*)(?:[=!<>~]=.*?)?['"]'''
)

# Pattern 11: pyproject.toml dependencies
RE_PYPROJECT = re.compile(
    r'''['"]([a-zA-Z0-9][a-zA-Z0-9._-]*)(?:\s*[=!<>~]=.*?)?['"]'''
)

# ── Embedded dependency block keywords (for JS files) ────────────────────
# When we find these keywords in JS content, we extract the JSON block after
# them and parse it to get package names + versions.

JS_DEP_BLOCK_KEYS = [
    # npm / Node.js
    "dependencies", "devDependencies", "peerDependencies",
    "optionalDependencies", "bundleDependencies", "bundledDependencies",
    "resolutions", "overrides",
    # PHP / Composer
    "require", "require-dev",
]

# ── Manifest dependency block keywords ───────────────────────────────────

DEP_BLOCK_KEYWORDS = [
    '"dependencies"', "'dependencies'",
    '"devDependencies"', "'devDependencies'",
    '"peerDependencies"', "'peerDependencies'",
    '"optionalDependencies"', "'optionalDependencies'",
    '"bundleDependencies"', "'bundleDependencies'",
    '"bundledDependencies"', "'bundledDependencies'",
    "install_requires",
    "setup_requires",
    "tests_require",
    "extras_require",
]


def _clean_package_name(raw_name):
    """Clean a raw extracted string into a valid package name.

    Returns the cleaned name or None if invalid.
    """
    if not raw_name:
        return None

    name = raw_name.strip()

    # Strip quotes
    name = name.strip("'\"")

    # Strip trailing colons, commas, semicolons
    name = name.rstrip(":,;")

    # Strip version suffixes (e.g., @^1.2.3, @latest, @1.0.0)
    # But preserve scoped names like @scope/pkg
    if name.startswith("@") and "/" in name:
        # Scoped package: @scope/pkg@version → @scope/pkg
        scope_and_name = name.split("/", 1)
        if "@" in scope_and_name[1]:
            scope_and_name[1] = scope_and_name[1].split("@")[0]
        name = "/".join(scope_and_name)
    elif "@" in name and not name.startswith("@"):
        # Unscoped with version: pkg@1.0.0 → pkg
        name = name.split("@")[0]

    # Strip trailing path segments: pkg/dist/index.js → pkg
    # For scoped: @scope/pkg/dist/index.js → @scope/pkg
    if name.startswith("@") and "/" in name:
        parts = name.split("/")
        if len(parts) > 2:
            name = parts[0] + "/" + parts[1]
    elif "/" in name:
        name = name.split("/")[0]

    # Strip any remaining whitespace
    name = name.strip()

    # Validate: must not be empty
    if not name:
        return None

    # Validate: minimum length
    if len(name) < 2 and not name.startswith("@"):
        return None

    # Validate: max length (npm limit is 214)
    if len(name) > 214:
        return None

    # Validate: must not start with . or _
    if name.startswith(".") or name.startswith("_"):
        return None

    # Validate: no spaces
    if " " in name:
        return None

    # Validate: only allowed characters
    # npm: lowercase, numbers, hyphens, dots, underscores, @, /
    if not re.match(r'^(@[a-zA-Z0-9._-]+/)?[a-zA-Z0-9._-]+$', name):
        return None

    # Convert to lowercase (npm packages are always lowercase)
    name = name.lower()

    # Block obvious non-packages
    # Filenames with extensions (popper.553719d0.js)
    if name.endswith(".js") or name.endswith(".ts") or name.endswith(".css") or name.endswith(".json") or name.endswith(".html"):
        return None
    # Literal directory name
    if name == "node_modules":
        return None

    return name


def _extract_json_block(content, start_pos):
    """Extract a JSON object {...} or array [...] starting from start_pos.

    Returns the parsed object/list or None.
    """
    # Find the opening brace/bracket after the key
    i = start_pos
    while i < len(content) and content[i] in ' \t\n\r:':
        i += 1

    if i >= len(content):
        return None

    opener = content[i]
    if opener == '{':
        closer = '}'
    elif opener == '[':
        closer = ']'
    else:
        return None

    # Track nesting to find the matching closer
    depth = 0
    j = i
    while j < len(content):
        ch = content[j]
        if ch == '"':
            # Skip string content
            j += 1
            while j < len(content) and content[j] != '"':
                if content[j] == '\\':
                    j += 1  # skip escaped char
                j += 1
        elif ch == opener:
            depth += 1
        elif ch == closer:
            depth -= 1
            if depth == 0:
                block = content[i:j + 1]
                try:
                    return json.loads(block)
                except (json.JSONDecodeError, ValueError):
                    return None
        j += 1

    return None


def _extract_dep_blocks(content):
    """Find embedded dependency blocks in JS content and extract packages with versions.

    Searches for keywords like "dependencies":{...}, "devDependencies":{...} etc.
    Parses the JSON block to get exact package names and versions.

    Returns dict of {package_name: version_string}.
    """
    packages = {}

    for key in JS_DEP_BLOCK_KEYS:
        # Search for "key": or 'key': patterns
        for quote in ['"', "'"]:
            pattern = f'{quote}{key}{quote}'
            search_start = 0
            while True:
                pos = content.find(pattern, search_start)
                if pos == -1:
                    break

                # Move past the key and find the colon
                after_key = pos + len(pattern)
                parsed = _extract_json_block(content, after_key)

                if isinstance(parsed, dict):
                    for pkg_name, version in parsed.items():
                        # Real dep values are version strings ("^1.0.0", "~2.3", "*", "latest")
                        # Reject non-strings (JSON Schema objects/arrays/bools)
                        if not isinstance(version, str):
                            continue
                        # Reject values that aren't version-like
                        # Valid: "^1.0.0", "~2.3", ">=1.0", "*", "latest", "1.x", "npm:pkg@1"
                        # Invalid: "object", "string", "array", "boolean" (JSON Schema types)
                        v = version.strip()
                        if v and not re.match(r'^[\^~>=<*|0-9]|^latest$|^next$|^npm:', v):
                            continue
                        cleaned = _clean_package_name(pkg_name)
                        if cleaned:
                            packages[cleaned] = version
                elif isinstance(parsed, list):
                    # bundleDependencies is an array of strings
                    for item in parsed:
                        if isinstance(item, str):
                            cleaned = _clean_package_name(item)
                            if cleaned:
                                packages[cleaned] = ""

                search_start = after_key

    return packages


def _is_bundled_js(content):
    """Detect if a JS file is a bundle (webpack, AMD, Browserify, etc.).

    In bundles, ALL original require/import/export statements are compiled
    away by the bundler. Any remaining require("name") or import "name"
    text is inside STRING LITERALS (error messages, docs, etc.) — not real
    code. For example, webpack compiles import to __webpack_require__(id).

    Real npm packages in bundles show up via:
    - node_modules/ paths in webpack module maps (always extracted)
    - Embedded "dependencies":{...} blocks (always extracted)

    Returns True if the file is a bundle (skip require AND import patterns).
    """
    # Large files are always bundles — no raw source file served from a
    # website is this big. 200K chars catches most bundles.
    if len(content) > 200000:
        return True

    # Module system / bundler patterns that mean require/import is NOT real code.
    # Python's `in` on strings uses fast C-level search — microseconds per check.
    skip_patterns = [
        # AMD / RequireJS
        "define.amd",
        "define([",
        'define("',
        "define('",
        'typeof define==="function"',
        "typeof define === 'function'",
        'typeof define=="function"',
        # Webpack (compiles require/import → __webpack_require__(id))
        "__webpack_require__",
        "__webpack_modules__",
        "webpackChunk",
        "webpackJsonp",
        # SystemJS
        "System.register(",
        # Browserify
        "_dereq_(",
        # Rollup / generic bundler markers
        "Object.defineProperty(exports",
        # Minified bundles (single-letter variable names in function chains)
        "!function(e,t){",
        "!function(t,e){",
        "!function(e,n){",
        "(function(e,t){",
    ]

    for pattern in skip_patterns:
        if pattern in content:
            return True

    return False


def extract_from_js(content):
    """Extract package names from JavaScript file content.

    Uses two approaches:
    1. Code references: node_modules paths, import/export, require() (safe)
    2. Embedded dep blocks: "dependencies":{...} parsed as JSON (accurate)

    In bundled JS (webpack/AMD/Browserify), require() AND import/export are
    skipped — bundlers compile them away, so any remaining text with these
    keywords is inside string literals (error messages, docs), not real code.
    Only node_modules/ paths and embedded dep blocks are used in bundles.

    Returns a dict of {package_name: version_string}.
    """
    packages = {}

    is_bundle = _is_bundled_js(content)

    # ── Method 1: Code references ──

    # node_modules/ paths — always safe regardless of module system
    for regex in [RE_NODE_MODULES, RE_WEBPACK_MODULE]:
        for match in regex.finditer(content):
            name = _clean_package_name(match.group(1))
            if name:
                if name not in packages:
                    packages[name] = ""

    # require() and import/export — SKIP in bundles
    # Bundlers compile these away; any remaining text is inside string literals
    if not is_bundle:
        for match in RE_REQUIRE.finditer(content):
            name = _clean_package_name(match.group(1))
            if name:
                if name not in packages:
                    packages[name] = ""

        for regex in [RE_IMPORT_FROM, RE_IMPORT_DIRECT, RE_DYNAMIC_IMPORT]:
            for match in regex.finditer(content):
                name = _clean_package_name(match.group(1))
                if name:
                    if name not in packages:
                        packages[name] = ""

    # ── Method 2: Embedded dependency blocks (JSON parsed) ──

    dep_block_packages = _extract_dep_blocks(content)
    for name, version in dep_block_packages.items():
        packages[name] = version  # overwrite with version if found

    # ── Filter ──

    filtered = {}
    for name, version in packages.items():
        if not is_known_package(name) and name not in CSS_PROPERTIES:
            filtered[name] = version

    return filtered


def extract_from_package_json(content):
    """Extract package names from package.json content.

    Returns a dict of {package_name: version_string}.
    """
    packages = {}

    # Primary: parse JSON properly
    try:
        data = json.loads(content)
        dep_keys = ["dependencies", "devDependencies", "peerDependencies",
                     "optionalDependencies", "resolutions", "overrides"]
        for key in dep_keys:
            deps = data.get(key, {})
            if isinstance(deps, dict):
                for name, version in deps.items():
                    cleaned = _clean_package_name(name)
                    if cleaned:
                        packages[cleaned] = str(version) if version else ""
        # bundleDependencies is an array
        for key in ["bundleDependencies", "bundledDependencies"]:
            deps = data.get(key, [])
            if isinstance(deps, list):
                for name in deps:
                    if isinstance(name, str):
                        cleaned = _clean_package_name(name)
                        if cleaned and cleaned not in packages:
                            packages[cleaned] = ""
    except (json.JSONDecodeError, AttributeError):
        # Fallback: regex extraction if JSON parsing fails
        for match in RE_DEP_ENTRY.finditer(content):
            name = _clean_package_name(match.group(1))
            version = match.group(2) if match.lastindex >= 2 else ""
            if name:
                packages[name] = version

    # Also extract from node_modules refs if present
    for match in RE_NODE_MODULES.finditer(content):
        name = _clean_package_name(match.group(1))
        if name and name not in packages:
            packages[name] = ""

    # Filter
    filtered = {}
    for name, version in packages.items():
        if not is_known_package(name) and name not in CSS_PROPERTIES:
            filtered[name] = version

    return filtered


def extract_from_requirements_txt(content):
    """Extract package names from requirements.txt content.

    Returns a dict of {package_name: version_string}.
    """
    packages = {}

    for line in content.splitlines():
        line = line.strip()
        if not line or line.startswith("#") or line.startswith("-"):
            continue

        match = RE_PYTHON_REQ.match(line)
        if match:
            name = match.group(1).strip().lower()
            # Extract version from the line
            ver_match = re.search(r'[=!<>~]=\s*([\d][\d.]*)', line)
            version = ver_match.group(1) if ver_match else ""
            if name and not is_known_package(name):
                packages[name] = version
        else:
            if re.match(r'^[a-zA-Z0-9][a-zA-Z0-9._-]*$', line):
                name = line.strip().lower()
                if name and not is_known_package(name):
                    packages[name] = ""

    return packages


def extract_from_gemfile(content):
    """Extract gem names from Gemfile content.

    Returns a dict of {gem_name: version_string}.
    """
    packages = {}
    # Match gem "name", "~> 1.0"
    re_gem_ver = re.compile(
        r'''gem\s+['"]([a-zA-Z0-9][a-zA-Z0-9._-]*)['"](?:\s*,\s*['"]([~>=<!\s\d.]+)['"])?'''
    )

    for match in re_gem_ver.finditer(content):
        name = match.group(1).strip().lower()
        version = match.group(2).strip() if match.group(2) else ""
        if name and not is_known_package(name):
            packages[name] = version

    return packages


def extract_from_setup_py(content):
    """Extract package names from setup.py content.

    Returns a dict of {package_name: version_string}.
    """
    packages = {}

    for key in ["install_requires", "setup_requires", "tests_require"]:
        req_match = re.search(
            rf'{key}\s*=\s*\[(.*?)\]', content, re.DOTALL
        )
        if req_match:
            block = req_match.group(1)
            for match in RE_SETUP_PY.finditer(block):
                name = match.group(1).strip().lower()
                # Try to get version from full match
                full = match.group(0)
                ver_match = re.search(r'[=!<>~]=\s*([\d][\d.]*)', full)
                version = ver_match.group(1) if ver_match else ""
                if name and not is_known_package(name):
                    packages[name] = version

    return packages


def extract_from_pipfile(content):
    """Extract package names from Pipfile content.

    Returns a dict of {package_name: version_string}.
    """
    packages = {}
    in_packages_section = False

    for line in content.splitlines():
        line = line.strip()
        if line == "[packages]" or line == "[dev-packages]":
            in_packages_section = True
            continue
        elif line.startswith("["):
            in_packages_section = False
            continue

        if in_packages_section and "=" in line:
            parts = line.split("=", 1)
            name = parts[0].strip().strip('"').strip("'").lower()
            version = parts[1].strip().strip('"').strip("'") if len(parts) > 1 else ""
            if version == "*":
                version = ""
            if name and re.match(r'^[a-zA-Z0-9._-]+$', name):
                if not is_known_package(name):
                    packages[name] = version

    return packages


def extract_from_pyproject_toml(content):
    """Extract package names from pyproject.toml content.

    Returns a dict of {package_name: version_string}.
    """
    packages = {}
    in_deps = False

    for line in content.splitlines():
        stripped = line.strip()
        if "dependencies" in stripped and "=" in stripped:
            match = re.search(r'\[(.+)\]', stripped)
            if match:
                items = match.group(1)
                for dep_match in re.finditer(r'''['"]([a-zA-Z0-9][a-zA-Z0-9._-]*)(?:[>=<~!]+\s*([\d][\d.]*))?''', items):
                    name = dep_match.group(1).strip().lower()
                    version = dep_match.group(2) if dep_match.group(2) else ""
                    if name and not is_known_package(name):
                        packages[name] = version
        elif stripped == "[project.dependencies]" or stripped == "[project.optional-dependencies]":
            in_deps = True
            continue
        elif stripped.startswith("["):
            in_deps = False
            continue
        elif in_deps and stripped and not stripped.startswith("#"):
            dep_match = re.match(r'''['"]?([a-zA-Z0-9][a-zA-Z0-9._-]*)(?:[>=<~!]+\s*([\d][\d.]*))?''', stripped)
            if dep_match:
                name = dep_match.group(1).strip().lower()
                version = dep_match.group(2) if dep_match.group(2) else ""
                if name and not is_known_package(name):
                    packages[name] = version

    return packages


def extract_from_yarn_lock(content):
    """Extract package names from yarn.lock content.

    Returns a dict of {package_name: version_string}.
    """
    packages = {}

    for match in re.finditer(
        r'^"?(@[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+|[a-zA-Z0-9._-]+)@([^\s:]+)',
        content,
        re.MULTILINE
    ):
        name = _clean_package_name(match.group(1))
        version = match.group(2).strip('"').strip("'") if match.group(2) else ""
        if name and not is_known_package(name):
            packages[name] = version

    return packages


def extract_from_package_lock_json(content):
    """Extract package names from package-lock.json content.

    Returns a dict of {package_name: version_string}.
    """
    return extract_from_package_json(content)


def detect_manifest_type(filename, content=""):
    """Detect the type of manifest file and return the appropriate extractor.

    Returns (ecosystem, extractor_function) or (None, None) if unknown.
    """
    filename_lower = filename.lower()

    if filename_lower.endswith("package.json"):
        return "npm", extract_from_package_json
    elif filename_lower.endswith("package-lock.json"):
        return "npm", extract_from_package_lock_json
    elif filename_lower.endswith("yarn.lock"):
        return "npm", extract_from_yarn_lock
    elif filename_lower.endswith("requirements.txt") or filename_lower.endswith("-requirements.txt"):
        return "pypi", extract_from_requirements_txt
    elif filename_lower.endswith("setup.py"):
        return "pypi", extract_from_setup_py
    elif filename_lower.endswith("pipfile"):
        return "pypi", extract_from_pipfile
    elif filename_lower.endswith("pipfile.lock"):
        return "pypi", extract_from_pipfile
    elif filename_lower.endswith("pyproject.toml"):
        return "pypi", extract_from_pyproject_toml
    elif filename_lower.endswith("gemfile") or filename_lower == ".gemfile":
        return "rubygems", extract_from_gemfile
    elif filename_lower.endswith("gemfile.lock"):
        return "rubygems", extract_from_gemfile
    elif filename_lower.endswith(".gemspec"):
        return "rubygems", extract_from_gemfile

    return None, None


def detect_ecosystem_from_js(packages):
    """All packages from JS files are assumed npm/yarn ecosystem."""
    return "npm"