diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..9b241914 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,67 @@ +[extend] +useDefault = true + +[[rules]] +id = "aklt-key-pattern" +description = "AKLT key pattern" +regex = '''AKLT\w{40,70}''' + +[[rules]] +id = "akap-key-pattern" +description = "AKAP key pattern" +regex = '''AKAP\w{40,70}''' + +[[rules]] +id = "akip-key-pattern" +description = "AKIP key pattern" +regex = '''AKI\w{40,70}''' + +[[rules]] +id = "token-transformer-id-pattern" +description = "Tokenizer/Transformer/Token ID patterns" +regex = '''(tokenizer|transformer|token_id|tokenid|attention_head).{0,20}''' + +[[rules]] +id = "aws-style-key-pattern" +description = "AWS-style key pattern" +regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''' + +[[rules]] +id = "alibaba-ltai-pattern" +description = "Alibaba LTAI key pattern" +regex = '''(LTAI)[a-z0-9]{20}''' + +[[rules]] +id = "aktp-key-pattern" +description = "AKTP key pattern" +regex = '''AKTP\w{40,70}''' + +[[rules]] +id = "app-id-pattern" +description = "App ID patterns" +regex = '''([^*<\s|:>]{0,7})(app_id|appid)([^]()!<>;/@&,]{0,10}[(=:]\s{0,6}["']{0,1}[0-9]{6,32}["']{0,1})''' + +[[rules]] +id = "byted-org-domains" +description = "byted.org domains" +regex = '''.{0,15}\.?byted.org.{0,20}''' + +[[rules]] +id = "bytedance-net-domains" +description = "bytedance.net domains" +regex = '''.{0,15}\.?bytedance.net.{0,20}''' + +[[rules]] +id = "feishu-cn-domains" +description = "bytedance.feishu.cn domains" +regex = '''.{0,20}.bytedance\.feishu\.cn.{0,50}''' + +[[rules]] +id = "larkoffice-com-domains" +description = "bytedance.larkoffice.com domains" +regex = '''.{0,20}.bytedance\.larkoffice\.com.{0,50}''' + +[[rules]] +id = "private-ip-10-range" +description = "Private IP address pattern (10.x.x.x)" +regex = '''(10\.\d{1,3}\.\d{1,3}\.\d{1,3})''' \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d746a67f..44d1a3df 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,4 +9,8 @@ repos: args: [ --fix ] # Run the formatter. - id: ruff-format - types_or: [ python, pyi ] \ No newline at end of file + types_or: [ python, pyi ] +- repo: https://github.com/gitleaks/gitleaks + rev: v8.24.2 + hooks: + - id: gitleaks \ No newline at end of file diff --git a/veadk/cli/main.py b/veadk/cli/main.py index 5f5a8c73..b847f33e 100644 --- a/veadk/cli/main.py +++ b/veadk/cli/main.py @@ -20,6 +20,7 @@ import sys from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path +from typing import Any import typer import uvicorn @@ -206,22 +207,65 @@ def web( "--host", ), ): - from google.adk.memory import in_memory_memory_service + from google.adk.cli.utils.shared_value import SharedValue + + from veadk.memory.short_term_memory import ShortTermMemory + + def init_for_veadk( + self, + *, + agent_loader: Any, + session_service: Any, + memory_service: Any, + artifact_service: Any, + credential_service: Any, + eval_sets_manager: Any, + eval_set_results_manager: Any, + agents_dir: str, + ): + self.agent_loader = agent_loader + self.artifact_service = artifact_service + self.credential_service = credential_service + self.eval_sets_manager = eval_sets_manager + self.eval_set_results_manager = eval_set_results_manager + self.agents_dir = agents_dir + # Internal propeties we want to allow being modified from callbacks. + self.runners_to_clean = set() + self.current_app_name_ref = SharedValue(value="") + self.runner_dict = {} + + short_term_memory_backend = os.getenv("SHORT_TERM_MEMORY_BACKEND") + if not short_term_memory_backend: # prevent None or empty string + short_term_memory_backend = "local" + logger.info(f"Short term memory: backend={short_term_memory_backend}") + + long_term_memory_backend = os.getenv("LONG_TERM_MEMORY_BACKEND") + long_term_memory = None + + if long_term_memory_backend: + from veadk.memory.long_term_memory import LongTermMemory + + logger.info(f"Long term memory: backend={long_term_memory_backend}") + long_term_memory = LongTermMemory(backend=long_term_memory_backend) + else: + logger.info("No long term memory backend settings detected.") + + self.session_service = ShortTermMemory( + backend=short_term_memory_backend + ).session_service - from veadk.memory.long_term_memory import LongTermMemory + self.memory_service = long_term_memory - in_memory_memory_service.InMemoryMemoryService = LongTermMemory + import google.adk.cli.adk_web_server - from google.adk.cli import cli_tools_click + google.adk.cli.adk_web_server.AdkWebServer.__init__ = init_for_veadk + + import google.adk.cli.cli_tools_click as cli_tools_click - importlib.reload(cli_tools_click) agents_dir = os.getcwd() - if not session_service_uri: - session_service_uri = "" + logger.info(f"Load agents from {agents_dir}") - cli_tools_click.cli_web.main( - args=[agents_dir, "--session_service_uri", session_service_uri, "--host", host] - ) + cli_tools_click.cli_web.main(args=[agents_dir, "--host", host]) @app.command() diff --git a/veadk/cli/services/vefaas/template/src/run.sh b/veadk/cli/services/vefaas/template/src/run.sh index 34f2695f..7e56d247 100755 --- a/veadk/cli/services/vefaas/template/src/run.sh +++ b/veadk/cli/services/vefaas/template/src/run.sh @@ -43,6 +43,9 @@ python3 -m pip install fastmcp USE_STUDIO=${USE_STUDIO:-False} USE_ADK_WEB=${USE_ADK_WEB:-False} +export SHORT_TERM_MEMORY_BACKEND= # can be `mysql` +export LONG_TERM_MEMORY_BACKEND= # can be `opensearch` + if [ "$USE_STUDIO" = "True" ]; then echo "USE_STUDIO is True, running veadk studio" # running veadk studio