From 01bb6acdceb98bb211715db760a73f60b32fbd37 Mon Sep 17 00:00:00 2001 From: "fangyaozheng@bytedance.com" Date: Sat, 16 Aug 2025 00:21:57 +0800 Subject: [PATCH 1/3] support long and short term memort in web --- veadk/cli/main.py | 64 ++++++++++++++++--- veadk/cli/services/vefaas/template/src/run.sh | 3 + 2 files changed, 57 insertions(+), 10 deletions(-) diff --git a/veadk/cli/main.py b/veadk/cli/main.py index 5f5a8c73..09424911 100644 --- a/veadk/cli/main.py +++ b/veadk/cli/main.py @@ -20,6 +20,7 @@ import sys from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path +from typing import Any import typer import uvicorn @@ -206,22 +207,65 @@ def web( "--host", ), ): - from google.adk.memory import in_memory_memory_service + from google.adk.cli.utils.shared_value import SharedValue + + from veadk.memory.short_term_memory import ShortTermMemory + + def init_for_veadk( + self, + *, + agent_loader: Any, + session_service: Any, + memory_service: Any, + artifact_service: Any, + credential_service: Any, + eval_sets_manager: Any, + eval_set_results_manager: Any, + agents_dir: str, + ): + self.agent_loader = agent_loader + self.artifact_service = artifact_service + self.credential_service = credential_service + self.eval_sets_manager = eval_sets_manager + self.eval_set_results_manager = eval_set_results_manager + self.agents_dir = agents_dir + # Internal propeties we want to allow being modified from callbacks. + self.runners_to_clean = set() + self.current_app_name_ref = SharedValue(value="") + self.runner_dict = {} + + short_term_memory_backend = os.getenv("SHORT_TERM_MEMORY_BACKEND", "local") + logger.info(f"Short term memory: backend={short_term_memory_backend}") + + long_term_memory_backend = os.getenv("LONG_TERM_MEMORY_BACKEND") + long_term_memory = None + + if long_term_memory_backend is not None: + from veadk.memory.long_term_memory import LongTermMemory + + logger.info( + f"Long term memory: backend={os.getenv('LONG_TERM_MEMORY_BACKEND')}" + ) + long_term_memory = LongTermMemory(backend=long_term_memory_backend) + else: + logger.info("No long term memory backend settings detected.") + + self.session_service = ShortTermMemory( + backend=short_term_memory_backend + ).session_service - from veadk.memory.long_term_memory import LongTermMemory + self.memory_service = long_term_memory - in_memory_memory_service.InMemoryMemoryService = LongTermMemory + import google.adk.cli.adk_web_server - from google.adk.cli import cli_tools_click + google.adk.cli.adk_web_server.AdkWebServer.__init__ = init_for_veadk + + import google.adk.cli.cli_tools_click as cli_tools_click - importlib.reload(cli_tools_click) agents_dir = os.getcwd() - if not session_service_uri: - session_service_uri = "" + logger.info(f"Load agents from {agents_dir}") - cli_tools_click.cli_web.main( - args=[agents_dir, "--session_service_uri", session_service_uri, "--host", host] - ) + cli_tools_click.cli_web.main(args=[agents_dir, "--host", host]) @app.command() diff --git a/veadk/cli/services/vefaas/template/src/run.sh b/veadk/cli/services/vefaas/template/src/run.sh index 34f2695f..447a486f 100755 --- a/veadk/cli/services/vefaas/template/src/run.sh +++ b/veadk/cli/services/vefaas/template/src/run.sh @@ -43,6 +43,9 @@ python3 -m pip install fastmcp USE_STUDIO=${USE_STUDIO:-False} USE_ADK_WEB=${USE_ADK_WEB:-False} +export SHORT_TERM_MEMORY_BACKEND=mysql +export LONG_TERM_MEMORY_BACKEND=opensearch + if [ "$USE_STUDIO" = "True" ]; then echo "USE_STUDIO is True, running veadk studio" # running veadk studio From d2c3a8dc8f6edbbc6498a83ce55f9e0d525c7587 Mon Sep 17 00:00:00 2001 From: "fangyaozheng@bytedance.com" Date: Sat, 16 Aug 2025 11:28:29 +0800 Subject: [PATCH 2/3] fix env bugs --- veadk/cli/main.py | 10 +++++----- veadk/cli/services/vefaas/template/src/run.sh | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/veadk/cli/main.py b/veadk/cli/main.py index 09424911..b847f33e 100644 --- a/veadk/cli/main.py +++ b/veadk/cli/main.py @@ -234,18 +234,18 @@ def init_for_veadk( self.current_app_name_ref = SharedValue(value="") self.runner_dict = {} - short_term_memory_backend = os.getenv("SHORT_TERM_MEMORY_BACKEND", "local") + short_term_memory_backend = os.getenv("SHORT_TERM_MEMORY_BACKEND") + if not short_term_memory_backend: # prevent None or empty string + short_term_memory_backend = "local" logger.info(f"Short term memory: backend={short_term_memory_backend}") long_term_memory_backend = os.getenv("LONG_TERM_MEMORY_BACKEND") long_term_memory = None - if long_term_memory_backend is not None: + if long_term_memory_backend: from veadk.memory.long_term_memory import LongTermMemory - logger.info( - f"Long term memory: backend={os.getenv('LONG_TERM_MEMORY_BACKEND')}" - ) + logger.info(f"Long term memory: backend={long_term_memory_backend}") long_term_memory = LongTermMemory(backend=long_term_memory_backend) else: logger.info("No long term memory backend settings detected.") diff --git a/veadk/cli/services/vefaas/template/src/run.sh b/veadk/cli/services/vefaas/template/src/run.sh index 447a486f..7e56d247 100755 --- a/veadk/cli/services/vefaas/template/src/run.sh +++ b/veadk/cli/services/vefaas/template/src/run.sh @@ -43,8 +43,8 @@ python3 -m pip install fastmcp USE_STUDIO=${USE_STUDIO:-False} USE_ADK_WEB=${USE_ADK_WEB:-False} -export SHORT_TERM_MEMORY_BACKEND=mysql -export LONG_TERM_MEMORY_BACKEND=opensearch +export SHORT_TERM_MEMORY_BACKEND= # can be `mysql` +export LONG_TERM_MEMORY_BACKEND= # can be `opensearch` if [ "$USE_STUDIO" = "True" ]; then echo "USE_STUDIO is True, running veadk studio" From 56bb489f105a14caaef167b009ceb945509fb0a3 Mon Sep 17 00:00:00 2001 From: "fangyaozheng@bytedance.com" Date: Sat, 16 Aug 2025 13:00:26 +0800 Subject: [PATCH 3/3] add secret scan --- .gitleaks.toml | 67 +++++++++++++++++++++++++++++++++++++++++ .pre-commit-config.yaml | 6 +++- 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 .gitleaks.toml diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..9b241914 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,67 @@ +[extend] +useDefault = true + +[[rules]] +id = "aklt-key-pattern" +description = "AKLT key pattern" +regex = '''AKLT\w{40,70}''' + +[[rules]] +id = "akap-key-pattern" +description = "AKAP key pattern" +regex = '''AKAP\w{40,70}''' + +[[rules]] +id = "akip-key-pattern" +description = "AKIP key pattern" +regex = '''AKI\w{40,70}''' + +[[rules]] +id = "token-transformer-id-pattern" +description = "Tokenizer/Transformer/Token ID patterns" +regex = '''(tokenizer|transformer|token_id|tokenid|attention_head).{0,20}''' + +[[rules]] +id = "aws-style-key-pattern" +description = "AWS-style key pattern" +regex = '''(A3T[A-Z0-9]|AKIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA|ASIA)[A-Z0-9]{16}''' + +[[rules]] +id = "alibaba-ltai-pattern" +description = "Alibaba LTAI key pattern" +regex = '''(LTAI)[a-z0-9]{20}''' + +[[rules]] +id = "aktp-key-pattern" +description = "AKTP key pattern" +regex = '''AKTP\w{40,70}''' + +[[rules]] +id = "app-id-pattern" +description = "App ID patterns" +regex = '''([^*<\s|:>]{0,7})(app_id|appid)([^]()!<>;/@&,]{0,10}[(=:]\s{0,6}["']{0,1}[0-9]{6,32}["']{0,1})''' + +[[rules]] +id = "byted-org-domains" +description = "byted.org domains" +regex = '''.{0,15}\.?byted.org.{0,20}''' + +[[rules]] +id = "bytedance-net-domains" +description = "bytedance.net domains" +regex = '''.{0,15}\.?bytedance.net.{0,20}''' + +[[rules]] +id = "feishu-cn-domains" +description = "bytedance.feishu.cn domains" +regex = '''.{0,20}.bytedance\.feishu\.cn.{0,50}''' + +[[rules]] +id = "larkoffice-com-domains" +description = "bytedance.larkoffice.com domains" +regex = '''.{0,20}.bytedance\.larkoffice\.com.{0,50}''' + +[[rules]] +id = "private-ip-10-range" +description = "Private IP address pattern (10.x.x.x)" +regex = '''(10\.\d{1,3}\.\d{1,3}\.\d{1,3})''' \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d746a67f..44d1a3df 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,4 +9,8 @@ repos: args: [ --fix ] # Run the formatter. - id: ruff-format - types_or: [ python, pyi ] \ No newline at end of file + types_or: [ python, pyi ] +- repo: https://github.com/gitleaks/gitleaks + rev: v8.24.2 + hooks: + - id: gitleaks \ No newline at end of file