From 07ecd4356e01d72bf914c5eaec41eb9e19779771 Mon Sep 17 00:00:00 2001 From: Eric Apgar Date: Sun, 15 Mar 2026 16:45:48 -0500 Subject: [PATCH 1/8] Added github artifacts from template. --- .github/ISSUE_TEMPLATE/bug_report.md | 25 +++++++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 16 +++++++++++++++ .github/pull_request_template.md | 10 +++++++++ 3 files changed, 51 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/pull_request_template.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..6d1965d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,25 @@ +--- +name: Bug Report +about: Create a report to help us improve +title: "" +labels: ["bug"] +assignees: [] +--- + + + + +## What's the problem? +... + +## Steps to Reproduce +... + +## Screenshot +If applicable. + +## Proposed Solution +... + +## Notes +... diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..ed614bf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,16 @@ +--- +name: Enhancement Request +about: Ask for a change. +title: "" +labels: ["enhancement"] +assignees: [] +--- + +## Desired Change +... + +## Rationale +... + +## Notes +... diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..c2f077c --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,10 @@ +## Description +... + +## Summary of Changes +... + +Use ```Closes #X``` to automatically close an issue addressed by this PR. + +# Proof it Works +... From 07b18a917f79b339e0a3469d2b8b7e1a3636dd32 Mon Sep 17 00:00:00 2001 From: Eric Apgar Date: Sun, 15 Mar 2026 16:51:52 -0500 Subject: [PATCH 2/8] Moved demo file. --- {src/llm/other => scripts}/demo.py | 0 src/llm/__init__.py | 24 +----------------------- src/llm/models/embed.py | 5 ++--- 3 files changed, 3 insertions(+), 26 deletions(-) rename {src/llm/other => scripts}/demo.py (100%) diff --git a/src/llm/other/demo.py b/scripts/demo.py similarity index 100% rename from src/llm/other/demo.py rename to scripts/demo.py diff --git a/src/llm/__init__.py b/src/llm/__init__.py index 22c1ff1..66c4eda 100644 --- a/src/llm/__init__.py +++ b/src/llm/__init__.py @@ -1,27 +1,10 @@ -''' -Desired interface: - -import llm - -model = llm.model( - name='openai/gpt-oss-20b', - hf_token=) -model.load( - location=, - remote=true, - commit=, - quantization='4-bit') -response = model.ask(prompt='Tell me a joke.') -''' - from __future__ import annotations from typing import TYPE_CHECKING, Any __all__ = ( "model", - "embedding", - 'Conversation') + "embedding") if TYPE_CHECKING: @@ -40,11 +23,6 @@ def __getattr__(name: str) -> Any: from .models.embed import EmbeddingModel globals()[name] = EmbeddingModel return EmbeddingModel - - if name == 'Conversation': - from .other.conversations import Conversation - globals()[name] = Conversation - return Conversation raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/src/llm/models/embed.py b/src/llm/models/embed.py index fdc702a..d62b47a 100644 --- a/src/llm/models/embed.py +++ b/src/llm/models/embed.py @@ -5,10 +5,10 @@ class EmbeddingModel: - def __init__(self, hf_token: str|None=None): + def __init__(self, name: str='all-mpnet-base-v2', hf_token: str|None=None): + self.name: str = name self.hf_token: str = hf_token - self.name: str = None self.location: str = None self.remote: bool = False @@ -20,7 +20,6 @@ def __init__(self, hf_token: str|None=None): def load(self, location: str, - name: str='all-mpnet-base-v2', remote: bool=False, commit: str=None) -> None: From 3f1eeee9cb914063da866fc03e3b20dd8e502b92 Mon Sep 17 00:00:00 2001 From: EricApgar Date: Sun, 15 Mar 2026 17:12:12 -0500 Subject: [PATCH 3/8] Updated model selection for LLMs and embedding models. --- src/llm/models/embed.py | 42 ++++++++++++++++++++++++------------- src/llm/models/selection.py | 36 ++++++++++++++++++++++--------- 2 files changed, 54 insertions(+), 24 deletions(-) diff --git a/src/llm/models/embed.py b/src/llm/models/embed.py index d62b47a..a089ea1 100644 --- a/src/llm/models/embed.py +++ b/src/llm/models/embed.py @@ -1,11 +1,18 @@ +import os + from sentence_transformers import util, SentenceTransformer import numpy as np import torch +from llm.models.selection import SUPPORTED_EMBEDDING_MODELS + class EmbeddingModel: - def __init__(self, name: str='all-mpnet-base-v2', hf_token: str|None=None): + def __init__(self, name: str='all-mpnet-base-v2', hf_token: str=None): + + if name not in SUPPORTED_EMBEDDING_MODELS: + raise ValueError(f'Invalid embedding model "{self.name}". Supported: {SUPPORTED_EMBEDDING_MODELS}.') self.name: str = name self.hf_token: str = hf_token @@ -23,19 +30,16 @@ def load(self, remote: bool=False, commit: str=None) -> None: - VALID_MODELS = ['all-mpnet-base-v2', 'all-MiniLM-L6-v2'] + if (not remote) and (not os.path.isdir(location)): + raise ValueError(f'Nonexistant location ({location}) - fix or set remote=True.') - if name not in VALID_MODELS: - raise ValueError(f'Invalid embedding model "{name}". Supported: {VALID_MODELS}') - self.location = location - self.name = name self.remote = remote self.commit = commit self.model = SentenceTransformer( cache_folder=location, - model_name_or_path=name, + model_name_or_path=self.name, local_files_only=not self.remote, trust_remote_code=self.remote, token=self.hf_token) @@ -71,10 +75,19 @@ def get_similarity(self, similarity = self._torch_dot_similarity(v1=v1, v2=v2) return similarity - + + @staticmethod def _dot_similarity(v1: np.array, v2: np.array) -> float: + result = torch.dot(v1, v2) + + return result + + + @staticmethod + def _cosine_similarity(v1: np.array, v2: np.array) -> float: + dot_product = torch.dot(v1, v2) norm_vector_1 = torch.sqrt(torch.sum(v1**2)) @@ -94,12 +107,13 @@ def _torch_dot_similarity(v1: np.array, v2: np.array) -> float: if __name__ == '__main__': - model = EmbeddingModel() - model.load(location=r'') - e1 = model.embed(text='What shape is best?') - e2 = model.embed(text='Hexagons are the bestagons.') - similarity = model.get_similarity(v1=e1, v2=e2) - print(similarity) + # model = EmbeddingModel() + # model.load(location=r'') + # e1 = model.embed(text='What shape is best?') + # e2 = model.embed(text='Hexagons are the bestagons.') + # similarity = model.get_similarity(v1=e1, v2=e2) + + # print(similarity) pass \ No newline at end of file diff --git a/src/llm/models/selection.py b/src/llm/models/selection.py index 8f1f3bd..22afa0d 100644 --- a/src/llm/models/selection.py +++ b/src/llm/models/selection.py @@ -1,16 +1,32 @@ -''' -Import the specific model named by the user. -''' +from typing import Literal + +from .gpt_oss_20b import GptOss20b +from .phi4_multimodal_instruct import Phi4MultimodalInstruct + + +SUPPORTED_LLMS = { + GptOss20b().name: GptOss20b, + Phi4MultimodalInstruct().name: Phi4MultimodalInstruct +} + +SUPPORTED_EMBEDDING_MODELS = ['all-MiniLM-L6-v2', 'all-mpnet-base-v2'] + def model(name: str, hf_token: str=None): - if name == 'openai/gpt-oss-20b': - from .gpt_oss_20b import GptOss20b - return GptOss20b(hf_token=hf_token) + if name not in SUPPORTED_LLMS: + raise ValueError(f'Model "{name}" not supported! Call "list_models()" for options.') + + return SUPPORTED_LLMS[name](hf_token=hf_token) + - elif name == 'microsoft/Phi-4-multimodal-instruct': - from .phi4_multimodal_instruct import Phi4MultimodalInstruct - return Phi4MultimodalInstruct(hf_token=hf_token) +def list_models(kind: Literal['llm', 'embed']): + if kind == 'llm': + supported_models = list(SUPPORTED_LLMS.keys()) + elif kind == 'embed': + supported_models = SUPPORTED_EMBEDDING_MODELS else: - raise ValueError(f'Model "{name}" not supported!') \ No newline at end of file + raise ValueError(f'Unsupported "kind" ({kind}). Must be "llm" or "embed".') + + return supported_models \ No newline at end of file From 781650b94305e3f09554bb2b81afb90ad3f64c26 Mon Sep 17 00:00:00 2001 From: Eric Apgar Date: Sun, 15 Mar 2026 17:27:28 -0500 Subject: [PATCH 4/8] Updated demo script. --- scripts/demo.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/demo.py b/scripts/demo.py index 3737f5c..e23405d 100644 --- a/scripts/demo.py +++ b/scripts/demo.py @@ -9,15 +9,15 @@ if __name__ == '__main__': - # model = GptOss20b() model = GptOss20b() + model_cache_dir = input(prompt='Enter model cache dir: ') model.load(location=r'/home/eric/Repos/model_cache') c = Conversation() - c.set_overall_prompt(text='Your name is Seamus OFinnegan. Respond as the assistant in character.') - c.add_context(text='Youre a real salt of the earth Irish coal miner.') - c.add_context(text='You get straight to the point and dont waste words on small talk.') - c.add_context(text='You are married to your wife of 10 years, Gurdy. Your favorite hobby is fighting.') + c.set_overall_prompt(text='Your name is Samson McTavish. Respond in character.') + c.add_context(text="You're a 3rd year student at a school for magic.") + c.add_context(text='Your favorite class is potions.') + c.add_context(text='Your favorite spell is "Mimble Wimble".') # Loop: ''' @@ -35,6 +35,6 @@ system_response = model.ask(prompt=c) - print(f'[Seamus]: {system_response}\n') + print(f'[Samson]: {system_response}\n') c.add_response(role='assistant', text=system_response) \ No newline at end of file From 821c9e3d19982829a610b14459021fa8765cb1c3 Mon Sep 17 00:00:00 2001 From: Eric Apgar Date: Sun, 15 Mar 2026 17:30:20 -0500 Subject: [PATCH 5/8] Updated version and tests. --- pyproject.toml | 4 ++-- tests/test_gpt_oss_20b.py | 2 +- tests/test_phi4_multimodal_instruct.py | 2 +- uv.lock | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a49c4a8..efb1459 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "llm" # The pip install . -version = "0.4.0" +version = "0.4.1" description = "Library for easy use of LLMs." readme = "README.md" authors = [ @@ -48,7 +48,7 @@ explicit = true # Only used for torch packages explicitly mapped below. [tool.uv.sources] torch = { index = "pytorch-cu130" } torchvision = { index = "pytorch-cu130" } -llm-conversation = { git = "https://github.com/EricApgar/llm-conversation", rev = "v0.2.0" } +llm-conversation = { git = "https://github.com/EricApgar/llm-conversation" } [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/tests/test_gpt_oss_20b.py b/tests/test_gpt_oss_20b.py index 49eb7a6..77e13a2 100644 --- a/tests/test_gpt_oss_20b.py +++ b/tests/test_gpt_oss_20b.py @@ -6,7 +6,7 @@ invoking pytest so it only exists for that one command and does not persist in your shell: - uv sync --extra dev --extra openai + uv sync --extra openai LLM_MODEL_CACHE=/home/yourname/Repos/model_cache pytest Tests are skipped automatically if LLM_MODEL_CACHE is not set. diff --git a/tests/test_phi4_multimodal_instruct.py b/tests/test_phi4_multimodal_instruct.py index b399b08..2e0a6d3 100644 --- a/tests/test_phi4_multimodal_instruct.py +++ b/tests/test_phi4_multimodal_instruct.py @@ -6,7 +6,7 @@ invoking pytest so it only exists for that one command and does not persist in your shell: - uv sync --extra dev --extra microsoft + uv sync --extra microsoft LLM_MODEL_CACHE=/home/yourname/Repos/model_cache pytest Tests are skipped automatically if LLM_MODEL_CACHE is not set. diff --git a/uv.lock b/uv.lock index 05bacc9..9ce70fd 100644 --- a/uv.lock +++ b/uv.lock @@ -192,7 +192,7 @@ wheels = [ [[package]] name = "llm" -version = "0.4.0" +version = "0.4.1" source = { editable = "." } dependencies = [ { name = "llm-conversation" }, @@ -232,7 +232,7 @@ requires-dist = [ { name = "backoff", marker = "extra == 'microsoft'", specifier = ">=2.2.1" }, { name = "kernels", marker = "extra == 'all'", specifier = ">=0.11.1" }, { name = "kernels", marker = "extra == 'openai'", specifier = ">=0.11.1" }, - { name = "llm-conversation", git = "https://github.com/EricApgar/llm-conversation?rev=v0.2.0" }, + { name = "llm-conversation", git = "https://github.com/EricApgar/llm-conversation" }, { name = "openai-harmony", marker = "extra == 'all'", specifier = ">=0.0.8" }, { name = "openai-harmony", marker = "extra == 'openai'", specifier = ">=0.0.8" }, { name = "peft", marker = "extra == 'all'", specifier = ">=0.18.1" }, @@ -250,7 +250,7 @@ provides-extras = ["openai", "microsoft", "all"] [[package]] name = "llm-conversation" version = "0.2.0" -source = { git = "https://github.com/EricApgar/llm-conversation?rev=v0.2.0#ce9d59af41f4a01839bc01a0c8428d0f1775d8c2" } +source = { git = "https://github.com/EricApgar/llm-conversation#ce9d59af41f4a01839bc01a0c8428d0f1775d8c2" } [[package]] name = "markupsafe" From a4d2bee16fb68ee0f54a64a380c0c8d32f75818c Mon Sep 17 00:00:00 2001 From: Eric Apgar Date: Sun, 15 Mar 2026 17:31:49 -0500 Subject: [PATCH 6/8] Updated a comment in tests. --- tests/test_gpt_oss_20b.py | 2 ++ tests/test_phi4_multimodal_instruct.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/test_gpt_oss_20b.py b/tests/test_gpt_oss_20b.py index 77e13a2..57a6968 100644 --- a/tests/test_gpt_oss_20b.py +++ b/tests/test_gpt_oss_20b.py @@ -10,6 +10,8 @@ LLM_MODEL_CACHE=/home/yourname/Repos/model_cache pytest Tests are skipped automatically if LLM_MODEL_CACHE is not set. + +Make sure the virtual environment is active. """ import os diff --git a/tests/test_phi4_multimodal_instruct.py b/tests/test_phi4_multimodal_instruct.py index 2e0a6d3..427f05c 100644 --- a/tests/test_phi4_multimodal_instruct.py +++ b/tests/test_phi4_multimodal_instruct.py @@ -10,6 +10,8 @@ LLM_MODEL_CACHE=/home/yourname/Repos/model_cache pytest Tests are skipped automatically if LLM_MODEL_CACHE is not set. + +Make sure the virtual environment is active. """ import os From 6f5ef4f76ddff1e36060c6b37d6ac48175b8f2be Mon Sep 17 00:00:00 2001 From: Eric Apgar Date: Sun, 15 Mar 2026 17:37:12 -0500 Subject: [PATCH 7/8] Commented out base tests for all classes since now covered by pytest. --- src/llm/models/embed.py | 2 +- src/llm/models/gpt_oss_20b.py | 8 ++--- src/llm/models/gpt_oss_20b_dev.py | 8 ++--- src/llm/models/phi4_multimodal_instruct.py | 36 +++++++++++----------- tests/test_gpt_oss_20b.py | 2 +- tests/test_phi4_multimodal_instruct.py | 2 +- 6 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/llm/models/embed.py b/src/llm/models/embed.py index a089ea1..f39c976 100644 --- a/src/llm/models/embed.py +++ b/src/llm/models/embed.py @@ -109,7 +109,7 @@ def _torch_dot_similarity(v1: np.array, v2: np.array) -> float: if __name__ == '__main__': # model = EmbeddingModel() - # model.load(location=r'') + # model.load(location=) # NOTE: set . # e1 = model.embed(text='What shape is best?') # e2 = model.embed(text='Hexagons are the bestagons.') # similarity = model.get_similarity(v1=e1, v2=e2) diff --git a/src/llm/models/gpt_oss_20b.py b/src/llm/models/gpt_oss_20b.py index 4141c36..632c2a7 100644 --- a/src/llm/models/gpt_oss_20b.py +++ b/src/llm/models/gpt_oss_20b.py @@ -117,10 +117,10 @@ def _format_prompt(prompt: str | Conversation, reasoning_level: str=None) -> lis if __name__ == '__main__': - model = GptOss20b() - model.load(location=r'/home/eric/Repos/model_cache') - response = model.ask(prompt='Name a primary color.') + # model = GptOss20b() + # model.load(location=) # NOTE: set . + # response = model.ask(prompt='Name a primary color.') - print(response) + # print(response) pass \ No newline at end of file diff --git a/src/llm/models/gpt_oss_20b_dev.py b/src/llm/models/gpt_oss_20b_dev.py index 2855955..3d8f011 100644 --- a/src/llm/models/gpt_oss_20b_dev.py +++ b/src/llm/models/gpt_oss_20b_dev.py @@ -204,10 +204,10 @@ def get_good_token_start(token_list: list[str]) -> int: if __name__ == '__main__': - model = GptOss20bDev() - model.load(location=r'/home/eric/Repos/model_cache') - response = model.ask(prompt='Name a primary color.') + # model = GptOss20bDev() + # model.load(location=) # NOTE: set . + # response = model.ask(prompt='Name a primary color.') - print(response) + # print(response) pass \ No newline at end of file diff --git a/src/llm/models/phi4_multimodal_instruct.py b/src/llm/models/phi4_multimodal_instruct.py index 9b3a601..8bc59c1 100644 --- a/src/llm/models/phi4_multimodal_instruct.py +++ b/src/llm/models/phi4_multimodal_instruct.py @@ -198,23 +198,23 @@ def get_usable_length(self, new_seq_length: int, layer_idx: int=0) -> int: if __name__ == '__main__': - model = Phi4MultimodalInstruct() - model.load(location=r'/home/eric/Repos/model_cache') # NOTE: set . - - response = model.ask(prompt='Name a primary color. Be brief.', max_tokens=256) - print(f'{response}\n') - - convo = Conversation() - convo.set_overall_prompt(text='You are a helpful assistant.') - convo.add_context(text='Your favorite color is red.') - convo.add_context(text='Your favorite shape is the hexagon.') - convo.add_response(role='user', text='What is your favorite color-shape combination?') - response = model.ask(prompt=convo, max_tokens=256) - print(f'{response}\n') - - from PIL import Image as PillowImage - image = PillowImage.open(r'/home/eric/Desktop/monkey.png') # NOTE: Point to existing image. - response = model.ask(prompt='Describe the image.', images=[image], max_tokens=256) - print(f'{response}\n') + # model = Phi4MultimodalInstruct() + # model.load(location=) # NOTE: set . + + # response = model.ask(prompt='Name a primary color. Be brief.', max_tokens=256) + # print(f'{response}\n') + + # convo = Conversation() + # convo.set_overall_prompt(text='You are a helpful assistant.') + # convo.add_context(text='Your favorite color is red.') + # convo.add_context(text='Your favorite shape is the hexagon.') + # convo.add_response(role='user', text='What is your favorite color-shape combination?') + # response = model.ask(prompt=convo, max_tokens=256) + # print(f'{response}\n') + + # from PIL import Image as PillowImage + # image = PillowImage.open(r'/home/eric/Desktop/monkey.png') # NOTE: Point to existing image. + # response = model.ask(prompt='Describe the image.', images=[image], max_tokens=256) + # print(f'{response}\n') pass \ No newline at end of file diff --git a/tests/test_gpt_oss_20b.py b/tests/test_gpt_oss_20b.py index 57a6968..e5d8e83 100644 --- a/tests/test_gpt_oss_20b.py +++ b/tests/test_gpt_oss_20b.py @@ -7,7 +7,7 @@ in your shell: uv sync --extra openai - LLM_MODEL_CACHE=/home/yourname/Repos/model_cache pytest + LLM_MODEL_CACHE= pytest Tests are skipped automatically if LLM_MODEL_CACHE is not set. diff --git a/tests/test_phi4_multimodal_instruct.py b/tests/test_phi4_multimodal_instruct.py index 427f05c..a532da6 100644 --- a/tests/test_phi4_multimodal_instruct.py +++ b/tests/test_phi4_multimodal_instruct.py @@ -7,7 +7,7 @@ in your shell: uv sync --extra microsoft - LLM_MODEL_CACHE=/home/yourname/Repos/model_cache pytest + LLM_MODEL_CACHE= pytest Tests are skipped automatically if LLM_MODEL_CACHE is not set. From 7d5c8f852a58602df322adb19b108488a826ba9f Mon Sep 17 00:00:00 2001 From: Eric Apgar Date: Sun, 15 Mar 2026 17:46:28 -0500 Subject: [PATCH 8/8] Added test for embedding model. --- tests/test_embed.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 tests/test_embed.py diff --git a/tests/test_embed.py b/tests/test_embed.py new file mode 100644 index 0000000..3bf34ef --- /dev/null +++ b/tests/test_embed.py @@ -0,0 +1,43 @@ +""" +Tests for EmbeddingModel. + +These tests load a real model and call embed() and get_similarity(), so they +require local model weights. Sync the environment first, then pass +LLM_MODEL_CACHE inline when invoking pytest so it only exists for that one +command and does not persist in your shell: + + uv sync --extra openai + LLM_MODEL_CACHE= pytest + +Tests are skipped automatically if LLM_MODEL_CACHE is not set. + +Make sure the virtual environment is active. +""" + +import os + +import pytest +import torch + +from llm.models.embed import EmbeddingModel + + +MODEL_CACHE = os.environ.get('LLM_MODEL_CACHE') + + +@pytest.fixture(scope='module') +def model(): + if not MODEL_CACHE: + pytest.skip('LLM_MODEL_CACHE environment variable not set.') + m = EmbeddingModel() + m.load(location=MODEL_CACHE) + yield m + del m + torch.cuda.empty_cache() + + +def test_get_similarity(model): + e1 = model.embed(text='What shape is best?') + e2 = model.embed(text='Hexagons are the bestagons.') + similarity = model.get_similarity(v1=e1, v2=e2) + assert isinstance(similarity, (int, float))