Sean/Chartwell
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Commit Latest

Browse Source
This commit is contained in:
Sean
2026-04-15 13:09:43 -04:00
parent 268928e9c5
commit b00d583d0c
12 changed files with 1111 additions and 549 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
Chartwell.py
View File

@@ -13,6 +13,9 @@ import torch
#DOMAIN: music_theory
#PRIORITY: high | medium | low
# cleaning text documents
# https://www.text-utils.com/remove-special-characters/
# Ask ChatGPT to descrive narratives around tablature examples.
# I would like you to chunk this for my RAG system.
# Where you identify guitar tablature you are to replace it with a narrative
@@ -67,7 +70,7 @@ import torch
# .venv/Scripts/activate
# 3) Install Dependencies
# pip install -r requirements.txt
# 4) Meta-Llama-3-8B-Instruct.Q4_0.gguf
# 4) Meta-Llama-3.1-8B-Instruct.Q4_0.gguf
# \Users\skess\.cache\gpt4all\Meta-Llama-3-8B-Instruct.Q4_0.gguf
# The model will auto-download on the first run and then switch to allow_download=False (see below)
# The model is about 4.5G. The download is quick.
@@ -98,13 +101,21 @@ import torch
# ----------------------------------
# Weights for chunk weighting system
# -----------------------------------
# TYPE_WEIGHTS = {
# "fact": 1.50,
# "rule": 1.20,
# "reference": 1.00,
# "pedagogical": 0.85
# }
TYPE_WEIGHTS = {
"fact": 1.30,
"rule": 1.20,
"reference": 1.00,
"pedagogical": 0.85
"fact": 1.10, # The "Oak" gets a small boost
"rule": 1.05,
"reference": 1.00, # The baseline
"pedagogical": 0.95 # The "Undergrowth" is only slightly demoted
}
PRIORITY_WEIGHTS = {
"high": 1.10,
"medium": 1.00,
@@ -114,7 +125,7 @@ PRIORITY_WEIGHTS = {
# -------------------------
# Knowledge base selection
# -------------------------
BOOK_DIR = 'Books/Music' # just a string
BOOK_DIR = 'Books/History' # just a string
book_files = []
for f in Path(BOOK_DIR).rglob('*'):
@@ -129,12 +140,8 @@ for f in Path(BOOK_DIR).rglob('*'):
print(f"Found {len(book_files)} files")
# Overlap should be 10-20% of chunk size
# CHUNK_SIZE = 700
# CHUNK_OVERLAP = 100
CHUNK_SIZE = 1500
CHUNK_OVERLAP = 300
CHUNK_SIZE = 700
CHUNK_OVERLAP = 100
DEBUG = False
CACHE_FILE = "embeddings_cache.npz"
@@ -297,7 +304,7 @@ def clean_text(text):
# -------------------------
def chunk_text(text, chunk_size=CHUNK_SIZE, overlap=CHUNK_OVERLAP):
# 1. EXTRACT HEADERS (The "Metadata Inheritance" logic)
header_patterns = [r"TYPE:.*", r"PRIORITY:.*", r"DOMAIN:.*", r"TITLE:.*"]
header_patterns = [r"TYPE:.*", r"PRIORITY:.*", r"DOMAIN:.*", r"TITLE:.*",r"CONCEPTS:.*",r"SOURCE:.*"]
header_lines = []
top_of_file = text[:500]
for pattern in header_patterns:
@@ -598,7 +605,7 @@ def extract_metadata(chunk):
Defaults are safe and neutral.
"""
meta = {
"type": "fact",
"type": "reference",
"priority": "medium"
}
@@ -614,16 +621,6 @@ def extract_metadata(chunk):
return meta
# --------------------------
# determine if we are being asked to derive a response
# --------------------------
def is_derivation_request(question):
keywords = {
"derive", "construct", "starting on", "root note",
"apply the formula", "apply formula", "step by step"
}
q = question.lower()
return any(k in q for k in keywords)
# -------------------------
# Retrieve top relevant chunks
# -------------------------
@@ -698,7 +695,8 @@ def get_top_chunks(question, filter_term=None):
# -------------------------
# FINAL SCORE (composed signal)
# -------------------------
final_scores = semantic_scores + np.log(type_weights) + np.log(priority_weights)
final_scores = (semantic_scores + 1.5 * np.log(type_weights) + 0.3 * np.log(priority_weights)
)
# -------------------------
# DEBUG VIEW (optional but very useful)
@@ -801,20 +799,8 @@ def ask_question(question, show_sources=False, filter_term=None):
history_text += f"Q: {exchange['question']}\n"
history_text += f"A: {exchange['answer']}\n"
history_text += "\n"
if is_derivation_request(question):
prompt_instruction = (
"You are a deterministic logic engine. "
"1. Prioritize 'TYPE: rule' and 'GLOBAL CONSTRAINT' entries in the CONTEXT over all other data. "
"2. If the CONTEXT defines a multi-step procedure (e.g., Sequence Integrity or Alphabetical Anchors), execute those steps exactly. "
"3. Resolve all naming conflicts using the provided 'CORRECTION LOGIC'. "
"4. Output ONLY the final resolved string of elements. "
"5. Do not show your work, intermediate calculations, or code. "
"6. Do not use LaTeX, boxes, or mathematical notation. Output the result as plain text only. "
"End your answer with a single period."
)
elif is_creative_request(question):
if is_creative_request(question):
prompt_instruction = (
"You are a creative assistant. "
"Use the provided context as inspiration. "
@@ -822,12 +808,23 @@ def ask_question(question, show_sources=False, filter_term=None):
"End your response with a single period."
)
else:
prompt_instruction = (
"You are a helpful research assistant. "
"Answer ONLY using the provided context. "
"Be direct and concise. Never repeat the context or instructions. "
"Never echo the question. End your answer with a single period."
)
# prompt_instruction = (
# "You are a helpful research assistant. "
# "Answer ONLY using the provided context. "
# "Be direct and concise. Never repeat the context or instructions. "
# "Never echo the question. End your answer with a single period."
# )
prompt_instruction=(
"You are a helpful research assistant. "
"Restrict your response strictly to the provided context. "
"If the source material is exhausted, stop writing. "
"If a relationship or entity is not explicitly documented in the context, do not include it. "
"Do not infer, supplement, or use external training knowledge. "
"Be direct and concise. "
"Never repeat the context or instructions. "
"Never echo the question. "
"End your answer with a single period."
)
with lm_model.chat_session(system_prompt=prompt_instruction):
user_message = (