I have a Streamlit chatbot that uses Langchain and OpenAI. I want users to rate responses with thumbs up/down buttons and send this feedback to LangSmith for tracking. The button shows up and users can click it, but when I check LangSmith dashboard, no feedback data appears there. All the chat conversations are logged correctly in LangSmith, so the connection works fine. Just the user ratings are not being saved. Has anyone faced this issue before?
import os
import streamlit as st
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import FAISS
from langchain.chat_models import ChatOpenAI
from langchain.chains import RetrievalQA
from langchain.memory import ConversationSummaryMemory
from langchain.document_loaders import DirectoryLoader
from langchain.callbacks import collect_runs
from langsmith import Client
from streamlit_feedback import streamlit_feedback
from dotenv import load_dotenv
load_dotenv()
@st.cache_resource
def setup_qa_system():
model = ChatOpenAI(model='gpt-3.5-turbo', temperature=0.2, openai_api_key=os.environ['OPENAI_API_KEY'])
loader = DirectoryLoader('docs/')
splitter = RecursiveCharacterTextSplitter(chunk_size=2000, chunk_overlap=150)
chunks = splitter.split_documents(loader.load())
embeddings = OpenAIEmbeddings(chunk_size=2000)
vector_db = FAISS.from_documents(documents=chunks, embedding=embeddings)
memory = ConversationSummaryMemory(memory_key="history", return_messages=True)
qa_system = RetrievalQA.from_chain_type(
llm=model,
retriever=vector_db.as_retriever(),
memory=memory
)
return qa_system
def process_user_rating(rating_data, reaction=None):
st.toast(f"Rating received: {rating_data}", icon=reaction)
return rating_data.update({"extra_data": 456})
def save_rating_to_langsmith():
user_rating = st.session_state.get("rating_info")
current_run_id = st.session_state.current_run
rating_values = {
"👍": 1.0,
"👎": 0.0,
"😊": 0.8,
"😔": 0.2
}
rating_score = rating_values.get(user_rating.get("score"))
if rating_score is not None:
rating_label = f"user_rating {user_rating.get('score')}"
try:
result = langsmith_client.create_feedback(
run_id=current_run_id,
feedback_type=rating_label,
score=rating_score
)
st.session_state.saved_rating = {
"rating_id": str(result.id),
"value": rating_score
}
st.success(f"Rating saved successfully: {result.id}")
except Exception as error:
st.error(f"Could not save rating: {error}")
else:
st.warning("Rating value not recognized")
langsmith_client = Client()
qa_chain = setup_qa_system()
if "chat_history" not in st.session_state:
st.session_state["chat_history"] = []
for msg in st.session_state.chat_history:
with st.chat_message(msg["type"]):
st.markdown(msg["text"])
user_question = st.chat_input("Ask me anything...")
if user_question:
st.session_state.chat_history.append({"type": "user", "text": user_question})
with st.chat_message("user"):
st.markdown(user_question)
with collect_runs() as callback:
bot_response = qa_chain({"query": user_question})
if callback.traced_runs:
current_run = callback.traced_runs[0].id
st.session_state.current_run = current_run
else:
current_run = None
bot_answer = bot_response['result']
st.session_state.chat_history.append({"type": "assistant", "text": bot_answer})
with st.chat_message("assistant"):
st.markdown(bot_answer)
if bot_answer:
user_rating = streamlit_feedback(
feedback_type="thumbs",
key=f"rating_{current_run}"
)
if user_rating:
st.session_state["rating_info"] = user_rating
save_rating_to_langsmith()