Cortex Finetuning Experiments¶

This notebook takes you through evaluating a series of

In [ ]:

from snowflake.snowpark import Session

connection_params = {
    "account": "...",
    "user": "...",
    "password": "...",
    "role": "...",
    "database": "...",
    "schema": "...",
    "warehouse": "...",
}

# Create a Snowflake session
snowpark_session = Session.builder.configs(connection_params).create()

from snowflake.snowpark import Session connection_params = { "account": "...", "user": "...", "password": "...", "role": "...", "database": "...", "schema": "...", "warehouse": "...", } # Create a Snowflake session snowpark_session = Session.builder.configs(connection_params).create()

In [ ]:

from trulens.core import TruSession
from trulens.connectors.snowflake import SnowflakeConnector
conn = SnowflakeConnector(
    account="...",
    user="...",
    password="...",
    database="...",
    schema="...",
    warehouse="...",
    role="...",
)
session = TruSession(connector=conn)

from trulens.core import TruSession from trulens.connectors.snowflake import SnowflakeConnector conn = SnowflakeConnector( account="...", user="...", password="...", database="...", schema="...", warehouse="...", role="...", ) session = TruSession(connector=conn)

In [ ]:

from trulens.dashboard import run_dashboard

run_dashboard(session)

from trulens.dashboard import run_dashboard run_dashboard(session)

In [ ]:

instruction_prompt = """
        You are an agent that helps organize requests that come to our support team. 

        The request category is the reason why the customer reached out. These are the possible types of request categories:

        Roaming fees
        Slow data speed
        Lost phone
        Add new line
        Closing account

        Try doing it for this request and return only the request category only.
        
        """

instruction_prompt = """ You are an agent that helps organize requests that come to our support team. The request category is the reason why the customer reached out. These are the possible types of request categories: Roaming fees Slow data speed Lost phone Add new line Closing account Try doing it for this request and return only the request category only. """

In [ ]:

from trulens.apps.custom import instrument
import snowflake.connector
import json

# Create a Snowflake connection
snowflake_connection = snowflake.connector.connect(
    **connection_params
)

class Support_Ticket_Classifier:

    @instrument
    def __init__(self, model, instruction_prompt):
        self.model = model
        self.instruction_prompt = instruction_prompt

    @instrument
    def render_prompt(self, ticket):
        return self.instruction_prompt + ticket
        
    @instrument
    def classify_ticket(self, ticket):
        rendered_prompt = self.render_prompt(ticket)

        def escape_string_for_sql(input_string):
            escaped_string = input_string.replace("\\", "\\\\")
            escaped_string = escaped_string.replace("'", "''")
            return escaped_string

        rendered_prompt = escape_string_for_sql(rendered_prompt)

        cursor = snowpark_session.connection.cursor()
        try:
            # We use `snowflake.connector.cursor.SnowflakeCursor::execute` to
            # execute the query instead of
            # `snowflake.snowpark.session.Session::sql` since the latter is not
            # thread-safe.
            res = cursor.execute(f"""
                SELECT SNOWFLAKE.CORTEX.COMPLETE(
                    '{self.model}',
                    [
                        {{'role': 'user', 'content': '{rendered_prompt.replace("'", "''")}' }}
                    ], 
                    {{
                        'temperature': 0
                    }}
                )
            """).fetchall() 
        finally:
            cursor.close()

        if len(res) == 0:
            return "No response from cortex function"
        label = json.loads(res[0][0])["choices"][0]["messages"]

        return label

from trulens.apps.custom import instrument import snowflake.connector import json # Create a Snowflake connection snowflake_connection = snowflake.connector.connect( **connection_params ) class Support_Ticket_Classifier: @instrument def __init__(self, model, instruction_prompt): self.model = model self.instruction_prompt = instruction_prompt @instrument def render_prompt(self, ticket): return self.instruction_prompt + ticket @instrument def classify_ticket(self, ticket): rendered_prompt = self.render_prompt(ticket) def escape_string_for_sql(input_string): escaped_string = input_string.replace("\\", "\\\\") escaped_string = escaped_string.replace("'", "''") return escaped_string rendered_prompt = escape_string_for_sql(rendered_prompt) cursor = snowpark_session.connection.cursor() try: # We use `snowflake.connector.cursor.SnowflakeCursor::execute` to # execute the query instead of # `snowflake.snowpark.session.Session::sql` since the latter is not # thread-safe. res = cursor.execute(f""" SELECT SNOWFLAKE.CORTEX.COMPLETE( '{self.model}', [ {{'role': 'user', 'content': '{rendered_prompt.replace("'", "''")}' }} ], {{ 'temperature': 0 }} ) """).fetchall() finally: cursor.close() if len(res) == 0: return "No response from cortex function" label = json.loads(res[0][0])["choices"][0]["messages"] return label

In [ ]:

support_ticket_classifier_mistral_7b = Support_Ticket_Classifier("mistral-7b", instruction_prompt)
support_ticket_classifier_mistral_large = Support_Ticket_Classifier("mistral-large2", instruction_prompt)
support_ticket_classifier_mistral_7b_finetuned = Support_Ticket_Classifier("SUPPORT_TICKETS_FINETUNED_MISTRAL_7B", instruction_prompt)

support_ticket_classifier_mistral_7b = Support_Ticket_Classifier("mistral-7b", instruction_prompt) support_ticket_classifier_mistral_large = Support_Ticket_Classifier("mistral-large2", instruction_prompt) support_ticket_classifier_mistral_7b_finetuned = Support_Ticket_Classifier("SUPPORT_TICKETS_FINETUNED_MISTRAL_7B", instruction_prompt)

In [ ]:

support_tickets = ["I would like to close my account as I am no longer using the services. Please confirm the necessary steps to complete this process. Can you guide me through closing my account? I have found another provider that better suits my needs. I wish to terminate my account due to relocation. Kindly assist me with the required steps.",
    "I am writing to bring to your attention an issue with my recent cell phone bill. During my trip to Europe for two weeks, I noticed additional charges labeled as 'international fees' amounting to $130. These charges were not communicated to me. I request a detailed explanation and a refund. Thank you for addressing this matter promptly.",
    "Hello, I would like to add my daughter to my plan. I need it activated by her birthday at the end of the week.",
    "I am experiencing slow data speeds on my phone. I have attempted to restart my device and check for software updates, but the issue persists. Please provide guidance on resolving this problem. I heavily rely on my phone for work and require a swift solution. Thank you for your support.",
    "I misplaced my phone while using the subway. Despite multiple attempts to call it, it appears to be turned off. I am concerned about my personal data and would like to know the steps for remotely locking and erasing the data on my phone. Please advise on how to proceed. Thank you for your assistance.",
    "My bill is too high after my travel to Canada. I was not informed about additional fees for using my phone abroad. I request a detailed breakdown of these charges and a refund. I appreciate your prompt attention to this issue.",
    "I am moving to france and need to end my plan. Please help me do so by the end of the month.",
    "I am writing to bring to your attention an issue with my recent cell phone bill. During my trip to Europe for two weeks, I noticed additional charges labeled as 'international fees' amounting to $130. These charges were not communicated to me. I request a detailed explanation and a refund. Thank you for addressing this matter promptly.",
    "Hello, I would like to add a new line to my existing cell phone plan. Kindly activate it within the next 9 days. If there are any further steps or information needed, please inform me. Thank you for your prompt assistance.",
    "I am experiencing slow data speeds on my phone. I have attempted to restart my device and check for software updates, but the issue persists. Please provide guidance on resolving this problem. I heavily rely on my phone for work and require a swift solution. Thank you for your support.",
    "My phone screen is shattered and I need to replace it. Can you help me with the steps to do so?",
    "My kid purchased a game on my phone without my permission. I would like to dispute the charge and remove the game from my account. Can you assist me with this issue?",
    "I am moving to a new country and need to close my account. Can you help me with the steps to do so?",
    "I don't have service at my house. I tried restarting it and it didn't work. Can you help me?",
    "I am experiencing frequent call drops and poor call quality on my phone. This issue has been ongoing for the past week. Please assist me in resolving this problem as it is affecting my ability to communicate effectively.",
    "I accidentally subscribed to a premium SMS service and I am being charged for it. I did not authorize this subscription and would like to cancel it immediately. Kindly refund the charges as well.",
    "I am unable to send or receive text messages on my phone. I have checked my message settings and restarted my device, but the issue persists. Please provide a solution to restore my messaging functionality.",
    "I received a bill that includes charges for international calls that I did not make. I have not traveled outside the country and suspect fraudulent activity. Please investigate and remove these charges from my bill.",
    "I recently upgraded my phone and now I am unable to access mobile data. I have verified that my data plan is active and tried resetting network settings, but the issue remains. Please help me restore my mobile data connection.",
    "I have been charged for a device that I returned to your company. I have the tracking number and proof of return. Please update my account and refund the charges for the returned device.",
    "I am unable to access voicemail on my phone. When I try to retrieve my voicemail messages, I receive an error message. Please assist me in resolving this issue so that I can access my voicemail.",
    "I have been experiencing frequent network outages in my area. This is causing disruptions to my work and communication. Please investigate and resolve the network issues in my location.",
    "I received a promotional offer for a discounted plan, but I was charged the regular price on my bill. Please adjust my bill to reflect the correct discounted amount as per the promotional offer.",
    "I am unable to make or receive calls on my phone. When I try to make a call, I hear a busy tone. Please help me troubleshoot this issue and restore my calling functionality."
    ]

support_tickets = ["I would like to close my account as I am no longer using the services. Please confirm the necessary steps to complete this process. Can you guide me through closing my account? I have found another provider that better suits my needs. I wish to terminate my account due to relocation. Kindly assist me with the required steps.", "I am writing to bring to your attention an issue with my recent cell phone bill. During my trip to Europe for two weeks, I noticed additional charges labeled as 'international fees' amounting to $130. These charges were not communicated to me. I request a detailed explanation and a refund. Thank you for addressing this matter promptly.", "Hello, I would like to add my daughter to my plan. I need it activated by her birthday at the end of the week.", "I am experiencing slow data speeds on my phone. I have attempted to restart my device and check for software updates, but the issue persists. Please provide guidance on resolving this problem. I heavily rely on my phone for work and require a swift solution. Thank you for your support.", "I misplaced my phone while using the subway. Despite multiple attempts to call it, it appears to be turned off. I am concerned about my personal data and would like to know the steps for remotely locking and erasing the data on my phone. Please advise on how to proceed. Thank you for your assistance.", "My bill is too high after my travel to Canada. I was not informed about additional fees for using my phone abroad. I request a detailed breakdown of these charges and a refund. I appreciate your prompt attention to this issue.", "I am moving to france and need to end my plan. Please help me do so by the end of the month.", "I am writing to bring to your attention an issue with my recent cell phone bill. During my trip to Europe for two weeks, I noticed additional charges labeled as 'international fees' amounting to $130. These charges were not communicated to me. I request a detailed explanation and a refund. Thank you for addressing this matter promptly.", "Hello, I would like to add a new line to my existing cell phone plan. Kindly activate it within the next 9 days. If there are any further steps or information needed, please inform me. Thank you for your prompt assistance.", "I am experiencing slow data speeds on my phone. I have attempted to restart my device and check for software updates, but the issue persists. Please provide guidance on resolving this problem. I heavily rely on my phone for work and require a swift solution. Thank you for your support.", "My phone screen is shattered and I need to replace it. Can you help me with the steps to do so?", "My kid purchased a game on my phone without my permission. I would like to dispute the charge and remove the game from my account. Can you assist me with this issue?", "I am moving to a new country and need to close my account. Can you help me with the steps to do so?", "I don't have service at my house. I tried restarting it and it didn't work. Can you help me?", "I am experiencing frequent call drops and poor call quality on my phone. This issue has been ongoing for the past week. Please assist me in resolving this problem as it is affecting my ability to communicate effectively.", "I accidentally subscribed to a premium SMS service and I am being charged for it. I did not authorize this subscription and would like to cancel it immediately. Kindly refund the charges as well.", "I am unable to send or receive text messages on my phone. I have checked my message settings and restarted my device, but the issue persists. Please provide a solution to restore my messaging functionality.", "I received a bill that includes charges for international calls that I did not make. I have not traveled outside the country and suspect fraudulent activity. Please investigate and remove these charges from my bill.", "I recently upgraded my phone and now I am unable to access mobile data. I have verified that my data plan is active and tried resetting network settings, but the issue remains. Please help me restore my mobile data connection.", "I have been charged for a device that I returned to your company. I have the tracking number and proof of return. Please update my account and refund the charges for the returned device.", "I am unable to access voicemail on my phone. When I try to retrieve my voicemail messages, I receive an error message. Please assist me in resolving this issue so that I can access my voicemail.", "I have been experiencing frequent network outages in my area. This is causing disruptions to my work and communication. Please investigate and resolve the network issues in my location.", "I received a promotional offer for a discounted plan, but I was charged the regular price on my bill. Please adjust my bill to reflect the correct discounted amount as per the promotional offer.", "I am unable to make or receive calls on my phone. When I try to make a call, I hear a busy tone. Please help me troubleshoot this issue and restore my calling functionality." ]

In [ ]:

golden_set = [
    {
        "query": "I would like to close my account as I am no longer using the services. Please confirm the necessary steps to complete this process. Can you guide me through closing my account? I have found another provider that better suits my needs. I wish to terminate my account due to relocation. Kindly assist me with the required steps.",
        "expected_response": "Closing account"
    },
    {
        "query": "Hello, I would like to add my daughter to my plan. I need it activated by her birthday at the end of the week.",
        "expected_response": "Add new line"
    },
    {
        "query": "I am experiencing slow data speeds on my phone. I have attempted to restart my device and check for software updates, but the issue persists. Please provide guidance on resolving this problem. I heavily rely on my phone for work and require a swift solution. Thank you for your support.",
        "expected_response": "Slow data speed"
    },
    {
        "query": "I misplaced my phone while using the subway. Despite multiple attempts to call it, it appears to be turned off. I am concerned about my personal data and would like to know the steps for remotely locking and erasing the data on my phone. Please advise on how to proceed. Thank you for your assistance.",
        "expected_response": "Lost phone"
    },
    {
        "query": "My bill is too high after my travel to Canada. I was not informed about additional fees for using my phone abroad. I request a detailed breakdown of these charges and a refund. I appreciate your prompt attention to this issue.",
        "expected_response": "Roaming fees"
    },
    {
        "query": "I am moving to france and need to end my plan. Please help me do so by the end of the month.",
        "expected_response": "Closing account"
    },
    {
        "query": "I am writing to bring to your attention an issue with my recent cell phone bill. During my trip to Europe for two weeks, I noticed additional charges labeled as 'international fees' amounting to $130. These charges were not communicated to me. I request a detailed explanation and a refund. Thank you for addressing this matter promptly.",
        "expected_response": "Roaming fees"
    },
    {
        "query": "Hello, I would like to add a new line to my existing cell phone plan. Kindly activate it within the next 9 days. If there are any further steps or information needed, please inform me. Thank you for your prompt assistance.",
        "expected_response": "Add new line"
    },
    {
        "query": "I am experiencing slow data speeds on my phone. I have attempted to restart my device and check for software updates, but the issue persists. Please provide guidance on resolving this problem. I heavily rely on my phone for work and require a swift solution. Thank you for your support.",
        "expected_response": "Slow data speed"
    },
    {
        "query": "I am moving to a new country and need to close my account. Can you help me with the steps to do so?",
        "expected_response": "Closing account"
    }
]

golden_set = [ { "query": "I would like to close my account as I am no longer using the services. Please confirm the necessary steps to complete this process. Can you guide me through closing my account? I have found another provider that better suits my needs. I wish to terminate my account due to relocation. Kindly assist me with the required steps.", "expected_response": "Closing account" }, { "query": "Hello, I would like to add my daughter to my plan. I need it activated by her birthday at the end of the week.", "expected_response": "Add new line" }, { "query": "I am experiencing slow data speeds on my phone. I have attempted to restart my device and check for software updates, but the issue persists. Please provide guidance on resolving this problem. I heavily rely on my phone for work and require a swift solution. Thank you for your support.", "expected_response": "Slow data speed" }, { "query": "I misplaced my phone while using the subway. Despite multiple attempts to call it, it appears to be turned off. I am concerned about my personal data and would like to know the steps for remotely locking and erasing the data on my phone. Please advise on how to proceed. Thank you for your assistance.", "expected_response": "Lost phone" }, { "query": "My bill is too high after my travel to Canada. I was not informed about additional fees for using my phone abroad. I request a detailed breakdown of these charges and a refund. I appreciate your prompt attention to this issue.", "expected_response": "Roaming fees" }, { "query": "I am moving to france and need to end my plan. Please help me do so by the end of the month.", "expected_response": "Closing account" }, { "query": "I am writing to bring to your attention an issue with my recent cell phone bill. During my trip to Europe for two weeks, I noticed additional charges labeled as 'international fees' amounting to $130. These charges were not communicated to me. I request a detailed explanation and a refund. Thank you for addressing this matter promptly.", "expected_response": "Roaming fees" }, { "query": "Hello, I would like to add a new line to my existing cell phone plan. Kindly activate it within the next 9 days. If there are any further steps or information needed, please inform me. Thank you for your prompt assistance.", "expected_response": "Add new line" }, { "query": "I am experiencing slow data speeds on my phone. I have attempted to restart my device and check for software updates, but the issue persists. Please provide guidance on resolving this problem. I heavily rely on my phone for work and require a swift solution. Thank you for your support.", "expected_response": "Slow data speed" }, { "query": "I am moving to a new country and need to close my account. Can you help me with the steps to do so?", "expected_response": "Closing account" } ]

In [ ]:

from trulens.core import Feedback
from trulens.core import Select
import snowflake.connector
from trulens.providers.cortex import Cortex
from trulens.core import Provider
from string import punctuation
from trulens.feedback import GroundTruthAgreement

# Create a Snowflake connection
snowflake_connection = snowflake.connector.connect(
    **connection_params
)
provider = Cortex(
    snowflake_connection,
    model_engine="mistral-large2",
)

class CustomProvider(Provider):
    def valid_category(self, response: str) -> float:
        """
        Custom feedback function to validate the category of a support ticket.

        Args:
            response (str): text to be evaluated if it is in the list of valid categories.

        Returns:
            float: 0 if the response is not in the list of valid categories, 1 otherwise.
        """
        response = response.lower()
        response = response.translate(str.maketrans('', '', punctuation))
        response = response.strip()
        valid_categories = [
            "roaming fees",
            "slow data speed",
            "lost phone",
            "add new line",
            "closing account"
        ]
        if response in valid_categories:
            return 1.0
        else:
            return 0.0
    
# Question/answer relevance between overall question and answer.
f_answer_relevance = (
    Feedback(provider.relevance_with_cot_reasons, name="Answer Relevance (Label-Free)")
    .on(Select.RecordCalls.render_prompt.rets)
    .on_output()
)

custom_provider = CustomProvider()

f_valid_category = (
    Feedback(custom_provider.valid_category, name="Valid Category (Exact Match)")
    .on_output()
)

f_semantic_agreement = (
    Feedback(
    GroundTruthAgreement(golden_set, provider=provider).agreement_measure,
    name="Semantic Agreement with Ground Truth (LLM Judge)")
    .on_input()
    .on_output()
)

from trulens.core import Feedback from trulens.core import Select import snowflake.connector from trulens.providers.cortex import Cortex from trulens.core import Provider from string import punctuation from trulens.feedback import GroundTruthAgreement # Create a Snowflake connection snowflake_connection = snowflake.connector.connect( **connection_params ) provider = Cortex( snowflake_connection, model_engine="mistral-large2", ) class CustomProvider(Provider): def valid_category(self, response: str) -> float: """ Custom feedback function to validate the category of a support ticket. Args: response (str): text to be evaluated if it is in the list of valid categories. Returns: float: 0 if the response is not in the list of valid categories, 1 otherwise. """ response = response.lower() response = response.translate(str.maketrans('', '', punctuation)) response = response.strip() valid_categories = [ "roaming fees", "slow data speed", "lost phone", "add new line", "closing account" ] if response in valid_categories: return 1.0 else: return 0.0 # Question/answer relevance between overall question and answer. f_answer_relevance = ( Feedback(provider.relevance_with_cot_reasons, name="Answer Relevance (Label-Free)") .on(Select.RecordCalls.render_prompt.rets) .on_output() ) custom_provider = CustomProvider() f_valid_category = ( Feedback(custom_provider.valid_category, name="Valid Category (Exact Match)") .on_output() ) f_semantic_agreement = ( Feedback( GroundTruthAgreement(golden_set, provider=provider).agreement_measure, name="Semantic Agreement with Ground Truth (LLM Judge)") .on_input() .on_output() )

In [ ]:

from trulens.apps.custom import TruCustomApp

tru_recorder_support_ticket_classifier_mistral_7b = TruCustomApp(
    support_ticket_classifier_mistral_7b,
    app_name="Support Ticket Classifier",
    app_version="mistral 7b",
    metadata={"model": "mistral-7b"},
    feedbacks = [f_valid_category, f_answer_relevance, f_semantic_agreement]
)

from trulens.apps.custom import TruCustomApp tru_recorder_support_ticket_classifier_mistral_7b = TruCustomApp( support_ticket_classifier_mistral_7b, app_name="Support Ticket Classifier", app_version="mistral 7b", metadata={"model": "mistral-7b"}, feedbacks = [f_valid_category, f_answer_relevance, f_semantic_agreement] )

In [ ]:

for ticket in support_tickets:
    print(f"Ticket: {ticket}")
    with tru_recorder_support_ticket_classifier_mistral_7b as recording:
        label_small = support_ticket_classifier_mistral_7b.classify_ticket(ticket)
        print(f"mistral 7b label: {label_small}")

for ticket in support_tickets: print(f"Ticket: {ticket}") with tru_recorder_support_ticket_classifier_mistral_7b as recording: label_small = support_ticket_classifier_mistral_7b.classify_ticket(ticket) print(f"mistral 7b label: {label_small}")

In [ ]:

tru_recorder_support_ticket_classifier_mistral_large = TruCustomApp(
    support_ticket_classifier_mistral_large,
    app_name="Support Ticket Classifier",
    app_version="mistral large",
    metadata={"model": "llama3.1-405b"},
    feedbacks = [f_valid_category, f_answer_relevance, f_semantic_agreement],
)

tru_recorder_support_ticket_classifier_mistral_large = TruCustomApp( support_ticket_classifier_mistral_large, app_name="Support Ticket Classifier", app_version="mistral large", metadata={"model": "llama3.1-405b"}, feedbacks = [f_valid_category, f_answer_relevance, f_semantic_agreement], )

In [ ]:

for ticket in support_tickets:
    print(f"Ticket: {ticket}")
    with tru_recorder_support_ticket_classifier_mistral_large:
        label_large = support_ticket_classifier_mistral_large.classify_ticket(ticket)
        print(f"mistral large label: {label_large}")

for ticket in support_tickets: print(f"Ticket: {ticket}") with tru_recorder_support_ticket_classifier_mistral_large: label_large = support_ticket_classifier_mistral_large.classify_ticket(ticket) print(f"mistral large label: {label_large}")

In [ ]:

tru_recorder_support_ticket_classifier_mistral_7b_finetuned = TruCustomApp(
    support_ticket_classifier_mistral_7b_finetuned,
    app_name="Support Ticket Classifier",
    app_version="mistral 7b finetuned",
    metadata={"model": "mistral-7b finetuned"},
    feedbacks = [f_valid_category, f_answer_relevance, f_semantic_agreement],
)

tru_recorder_support_ticket_classifier_mistral_7b_finetuned = TruCustomApp( support_ticket_classifier_mistral_7b_finetuned, app_name="Support Ticket Classifier", app_version="mistral 7b finetuned", metadata={"model": "mistral-7b finetuned"}, feedbacks = [f_valid_category, f_answer_relevance, f_semantic_agreement], )

In [ ]:

for ticket in support_tickets:
    print(f"Ticket: {ticket}")
    with tru_recorder_support_ticket_classifier_mistral_7b_finetuned:
        label_finetuned = support_ticket_classifier_mistral_7b_finetuned.classify_ticket(ticket)
        print(f"mistral 7b finetuned label: {label_finetuned}")

for ticket in support_tickets: print(f"Ticket: {ticket}") with tru_recorder_support_ticket_classifier_mistral_7b_finetuned: label_finetuned = support_ticket_classifier_mistral_7b_finetuned.classify_ticket(ticket) print(f"mistral 7b finetuned label: {label_finetuned}")