From d3ae7baf3007267ff0261c26a5b2e12dfaadee78 Mon Sep 17 00:00:00 2001 From: jacoblee93 Date: Fri, 27 Dec 2024 15:35:00 -0800 Subject: [PATCH 01/17] Update with_structured_output default for OpenAI --- libs/partners/openai/langchain_openai/chat_models/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py index 142e7eca1a84b..f2b50e2f313af 100644 --- a/libs/partners/openai/langchain_openai/chat_models/base.py +++ b/libs/partners/openai/langchain_openai/chat_models/base.py @@ -1173,7 +1173,7 @@ def with_structured_output( *, method: Literal[ "function_calling", "json_mode", "json_schema" - ] = "function_calling", + ] = "json_schema", include_raw: bool = False, strict: Optional[bool] = None, **kwargs: Any, From f85fff9eb98f1b6cdca06d5bb652ee7d46fafa84 Mon Sep 17 00:00:00 2001 From: Bagatur Date: Sat, 28 Dec 2024 17:39:29 -0500 Subject: [PATCH 02/17] fmt --- .../langchain_openai/chat_models/base.py | 474 +++++++++++------- 1 file changed, 286 insertions(+), 188 deletions(-) diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py index f2b50e2f313af..5cbb12791395f 100644 --- a/libs/partners/openai/langchain_openai/chat_models/base.py +++ b/libs/partners/openai/langchain_openai/chat_models/base.py @@ -1173,7 +1173,7 @@ def with_structured_output( *, method: Literal[ "function_calling", "json_mode", "json_schema" - ] = "json_schema", + ] = "function_calling", include_raw: bool = False, strict: Optional[bool] = None, **kwargs: Any, @@ -1261,193 +1261,6 @@ def with_structured_output( Support for ``strict`` argument added. Support for ``method`` = "json_schema" added. - - .. note:: Planned breaking changes in version `0.3.0` - - - ``method`` default will be changed to "json_schema" from - "function_calling". - - ``strict`` will default to True when ``method`` is - "function_calling" as of version `0.3.0`. - - - .. dropdown:: Example: schema=Pydantic class, method="function_calling", include_raw=False, strict=True - - Note, OpenAI has a number of restrictions on what types of schemas can be - provided if ``strict`` = True. When using Pydantic, our model cannot - specify any Field metadata (like min/max constraints) and fields cannot - have default values. - - See all constraints here: https://platform.openai.com/docs/guides/structured-outputs/supported-schemas - - .. code-block:: python - - from typing import Optional - - from langchain_openai import ChatOpenAI - from pydantic import BaseModel, Field - - - class AnswerWithJustification(BaseModel): - '''An answer to the user question along with justification for the answer.''' - - answer: str - justification: Optional[str] = Field( - default=..., description="A justification for the answer." - ) - - - llm = ChatOpenAI(model="gpt-4o", temperature=0) - structured_llm = llm.with_structured_output( - AnswerWithJustification, strict=True - ) - - structured_llm.invoke( - "What weighs more a pound of bricks or a pound of feathers" - ) - - # -> AnswerWithJustification( - # answer='They weigh the same', - # justification='Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume or density of the objects may differ.' - # ) - - .. dropdown:: Example: schema=Pydantic class, method="function_calling", include_raw=True - - .. code-block:: python - - from langchain_openai import ChatOpenAI - from pydantic import BaseModel - - - class AnswerWithJustification(BaseModel): - '''An answer to the user question along with justification for the answer.''' - - answer: str - justification: str - - - llm = ChatOpenAI(model="gpt-4o", temperature=0) - structured_llm = llm.with_structured_output( - AnswerWithJustification, include_raw=True - ) - - structured_llm.invoke( - "What weighs more a pound of bricks or a pound of feathers" - ) - # -> { - # 'raw': AIMessage(content='', additional_kwargs={'tool_calls': [{'id': 'call_Ao02pnFYXD6GN1yzc0uXPsvF', 'function': {'arguments': '{"answer":"They weigh the same.","justification":"Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume or density of the objects may differ."}', 'name': 'AnswerWithJustification'}, 'type': 'function'}]}), - # 'parsed': AnswerWithJustification(answer='They weigh the same.', justification='Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume or density of the objects may differ.'), - # 'parsing_error': None - # } - - .. dropdown:: Example: schema=TypedDict class, method="function_calling", include_raw=False - - .. code-block:: python - - # IMPORTANT: If you are using Python <=3.8, you need to import Annotated - # from typing_extensions, not from typing. - from typing_extensions import Annotated, TypedDict - - from langchain_openai import ChatOpenAI - - - class AnswerWithJustification(TypedDict): - '''An answer to the user question along with justification for the answer.''' - - answer: str - justification: Annotated[ - Optional[str], None, "A justification for the answer." - ] - - - llm = ChatOpenAI(model="gpt-4o", temperature=0) - structured_llm = llm.with_structured_output(AnswerWithJustification) - - structured_llm.invoke( - "What weighs more a pound of bricks or a pound of feathers" - ) - # -> { - # 'answer': 'They weigh the same', - # 'justification': 'Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume and density of the two substances differ.' - # } - - .. dropdown:: Example: schema=OpenAI function schema, method="function_calling", include_raw=False - - .. code-block:: python - - from langchain_openai import ChatOpenAI - - oai_schema = { - 'name': 'AnswerWithJustification', - 'description': 'An answer to the user question along with justification for the answer.', - 'parameters': { - 'type': 'object', - 'properties': { - 'answer': {'type': 'string'}, - 'justification': {'description': 'A justification for the answer.', 'type': 'string'} - }, - 'required': ['answer'] - } - } - - llm = ChatOpenAI(model="gpt-4o", temperature=0) - structured_llm = llm.with_structured_output(oai_schema) - - structured_llm.invoke( - "What weighs more a pound of bricks or a pound of feathers" - ) - # -> { - # 'answer': 'They weigh the same', - # 'justification': 'Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume and density of the two substances differ.' - # } - - .. dropdown:: Example: schema=Pydantic class, method="json_mode", include_raw=True - - .. code-block:: - - from langchain_openai import ChatOpenAI - from pydantic import BaseModel - - class AnswerWithJustification(BaseModel): - answer: str - justification: str - - llm = ChatOpenAI(model="gpt-4o", temperature=0) - structured_llm = llm.with_structured_output( - AnswerWithJustification, - method="json_mode", - include_raw=True - ) - - structured_llm.invoke( - "Answer the following question. " - "Make sure to return a JSON blob with keys 'answer' and 'justification'.\\n\\n" - "What's heavier a pound of bricks or a pound of feathers?" - ) - # -> { - # 'raw': AIMessage(content='{\\n "answer": "They are both the same weight.",\\n "justification": "Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight." \\n}'), - # 'parsed': AnswerWithJustification(answer='They are both the same weight.', justification='Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight.'), - # 'parsing_error': None - # } - - .. dropdown:: Example: schema=None, method="json_mode", include_raw=True - - .. code-block:: - - structured_llm = llm.with_structured_output(method="json_mode", include_raw=True) - - structured_llm.invoke( - "Answer the following question. " - "Make sure to return a JSON blob with keys 'answer' and 'justification'.\\n\\n" - "What's heavier a pound of bricks or a pound of feathers?" - ) - # -> { - # 'raw': AIMessage(content='{\\n "answer": "They are both the same weight.",\\n "justification": "Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight." \\n}'), - # 'parsed': { - # 'answer': 'They are both the same weight.', - # 'justification': 'Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight.' - # }, - # 'parsing_error': None - # } """ # noqa: E501 if kwargs: raise ValueError(f"Received unsupported arguments {kwargs}") @@ -2094,6 +1907,291 @@ async def _astream( async for chunk in super()._astream(*args, **kwargs): yield chunk + def with_structured_output( + self, + schema: Optional[_DictOrPydanticClass] = None, + *, + method: Literal["function_calling", "json_mode", "json_schema"] = "json_schema", + include_raw: bool = False, + strict: Optional[bool] = None, + **kwargs: Any, + ) -> Runnable[LanguageModelInput, _DictOrPydantic]: + """Model wrapper that returns outputs formatted to match the given schema. + + Args: + schema: + The output schema. Can be passed in as: + + - an OpenAI function/tool schema, + - a JSON Schema, + - a TypedDict class, + - or a Pydantic class. + + If ``schema`` is a Pydantic class then the model output will be a + Pydantic instance of that class, and the model-generated fields will be + validated by the Pydantic class. Otherwise the model output will be a + dict and will not be validated. See :meth:`langchain_core.utils.function_calling.convert_to_openai_tool` + for more on how to properly specify types and descriptions of + schema fields when specifying a Pydantic or TypedDict class. + + method: The method for steering model generation, one of: + + - "json_schema": + Uses OpenAI's Structured Output API: + https://platform.openai.com/docs/guides/structured-outputs + Supported for "gpt-4o-mini", "gpt-4o-2024-08-06", and later + models. + - "function_calling": + Uses OpenAI's tool-calling (formerly called function calling) + API: https://platform.openai.com/docs/guides/function-calling + - "json_mode": + Uses OpenAI's JSON mode. Note that if using JSON mode then you + must include instructions for formatting the output into the + desired schema into the model call: + https://platform.openai.com/docs/guides/structured-outputs/json-mode + + Learn more about the differences between the methods and which models + support which methods here: + + - https://platform.openai.com/docs/guides/structured-outputs/structured-outputs-vs-json-mode + - https://platform.openai.com/docs/guides/structured-outputs/function-calling-vs-response-format + + include_raw: + If False then only the parsed structured output is returned. If + an error occurs during model output parsing it will be raised. If True + then both the raw model response (a BaseMessage) and the parsed model + response will be returned. If an error occurs during output parsing it + will be caught and returned as well. The final output is always a dict + with keys "raw", "parsed", and "parsing_error". + strict: + + - True: + Model output is guaranteed to exactly match the schema. + The input schema will also be validated according to + https://platform.openai.com/docs/guides/structured-outputs/supported-schemas + - False: + Input schema will not be validated and model output will not be + validated. + - None: + ``strict`` argument will not be passed to the model. + + If ``method`` is "json_schema" or "function_calling" defaults to True. + If ``method`` is "json_mode" defaults to None. Can only be non-null + if ``method`` is "function_calling" or "json_schema". + + kwargs: Additional keyword args aren't supported. + + Returns: + A Runnable that takes same inputs as a :class:`langchain_core.language_models.chat.BaseChatModel`. + + | If ``include_raw`` is False and ``schema`` is a Pydantic class, Runnable outputs an instance of ``schema`` (i.e., a Pydantic object). Otherwise, if ``include_raw`` is False then Runnable outputs a dict. + + | If ``include_raw`` is True, then Runnable outputs a dict with keys: + + - "raw": BaseMessage + - "parsed": None if there was a parsing error, otherwise the type depends on the ``schema`` as described above. + - "parsing_error": Optional[BaseException] + + .. versionchanged:: 0.1.20 + + Added support for TypedDict class ``schema``. + + .. versionchanged:: 0.1.21 + + Support for ``strict`` argument added. + Support for ``method`` = "json_schema" added. + + .. versionchanged:: 0.3.0 + + - ``method`` default changed from "function_calling" to "json_schema". + - ``strict`` defaults to True instead of False when ``method`` is + "function_calling". + + .. dropdown:: Example: schema=Pydantic class, method="json_schema", include_raw=False, strict=True + + Note, OpenAI has a number of restrictions on what types of schemas can be + provided if ``strict`` = True. When using Pydantic, our model cannot + specify any Field metadata (like min/max constraints) and fields cannot + have default values. + + See all constraints here: https://platform.openai.com/docs/guides/structured-outputs/supported-schemas + + .. code-block:: python + + from typing import Optional + + from langchain_openai import ChatOpenAI + from pydantic import BaseModel, Field + + + class AnswerWithJustification(BaseModel): + '''An answer to the user question along with justification for the answer.''' + + answer: str + justification: Optional[str] = Field( + default=..., description="A justification for the answer." + ) + + + llm = ChatOpenAI(model="gpt-4o", temperature=0) + structured_llm = llm.with_structured_output( + AnswerWithJustification + ) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + + # -> AnswerWithJustification( + # answer='They weigh the same', + # justification='Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume or density of the objects may differ.' + # ) + + .. dropdown:: Example: schema=Pydantic class, method="json_schema", include_raw=True + + .. code-block:: python + + from langchain_openai import ChatOpenAI + from pydantic import BaseModel + + + class AnswerWithJustification(BaseModel): + '''An answer to the user question along with justification for the answer.''' + + answer: str + justification: str + + + llm = ChatOpenAI(model="gpt-4o", temperature=0) + structured_llm = llm.with_structured_output( + AnswerWithJustification, include_raw=True + ) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + # -> { + # 'raw': AIMessage(content='', additional_kwargs={'tool_calls': [{'id': 'call_Ao02pnFYXD6GN1yzc0uXPsvF', 'function': {'arguments': '{"answer":"They weigh the same.","justification":"Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume or density of the objects may differ."}', 'name': 'AnswerWithJustification'}, 'type': 'function'}]}), + # 'parsed': AnswerWithJustification(answer='They weigh the same.', justification='Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume or density of the objects may differ.'), + # 'parsing_error': None + # } + + .. dropdown:: Example: schema=TypedDict class, method="json_schema", include_raw=False + + .. code-block:: python + + # IMPORTANT: If you are using Python <=3.8, you need to import Annotated + # from typing_extensions, not from typing. + from typing_extensions import Annotated, TypedDict + + from langchain_openai import ChatOpenAI + + + class AnswerWithJustification(TypedDict): + '''An answer to the user question along with justification for the answer.''' + + answer: str + justification: Annotated[ + Optional[str], None, "A justification for the answer." + ] + + + llm = ChatOpenAI(model="gpt-4o", temperature=0) + structured_llm = llm.with_structured_output(AnswerWithJustification) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + # -> { + # 'answer': 'They weigh the same', + # 'justification': 'Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume and density of the two substances differ.' + # } + + .. dropdown:: Example: schema=OpenAI function schema, method="json_schema", include_raw=False + + .. code-block:: python + + from langchain_openai import ChatOpenAI + + oai_schema = { + 'name': 'AnswerWithJustification', + 'description': 'An answer to the user question along with justification for the answer.', + 'parameters': { + 'type': 'object', + 'properties': { + 'answer': {'type': 'string'}, + 'justification': {'description': 'A justification for the answer.', 'type': 'string'} + }, + 'required': ['answer'] + } + } + + llm = ChatOpenAI(model="gpt-4o", temperature=0) + structured_llm = llm.with_structured_output(oai_schema) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + # -> { + # 'answer': 'They weigh the same', + # 'justification': 'Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume and density of the two substances differ.' + # } + + .. dropdown:: Example: schema=Pydantic class, method="json_mode", include_raw=True + + .. code-block:: + + from langchain_openai import ChatOpenAI + from pydantic import BaseModel + + class AnswerWithJustification(BaseModel): + answer: str + justification: str + + llm = ChatOpenAI(model="gpt-4o", temperature=0) + structured_llm = llm.with_structured_output( + AnswerWithJustification, + method="json_mode", + include_raw=True + ) + + structured_llm.invoke( + "Answer the following question. " + "Make sure to return a JSON blob with keys 'answer' and 'justification'.\\n\\n" + "What's heavier a pound of bricks or a pound of feathers?" + ) + # -> { + # 'raw': AIMessage(content='{\\n "answer": "They are both the same weight.",\\n "justification": "Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight." \\n}'), + # 'parsed': AnswerWithJustification(answer='They are both the same weight.', justification='Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight.'), + # 'parsing_error': None + # } + + .. dropdown:: Example: schema=None, method="json_mode", include_raw=True + + .. code-block:: + + structured_llm = llm.with_structured_output(method="json_mode", include_raw=True) + + structured_llm.invoke( + "Answer the following question. " + "Make sure to return a JSON blob with keys 'answer' and 'justification'.\\n\\n" + "What's heavier a pound of bricks or a pound of feathers?" + ) + # -> { + # 'raw': AIMessage(content='{\\n "answer": "They are both the same weight.",\\n "justification": "Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight." \\n}'), + # 'parsed': { + # 'answer': 'They are both the same weight.', + # 'justification': 'Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight.' + # }, + # 'parsing_error': None + # } + """ # noqa: E501 + if method == "function_calling" and strict is None: + strict = True + return super().with_structured_output( + schema, method=method, include_raw=include_raw, strict=strict, **kwargs + ) + def _is_pydantic_class(obj: Any) -> bool: return isinstance(obj, type) and is_basemodel_subclass(obj) From e4accc55a8e7c3948b9978a750ccf8b22d32836d Mon Sep 17 00:00:00 2001 From: Bagatur Date: Sat, 28 Dec 2024 17:48:54 -0500 Subject: [PATCH 03/17] fmt --- .../langchain_openai/chat_models/azure.py | 293 +++++++++++++++++- .../langchain_openai/chat_models/base.py | 8 +- 2 files changed, 295 insertions(+), 6 deletions(-) diff --git a/libs/partners/openai/langchain_openai/chat_models/azure.py b/libs/partners/openai/langchain_openai/chat_models/azure.py index 2e1e5f8abfe03..4fdab27db89f5 100644 --- a/libs/partners/openai/langchain_openai/chat_models/azure.py +++ b/libs/partners/openai/langchain_openai/chat_models/azure.py @@ -18,13 +18,15 @@ ) import openai +from langchain_core.language_models import LanguageModelInput from langchain_core.language_models.chat_models import LangSmithParams from langchain_core.messages import BaseMessage from langchain_core.outputs import ChatResult +from langchain_core.runnables import Runnable from langchain_core.utils import from_env, secret_from_env from langchain_core.utils.pydantic import is_basemodel_subclass from pydantic import BaseModel, Field, SecretStr, model_validator -from typing_extensions import Self +from typing_extensions import Literal, Self from langchain_openai.chat_models.base import BaseChatOpenAI @@ -737,3 +739,292 @@ def _create_chat_result( ) return chat_result + + def with_structured_output( + self, + schema: Optional[_DictOrPydanticClass] = None, + *, + method: Literal["function_calling", "json_mode", "json_schema"] = "json_schema", + include_raw: bool = False, + strict: Optional[bool] = None, + **kwargs: Any, + ) -> Runnable[LanguageModelInput, _DictOrPydantic]: + """Model wrapper that returns outputs formatted to match the given schema. + + Args: + schema: + The output schema. Can be passed in as: + + - a JSON Schema, + - a TypedDict class, + - or a Pydantic class, + - an OpenAI function/tool schema. + + If ``schema`` is a Pydantic class then the model output will be a + Pydantic instance of that class, and the model-generated fields will be + validated by the Pydantic class. Otherwise the model output will be a + dict and will not be validated. See :meth:`langchain_core.utils.function_calling.convert_to_openai_tool` + for more on how to properly specify types and descriptions of + schema fields when specifying a Pydantic or TypedDict class. + + method: The method for steering model generation, one of: + + - "json_schema": + Uses OpenAI's Structured Output API: + https://platform.openai.com/docs/guides/structured-outputs + Supported for "gpt-4o-mini", "gpt-4o-2024-08-06", and later + models. + - "function_calling": + Uses OpenAI's tool-calling (formerly called function calling) + API: https://platform.openai.com/docs/guides/function-calling + - "json_mode": + Uses OpenAI's JSON mode. Note that if using JSON mode then you + must include instructions for formatting the output into the + desired schema into the model call: + https://platform.openai.com/docs/guides/structured-outputs/json-mode + + Learn more about the differences between the methods and which models + support which methods here: + + - https://platform.openai.com/docs/guides/structured-outputs/structured-outputs-vs-json-mode + - https://platform.openai.com/docs/guides/structured-outputs/function-calling-vs-response-format + + include_raw: + If False then only the parsed structured output is returned. If + an error occurs during model output parsing it will be raised. If True + then both the raw model response (a BaseMessage) and the parsed model + response will be returned. If an error occurs during output parsing it + will be caught and returned as well. The final output is always a dict + with keys "raw", "parsed", and "parsing_error". + strict: + + - True: + Model output is guaranteed to exactly match the schema. + The input schema will also be validated according to + https://platform.openai.com/docs/guides/structured-outputs/supported-schemas + - False: + Input schema will not be validated and model output will not be + validated. + - None: + ``strict`` argument will not be passed to the model. + + If ``method`` is "json_schema" or "function_calling" defaults to True. + If ``method`` is "json_mode" defaults to None. Can only be non-null + if ``method`` is "function_calling" or "json_schema". + + kwargs: Additional keyword args aren't supported. + + Returns: + A Runnable that takes same inputs as a :class:`langchain_core.language_models.chat.BaseChatModel`. + + | If ``include_raw`` is False and ``schema`` is a Pydantic class, Runnable outputs an instance of ``schema`` (i.e., a Pydantic object). Otherwise, if ``include_raw`` is False then Runnable outputs a dict. + + | If ``include_raw`` is True, then Runnable outputs a dict with keys: + + - "raw": BaseMessage + - "parsed": None if there was a parsing error, otherwise the type depends on the ``schema`` as described above. + - "parsing_error": Optional[BaseException] + + .. versionchanged:: 0.1.20 + + Added support for TypedDict class ``schema``. + + .. versionchanged:: 0.1.21 + + Support for ``strict`` argument added. + Support for ``method`` = "json_schema" added. + + .. versionchanged:: 0.3.0 + + - ``method`` default changed from "function_calling" to "json_schema". + - ``strict`` defaults to True instead of False when ``method`` is + "function_calling". + + .. dropdown:: Example: schema=Pydantic class, method="json_schema", include_raw=False, strict=True + + Note, OpenAI has a number of restrictions on what types of schemas can be + provided if ``strict`` = True. When using Pydantic, our model cannot + specify any Field metadata (like min/max constraints) and fields cannot + have default values. + + See all constraints here: https://platform.openai.com/docs/guides/structured-outputs/supported-schemas + + .. code-block:: python + + from typing import Optional + + from langchain_openai import AzureChatOpenAI + from pydantic import BaseModel, Field + + + class AnswerWithJustification(BaseModel): + '''An answer to the user question along with justification for the answer.''' + + answer: str + justification: Optional[str] = Field( + default=..., description="A justification for the answer." + ) + + + llm = AzureChatOpenAI(azure_deployment="...", model="gpt-4o", temperature=0) + structured_llm = llm.with_structured_output(AnswerWithJustification) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + + # -> AnswerWithJustification( + # answer='They weigh the same', + # justification='Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume or density of the objects may differ.' + # ) + + .. dropdown:: Example: schema=Pydantic class, method="json_schema", include_raw=True + + .. code-block:: python + + from langchain_openai import AzureChatOpenAI + from pydantic import BaseModel + + + class AnswerWithJustification(BaseModel): + '''An answer to the user question along with justification for the answer.''' + + answer: str + justification: str + + + llm = AzureChatOpenAI(azure_deployment="...", model="gpt-4o", temperature=0) + structured_llm = llm.with_structured_output( + AnswerWithJustification, include_raw=True + ) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + # -> { + # 'raw': AIMessage(content='', additional_kwargs={'tool_calls': [{'id': 'call_Ao02pnFYXD6GN1yzc0uXPsvF', 'function': {'arguments': '{"answer":"They weigh the same.","justification":"Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume or density of the objects may differ."}', 'name': 'AnswerWithJustification'}, 'type': 'function'}]}), + # 'parsed': AnswerWithJustification(answer='They weigh the same.', justification='Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume or density of the objects may differ.'), + # 'parsing_error': None + # } + + .. dropdown:: Example: schema=TypedDict class, method="json_schema", include_raw=False + + .. code-block:: python + + from typing_extensions import Annotated, TypedDict + + from langchain_openai import AzureChatOpenAI + + + class AnswerWithJustification(TypedDict): + '''An answer to the user question along with justification for the answer.''' + + answer: str + justification: Annotated[ + Optional[str], None, "A justification for the answer." + ] + + + llm = AzureChatOpenAI(azure_deployment="...", model="gpt-4o", temperature=0) + structured_llm = llm.with_structured_output(AnswerWithJustification) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + # -> { + # 'answer': 'They weigh the same', + # 'justification': 'Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume and density of the two substances differ.' + # } + + .. dropdown:: Example: schema=OpenAI function schema, method="json_schema", include_raw=False + + .. code-block:: python + + from langchain_openai import AzureChatOpenAI + + oai_schema = { + 'name': 'AnswerWithJustification', + 'description': 'An answer to the user question along with justification for the answer.', + 'parameters': { + 'type': 'object', + 'properties': { + 'answer': {'type': 'string'}, + 'justification': {'description': 'A justification for the answer.', 'type': 'string'} + }, + 'required': ['answer'] + } + } + + llm = AzureChatOpenAI( + azure_deployment="...", + model="gpt-4o", + temperature=0, + ) + structured_llm = llm.with_structured_output(oai_schema) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + # -> { + # 'answer': 'They weigh the same', + # 'justification': 'Both a pound of bricks and a pound of feathers weigh one pound. The weight is the same, but the volume and density of the two substances differ.' + # } + + .. dropdown:: Example: schema=Pydantic class, method="json_mode", include_raw=True + + .. code-block:: + + from langchain_openai import AzureChatOpenAI + from pydantic import BaseModel + + class AnswerWithJustification(BaseModel): + answer: str + justification: str + + llm = AzureChatOpenAI( + azure_deployment="...", + model="gpt-4o", + temperature=0, + ) + structured_llm = llm.with_structured_output( + AnswerWithJustification, + method="json_mode", + include_raw=True + ) + + structured_llm.invoke( + "Answer the following question. " + "Make sure to return a JSON blob with keys 'answer' and 'justification'.\\n\\n" + "What's heavier a pound of bricks or a pound of feathers?" + ) + # -> { + # 'raw': AIMessage(content='{\\n "answer": "They are both the same weight.",\\n "justification": "Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight." \\n}'), + # 'parsed': AnswerWithJustification(answer='They are both the same weight.', justification='Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight.'), + # 'parsing_error': None + # } + + .. dropdown:: Example: schema=None, method="json_mode", include_raw=True + + .. code-block:: + + structured_llm = llm.with_structured_output(method="json_mode", include_raw=True) + + structured_llm.invoke( + "Answer the following question. " + "Make sure to return a JSON blob with keys 'answer' and 'justification'.\\n\\n" + "What's heavier a pound of bricks or a pound of feathers?" + ) + # -> { + # 'raw': AIMessage(content='{\\n "answer": "They are both the same weight.",\\n "justification": "Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight." \\n}'), + # 'parsed': { + # 'answer': 'They are both the same weight.', + # 'justification': 'Both a pound of bricks and a pound of feathers weigh one pound. The difference lies in the volume and density of the materials, not the weight.' + # }, + # 'parsing_error': None + # } + """ # noqa: E501 + if method == "function_calling" and strict is None: + strict = True + return super().with_structured_output( + schema, method=method, include_raw=include_raw, strict=strict, **kwargs + ) diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py index 5cbb12791395f..5808830036c61 100644 --- a/libs/partners/openai/langchain_openai/chat_models/base.py +++ b/libs/partners/openai/langchain_openai/chat_models/base.py @@ -1922,10 +1922,10 @@ def with_structured_output( schema: The output schema. Can be passed in as: - - an OpenAI function/tool schema, - a JSON Schema, - a TypedDict class, - - or a Pydantic class. + - or a Pydantic class, + - an OpenAI function/tool schema. If ``schema`` is a Pydantic class then the model output will be a Pydantic instance of that class, and the model-generated fields will be @@ -2034,9 +2034,7 @@ class AnswerWithJustification(BaseModel): llm = ChatOpenAI(model="gpt-4o", temperature=0) - structured_llm = llm.with_structured_output( - AnswerWithJustification - ) + structured_llm = llm.with_structured_output(AnswerWithJustification) structured_llm.invoke( "What weighs more a pound of bricks or a pound of feathers" From 1e66f69bf074d7e5b755624148ae95a621e7b629 Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Wed, 8 Jan 2025 12:55:04 -0500 Subject: [PATCH 04/17] handle pydantic v1 models --- .../openai/langchain_openai/chat_models/base.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py index 5808830036c61..b1fd78eb1a58b 100644 --- a/libs/partners/openai/langchain_openai/chat_models/base.py +++ b/libs/partners/openai/langchain_openai/chat_models/base.py @@ -92,6 +92,7 @@ ) from langchain_core.utils.utils import _build_model_kwargs, from_env, secret_from_env from pydantic import BaseModel, ConfigDict, Field, SecretStr, model_validator +from pydantic.v1 import BaseModel as BaseModelV1 from typing_extensions import Self logger = logging.getLogger(__name__) @@ -1269,6 +1270,21 @@ def with_structured_output( "Argument `strict` is not supported with `method`='json_mode'" ) is_pydantic_schema = _is_pydantic_class(schema) + + # Check for Pydantic BaseModel V1 + if ( + method == "json_schema" + and is_pydantic_schema + and issubclass(schema, BaseModelV1) # type: ignore[arg-type] + ): + warnings.warn( + "Received a Pydantic BaseModel V1 schema. This is not supported by " + 'method="json_schema". Please use method="function_calling" ' + "or specify schema via JSON Schema or Pydantic V2 BaseModel. " + 'Overriding to method="function_calling".' + ) + method = "function_calling" + if method == "function_calling": if schema is None: raise ValueError( From 05e3a930fbfcc401e2e4a3d8e297832857154523 Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Wed, 8 Jan 2025 13:42:57 -0500 Subject: [PATCH 05/17] add to standard tests and override for Azure legacy tests --- .../chat_models/test_azure_standard.py | 6 ++- .../integration_tests/chat_models.py | 52 ++++++++++++++----- .../langchain_tests/unit_tests/chat_models.py | 19 +++++++ 3 files changed, 62 insertions(+), 15 deletions(-) diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py b/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py index acf44a5ac0b3a..8477087a01f7e 100644 --- a/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py +++ b/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py @@ -1,7 +1,7 @@ """Standard LangChain interface tests""" import os -from typing import Type +from typing import Optional, Type import pytest from langchain_core.language_models import BaseChatModel @@ -55,6 +55,10 @@ def chat_model_params(self) -> dict: "azure_endpoint": OPENAI_API_BASE, } + @property + def structured_output_method(self) -> Optional[str]: + return "function_calling" + @pytest.mark.xfail(reason="Not yet supported.") def test_usage_metadata_streaming(self, model: BaseChatModel) -> None: super().test_usage_metadata_streaming(model) diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py index f569135004497..1d05e2cf5da24 100644 --- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py @@ -1,6 +1,6 @@ import base64 import json -from typing import List, Optional, cast +from typing import Any, List, Optional, cast import httpx import pytest @@ -191,6 +191,20 @@ def tool_choice_value(self) -> Optional[str]: def has_structured_output(self) -> bool: return True + .. dropdown:: structured_output_method + + Optional string property that can be used to override the default ``method`` + parameter for ``with_structured_output``. Useful for testing different + models. + + Example: + + .. code-block:: python + + @property + def structured_output_method(self) -> Optional[str]: + return "function_calling" + .. dropdown:: supports_json_mode Boolean property indicating whether the chat model supports JSON mode in @@ -1126,12 +1140,13 @@ def has_tool_calling(self) -> bool: if not self.has_tool_calling: pytest.skip("Test requires tool calling.") + kwargs: dict[str, Any] = {} + if self.structured_output_method: + kwargs["method"] = self.structured_output_method + Joke = _get_joke_class() # Pydantic class - # Type ignoring since the interface only officially supports pydantic 1 - # or pydantic.v1.BaseModel but not pydantic.BaseModel from pydantic 2. - # We'll need to do a pass updating the type signatures. - chat = model.with_structured_output(Joke) # type: ignore[arg-type] + chat = model.with_structured_output(Joke, **kwargs) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, Joke) @@ -1139,7 +1154,7 @@ def has_tool_calling(self) -> bool: assert isinstance(chunk, Joke) # Schema - chat = model.with_structured_output(Joke.model_json_schema()) + chat = model.with_structured_output(Joke.model_json_schema(), **kwargs) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, dict) assert set(result.keys()) == {"setup", "punchline"} @@ -1179,13 +1194,14 @@ def has_tool_calling(self) -> bool: if not self.has_tool_calling: pytest.skip("Test requires tool calling.") + kwargs: dict[str, Any] = {} + if self.structured_output_method: + kwargs["method"] = self.structured_output_method + Joke = _get_joke_class() # Pydantic class - # Type ignoring since the interface only officially supports pydantic 1 - # or pydantic.v1.BaseModel but not pydantic.BaseModel from pydantic 2. - # We'll need to do a pass updating the type signatures. - chat = model.with_structured_output(Joke) # type: ignore[arg-type] + chat = model.with_structured_output(Joke, **kwargs) result = await chat.ainvoke("Tell me a joke about cats.") assert isinstance(result, Joke) @@ -1193,7 +1209,7 @@ def has_tool_calling(self) -> bool: assert isinstance(chunk, Joke) # Schema - chat = model.with_structured_output(Joke.model_json_schema()) + chat = model.with_structured_output(Joke.model_json_schema(), **kwargs) result = await chat.ainvoke("Tell me a joke about cats.") assert isinstance(result, dict) assert set(result.keys()) == {"setup", "punchline"} @@ -1237,6 +1253,10 @@ def has_tool_calling(self) -> bool: if not self.has_tool_calling: pytest.skip("Test requires tool calling.") + kwargs: dict[str, Any] = {} + if self.structured_output_method: + kwargs["method"] = self.structured_output_method + class Joke(BaseModelV1): # Uses langchain_core.pydantic_v1.BaseModel """Joke to tell user.""" @@ -1244,7 +1264,7 @@ class Joke(BaseModelV1): # Uses langchain_core.pydantic_v1.BaseModel punchline: str = FieldV1(description="answer to resolve the joke") # Pydantic class - chat = model.with_structured_output(Joke) + chat = model.with_structured_output(Joke, **kwargs) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, Joke) @@ -1252,7 +1272,7 @@ class Joke(BaseModelV1): # Uses langchain_core.pydantic_v1.BaseModel assert isinstance(chunk, Joke) # Schema - chat = model.with_structured_output(Joke.schema()) + chat = model.with_structured_output(Joke.schema(), **kwargs) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, dict) assert set(result.keys()) == {"setup", "punchline"} @@ -1293,6 +1313,10 @@ def has_tool_calling(self) -> bool: if not self.has_tool_calling: pytest.skip("Test requires tool calling.") + kwargs = {} + if self.structured_output_method: + kwargs["method"] = self.structured_output_method + class Joke(BaseModel): """Joke to tell user.""" @@ -1301,7 +1325,7 @@ class Joke(BaseModel): default=None, description="answer to resolve the joke" ) - chat = model.with_structured_output(Joke) # type: ignore[arg-type] + chat = model.with_structured_output(Joke, **kwargs) # type: ignore[arg-type] setup_result = chat.invoke( "Give me the setup to a joke about cats, no punchline." ) diff --git a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py index 766367f7359c2..ebaea154004a1 100644 --- a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py @@ -132,6 +132,11 @@ def has_structured_output(self) -> bool: is not BaseChatModel.with_structured_output ) + @property + def structured_output_method(self) -> Optional[str]: + """If specified, override default for with_structured_output.""" + return None + @property def supports_json_mode(self) -> bool: """(bool) whether the chat model supports JSON mode.""" @@ -299,6 +304,20 @@ def tool_choice_value(self) -> Optional[str]: def has_structured_output(self) -> bool: return True + .. dropdown:: structured_output_method + + Optional string property that can be used to override the default ``method`` + parameter for ``with_structured_output``. Useful for testing different + models. + + Example: + + .. code-block:: python + + @property + def structured_output_method(self) -> Optional[str]: + return "function_calling" + .. dropdown:: supports_json_mode Boolean property indicating whether the chat model supports JSON mode in From 0dddca18ba46f034e6f9a3ccfabf78b58552432e Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Wed, 8 Jan 2025 13:48:44 -0500 Subject: [PATCH 06/17] update property --- .../chat_models/test_azure_standard.py | 4 +- .../integration_tests/chat_models.py | 49 +++++++------------ .../langchain_tests/unit_tests/chat_models.py | 17 +++---- 3 files changed, 29 insertions(+), 41 deletions(-) diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py b/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py index 8477087a01f7e..5f96e586f88e8 100644 --- a/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py +++ b/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py @@ -56,8 +56,8 @@ def chat_model_params(self) -> dict: } @property - def structured_output_method(self) -> Optional[str]: - return "function_calling" + def structured_output_kwargs(self) -> dict: + return {"method": "function_calling"} @pytest.mark.xfail(reason="Not yet supported.") def test_usage_metadata_streaming(self, model: BaseChatModel) -> None: diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py index 1d05e2cf5da24..7386d93f97520 100644 --- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py @@ -1,6 +1,6 @@ import base64 import json -from typing import Any, List, Optional, cast +from typing import List, Optional, cast import httpx import pytest @@ -191,19 +191,18 @@ def tool_choice_value(self) -> Optional[str]: def has_structured_output(self) -> bool: return True - .. dropdown:: structured_output_method + .. dropdown:: structured_output_kwargs - Optional string property that can be used to override the default ``method`` - parameter for ``with_structured_output``. Useful for testing different - models. + Dict property that can be used to specify additional kwargs for + ``with_structured_output``. Useful for testing different models. Example: .. code-block:: python @property - def structured_output_method(self) -> Optional[str]: - return "function_calling" + def structured_output_kwargs(self) -> dict: + return {"method": "function_calling"} .. dropdown:: supports_json_mode @@ -1140,13 +1139,9 @@ def has_tool_calling(self) -> bool: if not self.has_tool_calling: pytest.skip("Test requires tool calling.") - kwargs: dict[str, Any] = {} - if self.structured_output_method: - kwargs["method"] = self.structured_output_method - Joke = _get_joke_class() # Pydantic class - chat = model.with_structured_output(Joke, **kwargs) + chat = model.with_structured_output(Joke, **self.structured_output_kwargs) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, Joke) @@ -1154,7 +1149,9 @@ def has_tool_calling(self) -> bool: assert isinstance(chunk, Joke) # Schema - chat = model.with_structured_output(Joke.model_json_schema(), **kwargs) + chat = model.with_structured_output( + Joke.model_json_schema(), **self.structured_output_kwargs + ) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, dict) assert set(result.keys()) == {"setup", "punchline"} @@ -1194,14 +1191,10 @@ def has_tool_calling(self) -> bool: if not self.has_tool_calling: pytest.skip("Test requires tool calling.") - kwargs: dict[str, Any] = {} - if self.structured_output_method: - kwargs["method"] = self.structured_output_method - Joke = _get_joke_class() # Pydantic class - chat = model.with_structured_output(Joke, **kwargs) + chat = model.with_structured_output(Joke, **self.structured_output_kwargs) result = await chat.ainvoke("Tell me a joke about cats.") assert isinstance(result, Joke) @@ -1209,7 +1202,9 @@ def has_tool_calling(self) -> bool: assert isinstance(chunk, Joke) # Schema - chat = model.with_structured_output(Joke.model_json_schema(), **kwargs) + chat = model.with_structured_output( + Joke.model_json_schema(), **self.structured_output_kwargs + ) result = await chat.ainvoke("Tell me a joke about cats.") assert isinstance(result, dict) assert set(result.keys()) == {"setup", "punchline"} @@ -1253,10 +1248,6 @@ def has_tool_calling(self) -> bool: if not self.has_tool_calling: pytest.skip("Test requires tool calling.") - kwargs: dict[str, Any] = {} - if self.structured_output_method: - kwargs["method"] = self.structured_output_method - class Joke(BaseModelV1): # Uses langchain_core.pydantic_v1.BaseModel """Joke to tell user.""" @@ -1264,7 +1255,7 @@ class Joke(BaseModelV1): # Uses langchain_core.pydantic_v1.BaseModel punchline: str = FieldV1(description="answer to resolve the joke") # Pydantic class - chat = model.with_structured_output(Joke, **kwargs) + chat = model.with_structured_output(Joke, **self.structured_output_kwargs) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, Joke) @@ -1272,7 +1263,9 @@ class Joke(BaseModelV1): # Uses langchain_core.pydantic_v1.BaseModel assert isinstance(chunk, Joke) # Schema - chat = model.with_structured_output(Joke.schema(), **kwargs) + chat = model.with_structured_output( + Joke.schema(), **self.structured_output_kwargs + ) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, dict) assert set(result.keys()) == {"setup", "punchline"} @@ -1313,10 +1306,6 @@ def has_tool_calling(self) -> bool: if not self.has_tool_calling: pytest.skip("Test requires tool calling.") - kwargs = {} - if self.structured_output_method: - kwargs["method"] = self.structured_output_method - class Joke(BaseModel): """Joke to tell user.""" @@ -1325,7 +1314,7 @@ class Joke(BaseModel): default=None, description="answer to resolve the joke" ) - chat = model.with_structured_output(Joke, **kwargs) # type: ignore[arg-type] + chat = model.with_structured_output(Joke, **self.structured_output_kwargs) # type: ignore[arg-type] setup_result = chat.invoke( "Give me the setup to a joke about cats, no punchline." ) diff --git a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py index ebaea154004a1..84a51385b6d05 100644 --- a/libs/standard-tests/langchain_tests/unit_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/unit_tests/chat_models.py @@ -133,9 +133,9 @@ def has_structured_output(self) -> bool: ) @property - def structured_output_method(self) -> Optional[str]: - """If specified, override default for with_structured_output.""" - return None + def structured_output_kwargs(self) -> dict: + """If specified, additional kwargs for with_structured_output.""" + return {} @property def supports_json_mode(self) -> bool: @@ -304,19 +304,18 @@ def tool_choice_value(self) -> Optional[str]: def has_structured_output(self) -> bool: return True - .. dropdown:: structured_output_method + .. dropdown:: structured_output_kwargs - Optional string property that can be used to override the default ``method`` - parameter for ``with_structured_output``. Useful for testing different - models. + Dict property that can be used to specify additional kwargs for + ``with_structured_output``. Useful for testing different models. Example: .. code-block:: python @property - def structured_output_method(self) -> Optional[str]: - return "function_calling" + def structured_output_kwargs(self) -> dict: + return {"method": "function_calling"} .. dropdown:: supports_json_mode From eace01895328fc89dc7b459513398d7fecc9d402 Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Wed, 8 Jan 2025 13:50:04 -0500 Subject: [PATCH 07/17] lint --- .../tests/integration_tests/chat_models/test_azure_standard.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py b/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py index 5f96e586f88e8..f069750934729 100644 --- a/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py +++ b/libs/partners/openai/tests/integration_tests/chat_models/test_azure_standard.py @@ -1,7 +1,7 @@ """Standard LangChain interface tests""" import os -from typing import Optional, Type +from typing import Type import pytest from langchain_core.language_models import BaseChatModel From cc7bb583a7f567c7212b683486f08e787d842725 Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Wed, 8 Jan 2025 14:23:18 -0500 Subject: [PATCH 08/17] fix test --- .../openai/tests/integration_tests/chat_models/test_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py index 93c08ce214178..876778b007c74 100644 --- a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py +++ b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py @@ -637,7 +637,7 @@ class MyModel(BaseModel): name: str age: int - llm = ChatOpenAI().with_structured_output(MyModel) + llm = ChatOpenAI(model="gpt-4o-mini").with_structured_output(MyModel) result = llm.invoke("I'm a 27 year old named Erick") assert isinstance(result, MyModel) assert result.name == "Erick" From 1fb3ac3f36dec5faa8fcbc2fe6066dca10e6c69b Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Tue, 7 Jan 2025 14:26:17 -0500 Subject: [PATCH 09/17] add test --- .../integration_tests/chat_models.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py index 7386d93f97520..3d38622617d1d 100644 --- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py @@ -21,6 +21,7 @@ from pydantic import BaseModel, Field from pydantic.v1 import BaseModel as BaseModelV1 from pydantic.v1 import Field as FieldV1 +from typing_extensions import Annotated, TypedDict from langchain_tests.unit_tests.chat_models import ( ChatModelTests, @@ -1306,6 +1307,7 @@ def has_tool_calling(self) -> bool: if not self.has_tool_calling: pytest.skip("Test requires tool calling.") + # Pydantic class Joke(BaseModel): """Joke to tell user.""" @@ -1323,6 +1325,22 @@ class Joke(BaseModel): joke_result = chat.invoke("Give me a joke about cats, include the punchline.") assert isinstance(joke_result, Joke) + # Schema + chat = model.with_structured_output(Joke.model_json_schema()) + result = chat.invoke("Tell me a joke about cats.") + assert isinstance(result, dict) + + # TypedDict + class JokeDict(TypedDict): + """Joke to tell user.""" + + setup: Annotated[str, ..., "question to set up a joke"] + punchline: Annotated[Optional[str], None, "answer to resolve the joke"] + + chat = model.with_structured_output(JokeDict) + result = chat.invoke("Tell me a joke about cats.") + assert isinstance(result, dict) + def test_json_mode(self, model: BaseChatModel) -> None: """Test structured output via `JSON mode. `_ From e6a9a65dd3b30a867474aed00e98e9b96180299c Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Wed, 8 Jan 2025 14:29:31 -0500 Subject: [PATCH 10/17] update --- .../langchain_tests/integration_tests/chat_models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py index 3d38622617d1d..b726c45d28378 100644 --- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py @@ -1326,7 +1326,9 @@ class Joke(BaseModel): assert isinstance(joke_result, Joke) # Schema - chat = model.with_structured_output(Joke.model_json_schema()) + chat = model.with_structured_output( + Joke.model_json_schema(), **self.structured_output_kwargs + ) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, dict) @@ -1337,7 +1339,7 @@ class JokeDict(TypedDict): setup: Annotated[str, ..., "question to set up a joke"] punchline: Annotated[Optional[str], None, "answer to resolve the joke"] - chat = model.with_structured_output(JokeDict) + chat = model.with_structured_output(JokeDict, **self.structured_output_kwargs) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, dict) From 075021bce19451cc7448feaa7938fcb2906df712 Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Wed, 8 Jan 2025 14:29:48 -0500 Subject: [PATCH 11/17] default to strict=False --- libs/partners/openai/langchain_openai/chat_models/azure.py | 4 ++-- libs/partners/openai/langchain_openai/chat_models/base.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/partners/openai/langchain_openai/chat_models/azure.py b/libs/partners/openai/langchain_openai/chat_models/azure.py index 4fdab27db89f5..1d5ae0141565a 100644 --- a/libs/partners/openai/langchain_openai/chat_models/azure.py +++ b/libs/partners/openai/langchain_openai/chat_models/azure.py @@ -1023,8 +1023,8 @@ class AnswerWithJustification(BaseModel): # 'parsing_error': None # } """ # noqa: E501 - if method == "function_calling" and strict is None: - strict = True + if method in ("json_schema", "function_calling") and strict is None: + strict = False return super().with_structured_output( schema, method=method, include_raw=include_raw, strict=strict, **kwargs ) diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py index b1fd78eb1a58b..32037ac4d7b45 100644 --- a/libs/partners/openai/langchain_openai/chat_models/base.py +++ b/libs/partners/openai/langchain_openai/chat_models/base.py @@ -2200,8 +2200,8 @@ class AnswerWithJustification(BaseModel): # 'parsing_error': None # } """ # noqa: E501 - if method == "function_calling" and strict is None: - strict = True + if method in ("json_schema", "function_calling") and strict is None: + strict = False return super().with_structured_output( schema, method=method, include_raw=include_raw, strict=strict, **kwargs ) From e0ca601c134d18a4340973f94fdcbe95ff984829 Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Wed, 8 Jan 2025 14:31:38 -0500 Subject: [PATCH 12/17] remove unnecessary type ignore --- .../langchain_tests/integration_tests/chat_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py index b726c45d28378..69116b6cfb7a9 100644 --- a/libs/standard-tests/langchain_tests/integration_tests/chat_models.py +++ b/libs/standard-tests/langchain_tests/integration_tests/chat_models.py @@ -1316,7 +1316,7 @@ class Joke(BaseModel): default=None, description="answer to resolve the joke" ) - chat = model.with_structured_output(Joke, **self.structured_output_kwargs) # type: ignore[arg-type] + chat = model.with_structured_output(Joke, **self.structured_output_kwargs) setup_result = chat.invoke( "Give me the setup to a joke about cats, no punchline." ) From 9145a5030224d7d3d83ecd451cff0ba37d3183ea Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Wed, 8 Jan 2025 14:58:57 -0500 Subject: [PATCH 13/17] fix test --- .../chat_models/test_base.py | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py index 876778b007c74..1b1964559fb86 100644 --- a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py +++ b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py @@ -630,14 +630,15 @@ def test_bind_tools_tool_choice() -> None: assert not msg.tool_calls -def test_openai_structured_output() -> None: +@pytest.mark.parametrize("model", ["gpt-4o-mini", "o1"]) +def test_openai_structured_output(model: str) -> None: class MyModel(BaseModel): """A Person""" name: str age: int - llm = ChatOpenAI(model="gpt-4o-mini").with_structured_output(MyModel) + llm = ChatOpenAI(model=model).with_structured_output(MyModel) result = llm.invoke("I'm a 27 year old named Erick") assert isinstance(result, MyModel) assert result.name == "Erick" @@ -820,20 +821,18 @@ class magic_function(BaseModel): @pytest.mark.parametrize( - ("model", "method", "strict"), - [("gpt-4o", "function_calling", True), ("gpt-4o-2024-08-06", "json_schema", None)], + ("model", "method"), + [("gpt-4o", "function_calling"), ("gpt-4o-2024-08-06", "json_schema")], ) def test_structured_output_strict( - model: str, - method: Literal["function_calling", "json_schema"], - strict: Optional[bool], + model: str, method: Literal["function_calling", "json_schema"] ) -> None: """Test to verify structured output with strict=True.""" from pydantic import BaseModel as BaseModelProper from pydantic import Field as FieldProper - llm = ChatOpenAI(model=model, temperature=0) + llm = ChatOpenAI(model=model) class Joke(BaseModelProper): """Joke to tell user.""" @@ -842,10 +841,7 @@ class Joke(BaseModelProper): punchline: str = FieldProper(description="answer to resolve the joke") # Pydantic class - # Type ignoring since the interface only officially supports pydantic 1 - # or pydantic.v1.BaseModel but not pydantic.BaseModel from pydantic 2. - # We'll need to do a pass updating the type signatures. - chat = llm.with_structured_output(Joke, method=method, strict=strict) + chat = llm.with_structured_output(Joke, method=method, strict=True) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, Joke) @@ -854,7 +850,7 @@ class Joke(BaseModelProper): # Schema chat = llm.with_structured_output( - Joke.model_json_schema(), method=method, strict=strict + Joke.model_json_schema(), method=method, strict=True ) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, dict) @@ -875,14 +871,14 @@ class InvalidJoke(BaseModelProper): default="foo", description="answer to resolve the joke" ) - chat = llm.with_structured_output(InvalidJoke, method=method, strict=strict) + chat = llm.with_structured_output(InvalidJoke, method=method, strict=True) with pytest.raises(openai.BadRequestError): chat.invoke("Tell me a joke about cats.") with pytest.raises(openai.BadRequestError): next(chat.stream("Tell me a joke about cats.")) chat = llm.with_structured_output( - InvalidJoke.model_json_schema(), method=method, strict=strict + InvalidJoke.model_json_schema(), method=method, strict=True ) with pytest.raises(openai.BadRequestError): chat.invoke("Tell me a joke about cats.") @@ -890,11 +886,9 @@ class InvalidJoke(BaseModelProper): next(chat.stream("Tell me a joke about cats.")) -@pytest.mark.parametrize( - ("model", "method", "strict"), [("gpt-4o-2024-08-06", "json_schema", None)] -) +@pytest.mark.parametrize(("model", "method"), [("gpt-4o-2024-08-06", "json_schema")]) def test_nested_structured_output_strict( - model: str, method: Literal["json_schema"], strict: Optional[bool] + model: str, method: Literal["json_schema"] ) -> None: """Test to verify structured output with strict=True for nested object.""" @@ -914,7 +908,7 @@ class JokeWithEvaluation(TypedDict): self_evaluation: SelfEvaluation # Schema - chat = llm.with_structured_output(JokeWithEvaluation, method=method, strict=strict) + chat = llm.with_structured_output(JokeWithEvaluation, method=method, strict=True) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, dict) assert set(result.keys()) == {"setup", "punchline", "self_evaluation"} From b6d52892d37eff37c7918b67b69b1162ea6edf8c Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Thu, 9 Jan 2025 11:33:25 -0500 Subject: [PATCH 14/17] update default handling of strict --- .../langchain_openai/chat_models/azure.py | 22 ++++++++++++----- .../langchain_openai/chat_models/base.py | 22 ++++++++++++----- .../chat_models/test_base.py | 24 +++++++++++-------- 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/libs/partners/openai/langchain_openai/chat_models/azure.py b/libs/partners/openai/langchain_openai/chat_models/azure.py index 3f7106f84d853..5960e7aba7a14 100644 --- a/libs/partners/openai/langchain_openai/chat_models/azure.py +++ b/libs/partners/openai/langchain_openai/chat_models/azure.py @@ -746,7 +746,9 @@ def with_structured_output( self, schema: Optional[_DictOrPydanticClass] = None, *, - method: Literal["function_calling", "json_mode", "json_schema"] = "json_schema", + method: Optional[ + Literal["function_calling", "json_mode", "json_schema"] + ] = None, include_raw: bool = False, strict: Optional[bool] = None, **kwargs: Any, @@ -791,6 +793,9 @@ def with_structured_output( - https://platform.openai.com/docs/guides/structured-outputs/structured-outputs-vs-json-mode - https://platform.openai.com/docs/guides/structured-outputs/function-calling-vs-response-format + Defaults to ``"json_schema"``. If both ``method`` and ``strict`` are not + supplied, we default to ``"json_schema"`` with ``strict=False`` (see below). + include_raw: If False then only the parsed structured output is returned. If an error occurs during model output parsing it will be raised. If True @@ -810,9 +815,9 @@ def with_structured_output( - None: ``strict`` argument will not be passed to the model. - If ``method`` is "json_schema" or "function_calling" defaults to True. - If ``method`` is "json_mode" defaults to None. Can only be non-null - if ``method`` is "function_calling" or "json_schema". + If ``method`` is not supplied, defaults to False. + If ``method`` is "json_schema", defaults to True. + If ``method`` is "function_calling" or "json_mode", defaults to None. kwargs: Additional keyword args aren't supported. @@ -1025,8 +1030,13 @@ class AnswerWithJustification(BaseModel): # 'parsing_error': None # } """ # noqa: E501 - if method in ("json_schema", "function_calling") and strict is None: - strict = False + if method is None: + method = "json_schema" + strict = False if strict is None else strict + + if strict is None: + strict = True if method == "json_schema" else None + return super().with_structured_output( schema, method=method, include_raw=include_raw, strict=strict, **kwargs ) diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py index 2a9ec0350ed9d..a8890e9a5f3ca 100644 --- a/libs/partners/openai/langchain_openai/chat_models/base.py +++ b/libs/partners/openai/langchain_openai/chat_models/base.py @@ -1929,7 +1929,9 @@ def with_structured_output( self, schema: Optional[_DictOrPydanticClass] = None, *, - method: Literal["function_calling", "json_mode", "json_schema"] = "json_schema", + method: Optional[ + Literal["function_calling", "json_mode", "json_schema"] + ] = None, include_raw: bool = False, strict: Optional[bool] = None, **kwargs: Any, @@ -1974,6 +1976,9 @@ def with_structured_output( - https://platform.openai.com/docs/guides/structured-outputs/structured-outputs-vs-json-mode - https://platform.openai.com/docs/guides/structured-outputs/function-calling-vs-response-format + Defaults to ``"json_schema"``. If both ``method`` and ``strict`` are not + supplied, we default to ``"json_schema"`` with ``strict=False`` (see below). + include_raw: If False then only the parsed structured output is returned. If an error occurs during model output parsing it will be raised. If True @@ -1993,9 +1998,9 @@ def with_structured_output( - None: ``strict`` argument will not be passed to the model. - If ``method`` is "json_schema" or "function_calling" defaults to True. - If ``method`` is "json_mode" defaults to None. Can only be non-null - if ``method`` is "function_calling" or "json_schema". + If ``method`` is not supplied, defaults to False. + If ``method`` is "json_schema", defaults to True. + If ``method`` is "function_calling" or "json_mode", defaults to None. kwargs: Additional keyword args aren't supported. @@ -2202,8 +2207,13 @@ class AnswerWithJustification(BaseModel): # 'parsing_error': None # } """ # noqa: E501 - if method in ("json_schema", "function_calling") and strict is None: - strict = False + if method is None: + method = "json_schema" + strict = False if strict is None else strict + + if strict is None: + strict = True if method == "json_schema" else None + return super().with_structured_output( schema, method=method, include_raw=include_raw, strict=strict, **kwargs ) diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py index 1b1964559fb86..95978933f5ce2 100644 --- a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py +++ b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py @@ -821,11 +821,13 @@ class magic_function(BaseModel): @pytest.mark.parametrize( - ("model", "method"), - [("gpt-4o", "function_calling"), ("gpt-4o-2024-08-06", "json_schema")], + ("model", "method", "strict"), + [("gpt-4o", "function_calling", True), ("gpt-4o-2024-08-06", "json_schema", None)], ) def test_structured_output_strict( - model: str, method: Literal["function_calling", "json_schema"] + model: str, + method: Literal["function_calling", "json_schema"], + strict: Optional[bool], ) -> None: """Test to verify structured output with strict=True.""" @@ -841,7 +843,7 @@ class Joke(BaseModelProper): punchline: str = FieldProper(description="answer to resolve the joke") # Pydantic class - chat = llm.with_structured_output(Joke, method=method, strict=True) + chat = llm.with_structured_output(Joke, method=method, strict=strict) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, Joke) @@ -850,7 +852,7 @@ class Joke(BaseModelProper): # Schema chat = llm.with_structured_output( - Joke.model_json_schema(), method=method, strict=True + Joke.model_json_schema(), method=method, strict=strict ) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, dict) @@ -871,14 +873,14 @@ class InvalidJoke(BaseModelProper): default="foo", description="answer to resolve the joke" ) - chat = llm.with_structured_output(InvalidJoke, method=method, strict=True) + chat = llm.with_structured_output(InvalidJoke, method=method, strict=strict) with pytest.raises(openai.BadRequestError): chat.invoke("Tell me a joke about cats.") with pytest.raises(openai.BadRequestError): next(chat.stream("Tell me a joke about cats.")) chat = llm.with_structured_output( - InvalidJoke.model_json_schema(), method=method, strict=True + InvalidJoke.model_json_schema(), method=method, strict=strict ) with pytest.raises(openai.BadRequestError): chat.invoke("Tell me a joke about cats.") @@ -886,9 +888,11 @@ class InvalidJoke(BaseModelProper): next(chat.stream("Tell me a joke about cats.")) -@pytest.mark.parametrize(("model", "method"), [("gpt-4o-2024-08-06", "json_schema")]) +@pytest.mark.parametrize( + ("model", "method", "strict"), [("gpt-4o-2024-08-06", "json_schema", None)] +) def test_nested_structured_output_strict( - model: str, method: Literal["json_schema"] + model: str, method: Literal["json_schema"], strict: Optional[bool] ) -> None: """Test to verify structured output with strict=True for nested object.""" @@ -908,7 +912,7 @@ class JokeWithEvaluation(TypedDict): self_evaluation: SelfEvaluation # Schema - chat = llm.with_structured_output(JokeWithEvaluation, method=method, strict=True) + chat = llm.with_structured_output(JokeWithEvaluation, method=method, strict=strict) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, dict) assert set(result.keys()) == {"setup", "punchline", "self_evaluation"} From cf891a49a0ec5f7595056c7bfd1d1716f8799947 Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Thu, 9 Jan 2025 13:15:10 -0500 Subject: [PATCH 15/17] raise informative error message for legacy models with default method --- .../langchain_openai/chat_models/base.py | 29 ++++++++++++++++--- .../chat_models/test_base.py | 20 ++++++++----- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py index a8890e9a5f3ca..8af4bf90ddaea 100644 --- a/libs/partners/openai/langchain_openai/chat_models/base.py +++ b/libs/partners/openai/langchain_openai/chat_models/base.py @@ -388,6 +388,21 @@ def _update_token_usage( return new_usage +def _handle_openai_bad_request(e: openai.BadRequestError) -> None: + if ( + "'response_format' of type 'json_schema' is not supported with " "this model" + ) in e.message: + raise ValueError( + "This model does not support OpenAI's structured output " + "feature, which is the default method for " + "`with_structured_output` as of langchain-openai==0.3. To use " + "`with_structured_output` with this model, specify " + '`method="function_calling"`.' + ) + else: + raise + + class _FunctionCall(TypedDict): name: str @@ -711,7 +726,10 @@ def _generate( "specified." ) payload.pop("stream") - response = self.root_client.beta.chat.completions.parse(**payload) + try: + response = self.root_client.beta.chat.completions.parse(**payload) + except openai.BadRequestError as e: + _handle_openai_bad_request(e) elif self.include_response_headers: raw_response = self.client.with_raw_response.create(**payload) response = raw_response.parse() @@ -845,9 +863,12 @@ async def _agenerate( "specified." ) payload.pop("stream") - response = await self.root_async_client.beta.chat.completions.parse( - **payload - ) + try: + response = await self.root_async_client.beta.chat.completions.parse( + **payload + ) + except openai.BadRequestError as e: + _handle_openai_bad_request(e) elif self.include_response_headers: raw_response = await self.async_client.with_raw_response.create(**payload) response = raw_response.parse() diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py index 95978933f5ce2..afa330144bc64 100644 --- a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py +++ b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py @@ -630,19 +630,25 @@ def test_bind_tools_tool_choice() -> None: assert not msg.tool_calls -@pytest.mark.parametrize("model", ["gpt-4o-mini", "o1"]) -def test_openai_structured_output(model: str) -> None: +def test_openai_structured_output() -> None: class MyModel(BaseModel): """A Person""" name: str age: int - llm = ChatOpenAI(model=model).with_structured_output(MyModel) - result = llm.invoke("I'm a 27 year old named Erick") - assert isinstance(result, MyModel) - assert result.name == "Erick" - assert result.age == 27 + for model in ["gpt-4o-mini", "o1"]: + llm = ChatOpenAI(model=model).with_structured_output(MyModel) + result = llm.invoke("I'm a 27 year old named Erick") + assert isinstance(result, MyModel) + assert result.name == "Erick" + assert result.age == 27 + + # Test legacy models raise error + llm = ChatOpenAI(model="gpt-4").with_structured_output(MyModel) + with pytest.raises(ValueError) as exception_info: + _ = llm.invoke("I'm a 27 year old named Erick") + assert "with_structured_output" in str(exception_info.value) def test_openai_proxy() -> None: From 315407720a568c8233b8bec57e1cb97404ae1c4c Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Thu, 9 Jan 2025 13:17:27 -0500 Subject: [PATCH 16/17] format --- libs/partners/openai/langchain_openai/chat_models/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py index 8af4bf90ddaea..92bccb84c3a25 100644 --- a/libs/partners/openai/langchain_openai/chat_models/base.py +++ b/libs/partners/openai/langchain_openai/chat_models/base.py @@ -390,7 +390,7 @@ def _update_token_usage( def _handle_openai_bad_request(e: openai.BadRequestError) -> None: if ( - "'response_format' of type 'json_schema' is not supported with " "this model" + "'response_format' of type 'json_schema' is not supported with this model" ) in e.message: raise ValueError( "This model does not support OpenAI's structured output " From e92eedcf169dd3de2fc5260ce477b8f490de6a4e Mon Sep 17 00:00:00 2001 From: Chester Curme Date: Thu, 9 Jan 2025 14:08:21 -0500 Subject: [PATCH 17/17] revert update to default handling of strict --- .../langchain_openai/chat_models/azure.py | 24 ++++--------- .../langchain_openai/chat_models/base.py | 36 +++++++------------ .../chat_models/test_base.py | 24 ++++++------- 3 files changed, 30 insertions(+), 54 deletions(-) diff --git a/libs/partners/openai/langchain_openai/chat_models/azure.py b/libs/partners/openai/langchain_openai/chat_models/azure.py index 5960e7aba7a14..57e053ecd1253 100644 --- a/libs/partners/openai/langchain_openai/chat_models/azure.py +++ b/libs/partners/openai/langchain_openai/chat_models/azure.py @@ -746,9 +746,7 @@ def with_structured_output( self, schema: Optional[_DictOrPydanticClass] = None, *, - method: Optional[ - Literal["function_calling", "json_mode", "json_schema"] - ] = None, + method: Literal["function_calling", "json_mode", "json_schema"] = "json_schema", include_raw: bool = False, strict: Optional[bool] = None, **kwargs: Any, @@ -776,7 +774,7 @@ def with_structured_output( - "json_schema": Uses OpenAI's Structured Output API: https://platform.openai.com/docs/guides/structured-outputs - Supported for "gpt-4o-mini", "gpt-4o-2024-08-06", and later + Supported for "gpt-4o-mini", "gpt-4o-2024-08-06", "o1", and later models. - "function_calling": Uses OpenAI's tool-calling (formerly called function calling) @@ -793,9 +791,6 @@ def with_structured_output( - https://platform.openai.com/docs/guides/structured-outputs/structured-outputs-vs-json-mode - https://platform.openai.com/docs/guides/structured-outputs/function-calling-vs-response-format - Defaults to ``"json_schema"``. If both ``method`` and ``strict`` are not - supplied, we default to ``"json_schema"`` with ``strict=False`` (see below). - include_raw: If False then only the parsed structured output is returned. If an error occurs during model output parsing it will be raised. If True @@ -815,9 +810,9 @@ def with_structured_output( - None: ``strict`` argument will not be passed to the model. - If ``method`` is not supplied, defaults to False. - If ``method`` is "json_schema", defaults to True. - If ``method`` is "function_calling" or "json_mode", defaults to None. + Defaults to False if ``method`` is ``"json_schema"`` or + ``"function_calling"``. Can only be non-null if ``method`` is + ``"json_schema"`` or ``"function_calling"``. kwargs: Additional keyword args aren't supported. @@ -1030,13 +1025,8 @@ class AnswerWithJustification(BaseModel): # 'parsing_error': None # } """ # noqa: E501 - if method is None: - method = "json_schema" - strict = False if strict is None else strict - - if strict is None: - strict = True if method == "json_schema" else None - + if method in ("json_schema", "function_calling") and strict is None: + strict = False return super().with_structured_output( schema, method=method, include_raw=include_raw, strict=strict, **kwargs ) diff --git a/libs/partners/openai/langchain_openai/chat_models/base.py b/libs/partners/openai/langchain_openai/chat_models/base.py index 92bccb84c3a25..7878618c8268d 100644 --- a/libs/partners/openai/langchain_openai/chat_models/base.py +++ b/libs/partners/openai/langchain_openai/chat_models/base.py @@ -1222,13 +1222,13 @@ def with_structured_output( method: The method for steering model generation, one of: - - "function_calling": - Uses OpenAI's tool-calling (formerly called function calling) - API: https://platform.openai.com/docs/guides/function-calling - "json_schema": Uses OpenAI's Structured Output API: https://platform.openai.com/docs/guides/structured-outputs - Supported for "gpt-4o-mini", "gpt-4o-2024-08-06", and later + Supported for "gpt-4o-mini", "gpt-4o-2024-08-06", "o1", and later models. + - "function_calling": + Uses OpenAI's tool-calling (formerly called function calling) + API: https://platform.openai.com/docs/guides/function-calling - "json_mode": Uses OpenAI's JSON mode. Note that if using JSON mode then you must include instructions for formatting the output into the @@ -1260,9 +1260,9 @@ def with_structured_output( - None: ``strict`` argument will not be passed to the model. - If ``method`` is "json_schema" defaults to True. If ``method`` is - "function_calling" or "json_mode" defaults to None. Can only be - non-null if ``method`` is "function_calling" or "json_schema". + Defaults to False if ``method`` is ``"json_schema"`` or + ``"function_calling"``. Can only be non-null if ``method`` is + ``"json_schema"`` or ``"function_calling"``. kwargs: Additional keyword args aren't supported. @@ -1950,9 +1950,7 @@ def with_structured_output( self, schema: Optional[_DictOrPydanticClass] = None, *, - method: Optional[ - Literal["function_calling", "json_mode", "json_schema"] - ] = None, + method: Literal["function_calling", "json_mode", "json_schema"] = "json_schema", include_raw: bool = False, strict: Optional[bool] = None, **kwargs: Any, @@ -1997,9 +1995,6 @@ def with_structured_output( - https://platform.openai.com/docs/guides/structured-outputs/structured-outputs-vs-json-mode - https://platform.openai.com/docs/guides/structured-outputs/function-calling-vs-response-format - Defaults to ``"json_schema"``. If both ``method`` and ``strict`` are not - supplied, we default to ``"json_schema"`` with ``strict=False`` (see below). - include_raw: If False then only the parsed structured output is returned. If an error occurs during model output parsing it will be raised. If True @@ -2019,9 +2014,9 @@ def with_structured_output( - None: ``strict`` argument will not be passed to the model. - If ``method`` is not supplied, defaults to False. - If ``method`` is "json_schema", defaults to True. - If ``method`` is "function_calling" or "json_mode", defaults to None. + If ``method`` is "json_schema" or "function_calling" defaults to True. + If ``method`` is "json_mode" defaults to None. Can only be non-null + if ``method`` is "function_calling" or "json_schema". kwargs: Additional keyword args aren't supported. @@ -2228,13 +2223,8 @@ class AnswerWithJustification(BaseModel): # 'parsing_error': None # } """ # noqa: E501 - if method is None: - method = "json_schema" - strict = False if strict is None else strict - - if strict is None: - strict = True if method == "json_schema" else None - + if method in ("json_schema", "function_calling") and strict is None: + strict = False return super().with_structured_output( schema, method=method, include_raw=include_raw, strict=strict, **kwargs ) diff --git a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py index afa330144bc64..c950148b3280b 100644 --- a/libs/partners/openai/tests/integration_tests/chat_models/test_base.py +++ b/libs/partners/openai/tests/integration_tests/chat_models/test_base.py @@ -827,13 +827,11 @@ class magic_function(BaseModel): @pytest.mark.parametrize( - ("model", "method", "strict"), - [("gpt-4o", "function_calling", True), ("gpt-4o-2024-08-06", "json_schema", None)], + ("model", "method"), + [("gpt-4o", "function_calling"), ("gpt-4o-2024-08-06", "json_schema")], ) def test_structured_output_strict( - model: str, - method: Literal["function_calling", "json_schema"], - strict: Optional[bool], + model: str, method: Literal["function_calling", "json_schema"] ) -> None: """Test to verify structured output with strict=True.""" @@ -849,7 +847,7 @@ class Joke(BaseModelProper): punchline: str = FieldProper(description="answer to resolve the joke") # Pydantic class - chat = llm.with_structured_output(Joke, method=method, strict=strict) + chat = llm.with_structured_output(Joke, method=method, strict=True) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, Joke) @@ -858,7 +856,7 @@ class Joke(BaseModelProper): # Schema chat = llm.with_structured_output( - Joke.model_json_schema(), method=method, strict=strict + Joke.model_json_schema(), method=method, strict=True ) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, dict) @@ -879,14 +877,14 @@ class InvalidJoke(BaseModelProper): default="foo", description="answer to resolve the joke" ) - chat = llm.with_structured_output(InvalidJoke, method=method, strict=strict) + chat = llm.with_structured_output(InvalidJoke, method=method, strict=True) with pytest.raises(openai.BadRequestError): chat.invoke("Tell me a joke about cats.") with pytest.raises(openai.BadRequestError): next(chat.stream("Tell me a joke about cats.")) chat = llm.with_structured_output( - InvalidJoke.model_json_schema(), method=method, strict=strict + InvalidJoke.model_json_schema(), method=method, strict=True ) with pytest.raises(openai.BadRequestError): chat.invoke("Tell me a joke about cats.") @@ -894,11 +892,9 @@ class InvalidJoke(BaseModelProper): next(chat.stream("Tell me a joke about cats.")) -@pytest.mark.parametrize( - ("model", "method", "strict"), [("gpt-4o-2024-08-06", "json_schema", None)] -) +@pytest.mark.parametrize(("model", "method"), [("gpt-4o-2024-08-06", "json_schema")]) def test_nested_structured_output_strict( - model: str, method: Literal["json_schema"], strict: Optional[bool] + model: str, method: Literal["json_schema"] ) -> None: """Test to verify structured output with strict=True for nested object.""" @@ -918,7 +914,7 @@ class JokeWithEvaluation(TypedDict): self_evaluation: SelfEvaluation # Schema - chat = llm.with_structured_output(JokeWithEvaluation, method=method, strict=strict) + chat = llm.with_structured_output(JokeWithEvaluation, method=method, strict=True) result = chat.invoke("Tell me a joke about cats.") assert isinstance(result, dict) assert set(result.keys()) == {"setup", "punchline", "self_evaluation"}