Merge branch 'jxnl:main' into main

README.md

@@ -3,10 +3,9 @@
 Instructor is a Python library that makes it a breeze to work with structured outputs from large language models (LLMs). Built on top of Pydantic, it provides a simple, transparent, and user-friendly API to manage validation, retries, and streaming responses. Get ready to supercharge your LLM workflows!
 
 [![Twitter Follow](https://img.shields.io/twitter/follow/jxnlco?style=social)](https://twitter.com/jxnlco)
-[![Discord](https://img.shields.io/discord/1192334452110659664?label=discord)](https://discord.gg/CV8sPM5k5Y)
+[![Discord](https://img.shields.io/discord/1192334452110659664?label=discord)](https://discord.gg/bD9YE9JArw)
 [![Downloads](https://img.shields.io/pypi/dm/instructor.svg)](https://pypi.python.org/pypi/instructor)
 
-
 ## Key Features
 
 - **Response Models**: Specify Pydantic models to define the structure of your LLM outputs
@@ -125,7 +124,6 @@ assert resp.name == "Jason"
 assert resp.age == 25
 ```
 
-
 ### Using Litellm
 
 ```python
@@ -251,7 +249,6 @@ user, completion = client.chat.completions.create_with_completion(
 
 ![with_completion](./docs/blog/posts/img/with_completion.png)
 
-
 ### Streaming Partial Objects: `create_partial`
 
 In order to handle streams, we still support `Iterable[T]` and `Partial[T]` but to simply the type inference, we've added `create_iterable` and `create_partial` methods as well!

docs/concepts/philosophy.md

@@ -16,7 +16,7 @@ The instructor values [simplicity](https://eugeneyan.com/writing/simplicity/) an
 
 ### Proof that its flexible
 
-1. If you build a system with OpenAI dirrectly, it is easy to incrementally adopt instructor.
+1. If you build a system with OpenAI directly, it is easy to incrementally adopt instructor.
 2. Add `response_model` and if you want to revert, just remove it.
 
 ## The zen of `instructor`

docs/concepts/usage.md

@@ -25,3 +25,15 @@ user, completion = client.chat.completions.create_with_completion(
 print(completion.usage)
 #> CompletionUsage(completion_tokens=9, prompt_tokens=82, total_tokens=91)
 ```
+
+You can catch an IncompleteOutputException whenever the context length is exceeded and react accordingly, such as by trimming your prompt by the number of exceeding tokens.
+
+```python
+from instructor.exceptions import IncompleteOutputException
+
+try:
+    # call to your model
+except IncompleteOutputException as e:
+    token_count = e.last_completion.usage.total_tokens
+    # your logic here
+```

docs/examples/ollama.md

@@ -17,12 +17,12 @@ Instructor's patch enhances an openai api with the following features:
 
 ## Ollama
 
-Start by downloading [Ollama](https://ollama.ai/download), and then pull a model such as Llama 2 or Mistral.
+Start by downloading [Ollama](https://ollama.ai/download), and then pull a model such as Llama 3 or Mistral.
 
 !!! tip "Make sure you update your `ollama` to the latest version!"
 
 ```
-ollama pull llama2
+ollama pull llama3
 ```
 
 ```python
@@ -49,7 +49,7 @@ client = instructor.from_openai(
 )
 
 resp = client.chat.completions.create(
-    model="llama2",
+    model="llama3",
     messages=[
         {
             "role": "user",

docs/help.md

@@ -4,7 +4,7 @@ If you need help getting started with Instructor or with advanced usage, the fol
 
 ## :material-discord: Discord
 
-The [Discord](https://discord.gg/CV8sPM5k5Y) is a great place to ask questions and get help from the community.
+The [Discord](https://discord.gg/bD9YE9JArw) is a great place to ask questions and get help from the community.
 
 ## :material-creation: Concepts
 

docs/index.md

@@ -5,7 +5,7 @@ _Structured outputs powered by llms. Designed for simplicity, transparency, and
 ---
 
 [![Twitter Follow](https://img.shields.io/twitter/follow/jxnlco?style=social)](https://twitter.com/jxnlco)
-[![Discord](https://img.shields.io/discord/1192334452110659664?label=discord)](https://discord.gg/CV8sPM5k5Y)
+[![Discord](https://img.shields.io/discord/1192334452110659664?label=discord)](https://discord.gg/bD9YE9JArw)
 [![Downloads](https://img.shields.io/pypi/dm/instructor.svg)](https://pypi.python.org/pypi/instructor)
 [![GPT](https://img.shields.io/badge/docs-InstructorGPT-blue)](https://chat.openai.com/g/g-EvZweRWrE-instructor-gpt)
 

instructor/cli/usage.py

@@ -50,6 +50,9 @@ async def get_usage_for_past_n_days(n_days: int) -> list[dict[str, Any]]:
 # Define the cost per unit for each model
 MODEL_COSTS = {
     "gpt-4o": {"prompt": 0.005 / 1000, "completion": 0.015 / 1000},
+    "gpt-4o-2024-05-13": {"prompt": 0.005 / 1000, "completion": 0.015 / 1000},
+    "gpt-4-turbo": {"prompt": 0.01 / 1000, "completion": 0.03 / 1000},
+    "gpt-4-turbo-2024-04-09": {"prompt": 0.01 / 1000, "completion": 0.03 / 1000},
     "gpt-4-0125-preview": {"prompt": 0.01 / 1000, "completion": 0.03 / 1000},
     "gpt-4-turbo-preview": {"prompt": 0.01 / 1000, "completion": 0.03 / 1000},
     "gpt-4-1106-preview": {"prompt": 0.01 / 1000, "completion": 0.03 / 1000},
@@ -85,8 +88,12 @@ def get_model_cost(
         return MODEL_COSTS["gpt-3.5-turbo-16k"]
     elif model.startswith("gpt-3.5-turbo"):
         return MODEL_COSTS["gpt-3.5-turbo"]
+    elif model.startswith("gpt-4-turbo"):
+        return MODEL_COSTS["gpt-4-turbo-preview"]
     elif model.startswith("gpt-4-32k"):
         return MODEL_COSTS["gpt-4-32k"]
+    elif model.startswith("gpt-4o"):
+        return MODEL_COSTS["gpt-4o"]
     elif model.startswith("gpt-4"):
         return MODEL_COSTS["gpt-4"]
     else:
@@ -111,11 +118,11 @@ def calculate_cost(
 
 def group_and_sum_by_date_and_snapshot(usage_data: list[dict[str, Any]]) -> Table:
     """Group and sum the usage data by date and snapshot, including costs."""
-    summary: defaultdict[str, defaultdict[str, dict[str, Union[int, float]]]] = (
-        defaultdict(
-            lambda: defaultdict(
-                lambda: {"total_requests": 0, "total_tokens": 0, "total_cost": 0.0}
-            )
+    summary: defaultdict[
+        str, defaultdict[str, dict[str, Union[int, float]]]
+    ] = defaultdict(
+        lambda: defaultdict(
+            lambda: {"total_requests": 0, "total_tokens": 0, "total_cost": 0.0}
         )
     )
 

instructor/client.py

@@ -63,7 +63,8 @@ def create(
         validation_context: dict[str, Any] | None = None,
         strict: bool = True,
         **kwargs: Any,
-    ) -> Awaitable[T]: ...
+    ) -> Awaitable[T]:
+        ...
 
     @overload
     def create(
@@ -74,7 +75,8 @@ def create(
         validation_context: dict[str, Any] | None = None,
         strict: bool = True,
         **kwargs: Any,
-    ) -> T: ...
+    ) -> T:
+        ...
 
     # TODO: we should overload a case where response_model is None
     def create(
@@ -106,7 +108,8 @@ def create_partial(
         validation_context: dict[str, Any] | None = None,
         strict: bool = True,
         **kwargs: Any,
-    ) -> AsyncGenerator[T, None]: ...
+    ) -> AsyncGenerator[T, None]:
+        ...
 
     @overload
     def create_partial(
@@ -117,7 +120,8 @@ def create_partial(
         validation_context: dict[str, Any] | None = None,
         strict: bool = True,
         **kwargs: Any,
-    ) -> Generator[T, None, None]: ...
+    ) -> Generator[T, None, None]:
+        ...
 
     def create_partial(
         self,
@@ -128,8 +132,6 @@ def create_partial(
         strict: bool = True,
         **kwargs: Any,
     ) -> Generator[T, None, None] | AsyncGenerator[T, None]:
-        assert self.provider != Provider.ANTHROPIC, "Anthropic doesn't support partial"
-
         kwargs["stream"] = True
 
         kwargs = self.handle_kwargs(kwargs)
@@ -153,7 +155,8 @@ def create_iterable(
         validation_context: dict[str, Any] | None = None,
         strict: bool = True,
         **kwargs: Any,
-    ) -> AsyncGenerator[T, None]: ...
+    ) -> AsyncGenerator[T, None]:
+        ...
 
     @overload
     def create_iterable(
@@ -164,7 +167,8 @@ def create_iterable(
         validation_context: dict[str, Any] | None = None,
         strict: bool = True,
         **kwargs: Any,
-    ) -> Generator[T, None, None]: ...
+    ) -> Generator[T, None, None]:
+        ...
 
     def create_iterable(
         self,
@@ -199,7 +203,8 @@ def create_with_completion(
         validation_context: dict[str, Any] | None = None,
         strict: bool = True,
         **kwargs: Any,
-    ) -> Awaitable[tuple[T, Any]]: ...
+    ) -> Awaitable[tuple[T, Any]]:
+        ...
 
     @overload
     def create_with_completion(
@@ -210,7 +215,8 @@ def create_with_completion(
         validation_context: dict[str, Any] | None = None,
         strict: bool = True,
         **kwargs: Any,
-    ) -> tuple[T, Any]: ...
+    ) -> tuple[T, Any]:
+        ...
 
     def create_with_completion(
         self,
@@ -311,8 +317,6 @@ async def create_iterable(
         strict: bool = True,
         **kwargs: Any,
     ) -> AsyncGenerator[T, None]:
-        assert self.provider != Provider.ANTHROPIC, "Anthropic doesn't support iterable"
-
         kwargs = self.handle_kwargs(kwargs)
         kwargs["stream"] = True
         async for item in await self.create_fn(
@@ -423,7 +427,8 @@ def from_litellm(
     completion: Callable[..., Any],
     mode: instructor.Mode = instructor.Mode.TOOLS,
     **kwargs: Any,
-) -> Instructor: ...
+) -> Instructor:
+    ...
 
 
 @overload

instructor/client_anthropic.py

@@ -13,7 +13,8 @@ def from_anthropic(
     ),
     mode: instructor.Mode = instructor.Mode.ANTHROPIC_JSON,
     **kwargs: Any,
-) -> instructor.Instructor: ...
+) -> instructor.Instructor:
+    ...
 
 
 @overload
@@ -25,7 +26,8 @@ def from_anthropic(
     ),
     mode: instructor.Mode = instructor.Mode.ANTHROPIC_JSON,
     **kwargs: Any,
-) -> instructor.AsyncInstructor: ...
+) -> instructor.AsyncInstructor:
+    ...
 
 
 def from_anthropic(

instructor/client_cohere.py

@@ -23,15 +23,17 @@ def from_cohere(
     client: cohere.Client,
     mode: instructor.Mode = instructor.Mode.COHERE_TOOLS,
     **kwargs: Any,
-) -> instructor.Instructor: ...
+) -> instructor.Instructor:
+    ...
 
 
 @overload
 def from_cohere(
     client: cohere.AsyncClient,
     mode: instructor.Mode = instructor.Mode.COHERE_TOOLS,
     **kwargs: Any,
-) -> instructor.AsyncInstructor: ...
+) -> instructor.AsyncInstructor:
+    ...
 
 
 def from_cohere(

instructor/client_groq.py

@@ -11,15 +11,17 @@ def from_groq(
     client: groq.Groq,
     mode: instructor.Mode = instructor.Mode.TOOLS,
     **kwargs: Any,
-) -> instructor.Instructor: ...
+) -> instructor.Instructor:
+    ...
 
 
 @overload
 def from_groq(
     client: groq.AsyncGroq,
     mode: instructor.Mode = instructor.Mode.TOOLS,
     **kwargs: Any,
-) -> instructor.Instructor: ...
+) -> instructor.Instructor:
+    ...
 
 
 def from_groq(

instructor/client_mistral.py

@@ -12,15 +12,17 @@ def from_mistral(
     client: mistralai.client.MistralClient,
     mode: instructor.Mode = instructor.Mode.MISTRAL_TOOLS,
     **kwargs: Any,
-) -> instructor.Instructor: ...
+) -> instructor.Instructor:
+    ...
 
 
 @overload
 def from_mistral(
     client: mistralaiasynccli.MistralAsyncClient,
     mode: instructor.Mode = instructor.Mode.MISTRAL_TOOLS,
     **kwargs: Any,
-) -> instructor.AsyncInstructor: ...
+) -> instructor.AsyncInstructor:
+    ...
 
 
 def from_mistral(

instructor/dsl/iterable.py

@@ -79,7 +79,12 @@ def extract_json(
     ) -> Generator[str, None, None]:
         for chunk in completion:
             try:
-                if chunk.choices:
+                if mode == Mode.ANTHROPIC_JSON:
+                    if json_chunk := chunk.delta.text:
+                        yield json_chunk
+                if mode == Mode.ANTHROPIC_TOOLS:
+                    yield chunk.model_extra.get("delta", "").get("partial_json", "")
+                elif chunk.choices:
                     if mode == Mode.FUNCTIONS:
                         if json_chunk := chunk.choices[0].delta.function_call.arguments:
                             yield json_chunk
@@ -102,7 +107,12 @@ async def extract_json_async(
     ) -> AsyncGenerator[str, None]:
         async for chunk in completion:
             try:
-                if chunk.choices:
+                if mode == Mode.ANTHROPIC_JSON:
+                    if json_chunk := chunk.delta.text:
+                        yield json_chunk
+                if mode == Mode.ANTHROPIC_TOOLS:
+                    yield chunk.model_extra.get("delta", "").get("partial_json", "")
+                elif chunk.choices:
                     if mode == Mode.FUNCTIONS:
                         if json_chunk := chunk.choices[0].delta.function_call.arguments:
                             yield json_chunk

instructor/dsl/maybe.py

@@ -56,7 +56,7 @@ def __bool__(self):
     return create_model(
         f"Maybe{model.__name__}",
         __base__=MaybeBase,
-        reuslts=(
+        result=(
             Optional[model],
             Field(
                 default=None,

instructor/dsl/partial.py

@@ -150,7 +150,12 @@ def extract_json(
     ) -> Generator[str, None, None]:
         for chunk in completion:
             try:
-                if chunk.choices:
+                if mode == Mode.ANTHROPIC_JSON:
+                    if json_chunk := chunk.delta.text:
+                        yield json_chunk
+                if mode == Mode.ANTHROPIC_TOOLS:
+                    yield chunk.model_extra.get("delta", "").get("partial_json", "")
+                elif chunk.choices:
                     if mode == Mode.FUNCTIONS:
                         if json_chunk := chunk.choices[0].delta.function_call.arguments:
                             yield json_chunk
@@ -173,7 +178,12 @@ async def extract_json_async(
     ) -> AsyncGenerator[str, None]:
         async for chunk in completion:
             try:
-                if chunk.choices:
+                if mode == Mode.ANTHROPIC_JSON:
+                    if json_chunk := chunk.delta.text:
+                        yield json_chunk
+                if mode == Mode.ANTHROPIC_TOOLS:
+                    yield chunk.model_extra.get("delta", "").get("partial_json", "")
+                elif chunk.choices:
                     if mode == Mode.FUNCTIONS:
                         if json_chunk := chunk.choices[0].delta.function_call.arguments:
                             yield json_chunk