Merge pull request #216 from huggingface/unit_2_1_smolagents

[UNIT] Final review of Unit 2.1 on smolagents - release 25/2/25

notebooks/unit2/smolagents/vision_web_browser.py

@@ -0,0 +1,207 @@
+import argparse
+from io import BytesIO
+from time import sleep
+
+import helium
+from dotenv import load_dotenv
+from PIL import Image
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+
+from smolagents import CodeAgent, DuckDuckGoSearchTool, tool
+from smolagents.agents import ActionStep
+from smolagents.cli import load_model
+
+
+alfred_guest_list_request = """
+I am Alfred, the butler of Wayne Manor, responsible for verifying the identity of guests at party. A superhero has arrived at the entrance claiming to be Wonder Woman, but I need to confirm if she is who she says she is.
+
+Please search for images of Wonder Woman and generate a detailed visual description based on those images. Additionally, navigate to Wikipedia to gather key details about her appearance. With this information, I can determine whether to grant her access to the event.
+"""
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="Run a web browser automation script with a specified model.")
+    parser.add_argument(
+        "prompt",
+        type=str,
+        nargs="?",  # Makes it optional
+        default=alfred_guest_list_request,
+        help="The prompt to run with the agent",
+    )
+    parser.add_argument(
+        "--model-type",
+        type=str,
+        default="LiteLLMModel",
+        help="The model type to use (e.g., OpenAIServerModel, LiteLLMModel, TransformersModel, HfApiModel)",
+    )
+    parser.add_argument(
+        "--model-id",
+        type=str,
+        default="gpt-4o",
+        help="The model ID to use for the specified model type",
+    )
+    return parser.parse_args()
+
+
+def save_screenshot(memory_step: ActionStep, agent: CodeAgent) -> None:
+    sleep(1.0)  # Let JavaScript animations happen before taking the screenshot
+    driver = helium.get_driver()
+    current_step = memory_step.step_number
+    if driver is not None:
+        for previous_memory_step in agent.memory.steps:  # Remove previous screenshots from logs for lean processing
+            if isinstance(previous_memory_step, ActionStep) and previous_memory_step.step_number <= current_step - 2:
+                previous_memory_step.observations_images = None
+        png_bytes = driver.get_screenshot_as_png()
+        image = Image.open(BytesIO(png_bytes))
+        print(f"Captured a browser screenshot: {image.size} pixels")
+        memory_step.observations_images = [image.copy()]  # Create a copy to ensure it persists, important!
+
+    # Update observations with current URL
+    url_info = f"Current url: {driver.current_url}"
+    memory_step.observations = (
+        url_info if memory_step.observations is None else memory_step.observations + "\n" + url_info
+    )
+    return
+
+
+@tool
+def search_item_ctrl_f(text: str, nth_result: int = 1) -> str:
+    """
+    Searches for text on the current page via Ctrl + F and jumps to the nth occurrence.
+    Args:
+        text: The text to search for
+        nth_result: Which occurrence to jump to (default: 1)
+    """
+    elements = driver.find_elements(By.XPATH, f"//*[contains(text(), '{text}')]")
+    if nth_result > len(elements):
+        raise Exception(f"Match n°{nth_result} not found (only {len(elements)} matches found)")
+    result = f"Found {len(elements)} matches for '{text}'."
+    elem = elements[nth_result - 1]
+    driver.execute_script("arguments[0].scrollIntoView(true);", elem)
+    result += f"Focused on element {nth_result} of {len(elements)}"
+    return result
+
+
+@tool
+def go_back() -> None:
+    """Goes back to previous page."""
+    driver.back()
+
+
+@tool
+def close_popups() -> str:
+    """
+    Closes any visible modal or pop-up on the page. Use this to dismiss pop-up windows! This does not work on cookie consent banners.
+    """
+    webdriver.ActionChains(driver).send_keys(Keys.ESCAPE).perform()
+
+
+def initialize_driver():
+    """Initialize the Selenium WebDriver."""
+    chrome_options = webdriver.ChromeOptions()
+    chrome_options.add_argument("--force-device-scale-factor=1")
+    chrome_options.add_argument("--window-size=1000,1350")
+    chrome_options.add_argument("--disable-pdf-viewer")
+    chrome_options.add_argument("--window-position=0,0")
+    return helium.start_chrome(headless=False, options=chrome_options)
+
+
+def initialize_agent(model):
+    """Initialize the CodeAgent with the specified model."""
+    return CodeAgent(
+        tools=[DuckDuckGoSearchTool(), go_back, close_popups, search_item_ctrl_f],
+        model=model,
+        additional_authorized_imports=["helium"],
+        step_callbacks=[save_screenshot],
+        max_steps=20,
+        verbosity_level=2,
+    )
+
+
+helium_instructions = """
+Use your web_search tool when you want to get Google search results.
+Then you can use helium to access websites. Don't use helium for Google search, only for navigating websites!
+Don't bother about the helium driver, it's already managed.
+We've already ran "from helium import *"
+Then you can go to pages!
+Code:
+```py
+go_to('github.com/trending')
+```<end_code>
+
+You can directly click clickable elements by inputting the text that appears on them.
+Code:
+```py
+click("Top products")
+```<end_code>
+
+If it's a link:
+Code:
+```py
+click(Link("Top products"))
+```<end_code>
+
+If you try to interact with an element and it's not found, you'll get a LookupError.
+In general stop your action after each button click to see what happens on your screenshot.
+Never try to login in a page.
+
+To scroll up or down, use scroll_down or scroll_up with as an argument the number of pixels to scroll from.
+Code:
+```py
+scroll_down(num_pixels=1200) # This will scroll one viewport down
+```<end_code>
+
+When you have pop-ups with a cross icon to close, don't try to click the close icon by finding its element or targeting an 'X' element (this most often fails).
+Just use your built-in tool `close_popups` to close them:
+Code:
+```py
+close_popups()
+```<end_code>
+
+You can use .exists() to check for the existence of an element. For example:
+Code:
+```py
+if Text('Accept cookies?').exists():
+    click('I accept')
+```<end_code>
+
+Proceed in several steps rather than trying to solve the task in one shot.
+And at the end, only when you have your answer, return your final answer.
+Code:
+```py
+final_answer("YOUR_ANSWER_HERE")
+```<end_code>
+
+If pages seem stuck on loading, you might have to wait, for instance `import time` and run `time.sleep(5.0)`. But don't overuse this!
+To list elements on page, DO NOT try code-based element searches like 'contributors = find_all(S("ol > li"))': just look at the latest screenshot you have and read it visually, or use your tool search_item_ctrl_f.
+Of course, you can act on buttons like a user would do when navigating.
+After each code blob you write, you will be automatically provided with an updated screenshot of the browser and the current browser url.
+But beware that the screenshot will only be taken at the end of the whole action, it won't see intermediate states.
+Don't kill the browser.
+When you have modals or cookie banners on screen, you should get rid of them before you can click anything else.
+"""
+
+
+def main():
+    # Load environment variables
+    load_dotenv()
+
+    # Parse command line arguments
+    args = parse_arguments()
+
+    # Initialize the model based on the provided arguments
+    model = load_model(args.model_type, args.model_id)
+
+    global driver
+    driver = initialize_driver()
+    agent = initialize_agent(model)
+
+    # Run the agent with the provided prompt
+    agent.python_executor("from helium import *", agent.state)
+    agent.run(args.prompt + helium_instructions)
+
+
+if __name__ == "__main__":
+    main()

units/en/_toctree.yml

@@ -37,11 +37,41 @@
   - local: unit1/dummy-agent-library
     title: Dummy Agent Library
   - local: unit1/tutorial
-    title: Let’s Create Our First Agent Using Smolagents
+    title: Let’s Create Our First Agent Using smolagents
   - local: unit1/final-quiz
     title: Unit 1 Final Quiz
   - local: unit1/conclusion
     title: Conclusion
+- title: Unit 2. Frameworks for AI Agents
+  sections:
+  - local: unit2/introduction
+    title: Frameworks for AI Agents
+- title: Unit 2.1 The smolagents framework
+  sections:
+  - local: unit2/smolagents/introduction
+    title: Introduction to smolagents
+  - local: unit2/smolagents/why_use_smolagents
+    title: Why use smolagents? 
+  - local: unit2/smolagents/quiz1
+    title: Quick Quiz 1
+  - local: unit2/smolagents/code_agents
+    title: Building Agents That Use Code
+  - local: unit2/smolagents/tool_calling_agents
+    title: Integrating Agents With Tools
+  - local: unit2/smolagents/tools
+    title: Tools
+  - local: unit2/smolagents/retrieval_agents
+    title: Retrieval Agents
+  - local: unit2/smolagents/quiz2
+    title: Quick Quiz 2
+  - local: unit2/smolagents/multi_agent_systems
+    title: Multi-Agent Systems
+  - local: unit2/smolagents/vision_agents
+    title: Vision and Browser agents
+  - local: unit2/smolagents/final_quiz
+    title: Final Quiz
+  - local: unit2/smolagents/conclusion
+    title: Conclusion
 - title: Bonus Unit 1. Fine-tuning an LLM for Function-calling
   sections:
   - local: bonus-unit1/introduction

units/en/unit2/introduction.mdx

@@ -0,0 +1,37 @@
+# Introduction to Agentic Frameworks
+
+<img src="https://huggingface.co/datasets/agents-course/course-images/resolve/main/en/unit2/thumbnail.jpg" alt="Thumbnail"/>
+
+Welcome to this second unit, where **we'll explore different agentic frameworks** that can be used to build powerful agentic applications. 
+
+We will study:
+
+- In Unit 2.1: [smolagents](https://huggingface.co/docs/smolagents/en/index)  
+- In Unit 2.2: [LlamaIndex](https://www.llamaindex.ai/)
+- In Unit 2.3: [LangGraph](https://www.langchain.com/langgraph)
+
+Let's dive in! 🕵
+
+## When to Use an Agentic Framework
+
+An agentic framework is **not always needed when building an application around LLMs**. They provide flexibility in the workflow to efficiently solve a specific task, but they're not always necessary. 
+
+Sometimes, **predefined workflows are sufficient** to fulfill user requests, and there is no real need for an agentic framework. If the approach to build an agent is simple, like a chain of prompts, using plain code may be enough. The advantage is that the developer will have **full control and understanding of their system without abstractions**.
+
+However, when the workflow becomes more complex, such as letting an LLM call functions or using multiple agents, these abstractions start to become helpful.
+
+Considering these ideas, we can already identify the need for some features:
+
+* An *LLM engine* that powers the system.
+* A *list of tools* the agent can access.  
+* A *parser* for extracting tool calls from the LLM output.
+* A *system prompt* synced with the parser.
+* A *memory system*.
+* *Error logging and retry mechanisms* to control LLM mistakes.
+We'll explore how these topics are resolved in various frameworks, including `smolagents`, `LlamaIndex`, and `LangGraph`.
+
+## Agentic Frameworks Units
+
+| Framework  | Description | Unit Author |
+|------------|----------------|----------------|
+| [smolagents](./smolagents/introduction) | Agents framework developed by Hugging Face. | Sergio Paniego - [HF](https://huggingface.co/sergiopaniego) - [X](https://x.com/sergiopaniego) - [Linkedin](https://www.linkedin.com/in/sergio-paniego-blanco) |