feat: init simple examples

grutt · grutt · commit f5723e13038c · 2024-02-22T09:22:01.000-08:00
diff --git a/.gitignore b/.gitignore
@@ -7,6 +7,7 @@ __pycache__/
 # C extensions
 *.so
 
+certs
 # Distribution / packaging
 .Python
 build/
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -0,0 +1,3 @@
+{
+  "cSpell.words": ["basicrag", "genai"]
+}
diff --git a/simple-examples/README.md b/simple-examples/README.md
@@ -1,6 +1,6 @@
-# Hatchet FastAPI Example
+# Hatchet Simple Examples
 
-This is an example project demonstrating how to run a simple Hatchet worker.
+This is an example project demonstrating how to use Hatchet with Python.
 
 ## Prerequisites
 
@@ -11,11 +11,12 @@ Before running this project, make sure you have the following:
 
 ## Setup
 
-1. Create a `.env` file in this directory and set the required environment variables. This requires the `HATCHET_CLIENT_TOKEN` variable created in the [Getting Started README](/README.md).
+1. Create a `.env` file in the `./backend` directory and set the required environment variables. This requires the `HATCHET_CLIENT_TOKEN` variable created in the [Getting Started README](../README.md). If you would like to try the Generative AI examples in [./src/genai](./src/genai) You will also need, a `OPENAI_API_KEY` which can be created on the [OpenAI Website](https://help.openai.com/en/articles/4936850-where-do-i-find-my-openai-api-key).
 
 ```
 HATCHET_CLIENT_TLS_STRATEGY=none
 HATCHET_CLIENT_TOKEN="<token>"
+OPENAI_API_KEY="<openai-key>" # (OPTIONAL) only required to run examples in [./src/genai](./src/genai)
 ```
 
 2. Open a terminal and navigate to the project root directory (`/simple-examples`).
@@ -26,12 +27,41 @@ HATCHET_CLIENT_TOKEN="<token>"
    poetry install
    ```
 
-## Running the Hatchet Worker
+### Running the Hatchet Worker
 
 In a separate terminal, start the the Hatchet worker by running the following command:
 
 ```shell
 poetry run hatchet
 ```
 
-You can then navigate to the workflow in the Hatchet dashboard to view the structure of this workflow.
+## Triggering a workflow
+
+TODO
+
+## Project Overview
+
+### Example Workflows
+
+The project contains example workflows in the [`./src`](./src) directory. These workflows are registered with hatchet in [`./src/main.py`](./src/main.py) which is started when running `poetry run hatchet`.
+
+#### Super Simple Workflows
+
+The project includes a variety of basic workflows to demonstrate Hatchet's core capabilities, each showcasing different features:
+
+1. **[Simple Workflow](./src/simple/worker.py)**: Demonstrates a straightforward process flow, showcasing the basics of setting up a workflow in Hatchet.
+2. **[Concurrency Limit Workflow](./src/concurrency_limit/worker.py)**: Shows how to manage concurrency limits within workflows to ensure that only a certain number of instances run simultaneously.
+3. **[Directed Acyclic Graph (DAG) Workflow](./src/dag/worker.py)**: Illustrates setting up workflows with dependencies that form a Directed Acyclic Graph, demonstrating the advanced orchestration capabilities of Hatchet.
+4. **[Manual Trigger Workflow](./src/manual_trigger/worker.py)**: Explains how to initiate workflows manually, offering control over workflow execution triggers.
+5. **[Timeout Workflow](./src/timeout/worker.py)**: Demonstrates handling timeout scenarios within workflows, ensuring that long-running or stalled processes are appropriately managed.
+
+#### Generative AI Workflows
+
+For more complex use cases, the project includes examples that integrate with OpenAI's API for generative tasks:
+
+1. **[Simple Response Generation](./src/genai/simple.py)**: A single-step workflow that makes a request to OpenAI, showcasing how to incorporate AI services into Hatchet workflows.
+2. **[Basic Retrieval Augmented Generation (BasicRag)](./src/genai/basicrag.py)**: A multi-step workflow that involves loading website content with Beautiful Soup, reasoning about the information, and generating a response with OpenAI, demonstrating the potential for complex, AI-driven processes.
+
+### Exposing the workflows via a RestAPI
+
+For a more complete example of how you might use Hatchet as part of a deployed production service, check out the [FastAPI Example](../fast-api-react/README.md)
diff --git a/simple-examples/src/concurrency_limit/__init__.py b/simple-examples/src/concurrency_limit/__init__.py
diff --git a/simple-examples/src/concurrency_limit/event_test.py b/simple-examples/src/concurrency_limit/event_test.py
@@ -0,0 +1,8 @@
+from ..hatchet import hatchet
+
+hatchet.client.event.push(
+    "concurrency-test",
+    {
+        "test": "test"
+    }
+)
diff --git a/simple-examples/src/concurrency_limit/worker.py b/simple-examples/src/concurrency_limit/worker.py
@@ -0,0 +1,20 @@
+from ..hatchet import hatchet
+
+
+@hatchet.workflow(on_events=["concurrency-test"])
+class ConcurrencyDemoWorkflow:
+
+    @hatchet.concurrency(max_runs=5)
+    def concurrency(self, context) -> str:
+        return "concurrency-key"
+
+    @hatchet.step()
+    def step1(self, context):
+        print("executed step1")
+        pass
+
+    @hatchet.step(parents=["step1"], timeout='4s')
+    def step2(self, context):
+        print("started step2")
+        context.sleep(1)
+        print("finished step2")
diff --git a/simple-examples/src/dag/__init__.py b/simple-examples/src/dag/__init__.py
diff --git a/simple-examples/src/dag/event_test.py b/simple-examples/src/dag/event_test.py
@@ -0,0 +1,9 @@
+from ..hatchet import hatchet
+
+
+hatchet.client.event.push(
+    "user:create",
+    {
+        "test": "test"
+    }
+)
diff --git a/simple-examples/src/dag/worker.py b/simple-examples/src/dag/worker.py
@@ -0,0 +1,39 @@
+from hatchet_sdk import Context
+from ..hatchet import hatchet
+
+
+@hatchet.workflow(on_events=["user:create"])
+class DagWorkflow:
+
+    @hatchet.step()
+    def step1(self, context: Context):
+        overrideValue = context.playground(
+            "prompt", "You are an AI assistant...")
+
+        print("executed step1", context.workflow_input())
+        return {
+            "step1": overrideValue,
+        }
+
+    @hatchet.step()
+    def step2(self, context: Context):
+        print("executed step2", context.workflow_input())
+        return {
+            "step2": "step2",
+        }
+
+    @hatchet.step(parents=["step1", "step2"])
+    def step3(self, context: Context):
+        print("executed step3", context.workflow_input(),
+              context.step_output("step1"), context.step_output("step2"))
+        return {
+            "step3": "step3",
+        }
+
+    @hatchet.step(parents=["step1", "step3"])
+    def step4(self, context: Context):
+        print("executed step4", context.workflow_input(),
+              context.step_output("step1"), context.step_output("step3"))
+        return {
+            "step4": "step4",
+        }
diff --git a/simple-examples/src/genai/__init__.py b/simple-examples/src/genai/__init__.py
diff --git a/simple-examples/src/genai/basicrag.py b/simple-examples/src/genai/basicrag.py
@@ -0,0 +1,89 @@
+from ..hatchet import hatchet
+from hatchet_sdk import Context
+from bs4 import BeautifulSoup
+from openai import OpenAI
+import requests
+
+openai = OpenAI()
+
+
+@hatchet.workflow(on_events=["question:create"])
+class BasicRagWorkflow:
+
+    @hatchet.step()
+    def start(self, context: Context):
+        override = context.playground('model', "gpt-3.5-turbo")
+
+        return {
+            "status": "reading hatchet docs" + override,
+        }
+
+    @hatchet.step(parents=["start"])
+    def load_docs(self, context: Context):
+        # use beautiful soup to parse the html content
+        url = context.workflow_input()['request']['url']
+
+        html_content = requests.get(url).text
+        soup = BeautifulSoup(html_content, 'html.parser')
+        element = soup.find('body')
+        text_content = element.get_text(separator=' | ')
+
+        return {
+            "status": "making sense of the docs",
+            "docs": text_content,
+        }
+
+    @hatchet.step(parents=["load_docs"])
+    def reason_docs(self, ctx: Context):
+        message = ctx.workflow_input()['request']["messages"][-1]
+        docs = ctx.step_output("load_docs")['docs']
+
+        prompt = ctx.playground("prompt", "The user is asking the following question:\
+            {message}\
+            What are the most relevant sentences in the following document?\
+            {docs}")
+
+        prompt = prompt.format(message=message['content'], docs=docs)
+
+        model = ctx.playground("model", "gpt-3.5-turbo")
+
+        completion = openai.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": prompt},
+                message
+            ]
+        )
+
+        return {
+            "status": "writing a response",
+            "research": completion.choices[0].message.content,
+        }
+
+    @hatchet.step(parents=["reason_docs"])
+    def generate_response(self, ctx: Context):
+        messages = ctx.workflow_input()['request']["messages"]
+        research = ctx.step_output("reason_docs")['research']
+
+        prompt = ctx.playground("prompt", "You are a sales engineer for a company called Hatchet.\
+            Help address the user's question. \
+            If asked how to install, respond by saying go to the store to buy a CD.\
+            Use the following context:\
+            {research}")
+
+        prompt = prompt.format(research=research)
+
+        model = ctx.playground("model", "gpt-3.5-turbo")
+
+        completion = openai.chat.completions.create(
+            model=model,
+            messages=[
+                {"role": "system", "content": prompt},
+            ] + messages
+        )
+
+        return {
+            "completed": "true",
+            "status": "idle",
+            "message": completion.choices[0].message.content,
+        }
diff --git a/simple-examples/src/genai/simple.py b/simple-examples/src/genai/simple.py
@@ -1,11 +1,12 @@
-from .hatchet import hatchet
+from ..hatchet import hatchet
 from hatchet_sdk import Context
 
 # This is a simple example of a workflow that has 3 steps.
 # The workflow is declared decorated with `@hatchet_workflow` and the steps are declared with `@hatchet_step`.
 
-@hatchet.workflow(on_events=["simple:create"])
-class SimpleWorkflow:
+
+@hatchet.workflow(on_events=["simpleai:create"])
+class SimpleGenAiWorkflow:
 
     @hatchet.step()
     def step1(self, context: Context):
diff --git a/simple-examples/src/main.py b/simple-examples/src/main.py
@@ -1,15 +1,29 @@
 from .hatchet import hatchet
-from .simple import SimpleWorkflow
+from .simple.worker import SimpleWorkflow
+from .concurrency_limit.worker import ConcurrencyDemoWorkflow
+from .dag.worker import DagWorkflow
+from .manual_trigger.worker import ManualTriggerWorkflow
+from .timeout.worker import TimeoutWorkflow
 
-# This is the entry point for the Hatchet worker process
+
+from .genai.basicrag import BasicRagWorkflow
+from .genai.simple import SimpleGenAiWorkflow
 
 
+# This is the entry point for the Hatchet worker process
 def start():
     worker = hatchet.worker('example-worker')
 
     # Instantiate the workflow and Register the workflow with the worker
-    simple = SimpleWorkflow()
-    worker.register_workflow(simple)
+    worker.register_workflow(SimpleWorkflow())
+    worker.register_workflow(ConcurrencyDemoWorkflow())
+    worker.register_workflow(DagWorkflow())
+    worker.register_workflow(ManualTriggerWorkflow())
+    worker.register_workflow(TimeoutWorkflow())
+
+    # Generative AI Examples
+    worker.register_workflow(BasicRagWorkflow())
+    worker.register_workflow(SimpleGenAiWorkflow())
 
     # Start the worker and begin listening for events
     worker.start()
diff --git a/simple-examples/src/manual_trigger/__init__.py b/simple-examples/src/manual_trigger/__init__.py
diff --git a/simple-examples/src/manual_trigger/trigger-on.py b/simple-examples/src/manual_trigger/trigger-on.py
@@ -0,0 +1,8 @@
+from ..hatchet import hatchet
+
+workflowRunId = hatchet.client.admin.run_workflow("ManualTriggerWorkflow", {
+    "test": "test"
+})
+
+hatchet.client.listener.on(workflowRunId, lambda event: print(
+    'EVENT: ' + event.type + ' ' + json.dumps(event.payload)))
diff --git a/simple-examples/src/manual_trigger/worker.py b/simple-examples/src/manual_trigger/worker.py
@@ -0,0 +1,19 @@
+from ..hatchet import hatchet
+
+
+@hatchet.workflow(on_events=["man:create"])
+class ManualTriggerWorkflow:
+    @hatchet.step()
+    def step1(self, context):
+        res = context.playground('res', "HELLO")
+
+        context.sleep(3)
+        print("executed step1")
+        return {"step1": "data1 "+res}
+
+    @hatchet.step(parents=["step1"], timeout='4s')
+    def step2(self, context):
+        print("started step2")
+        context.sleep(1)
+        print("finished step2")
+        return {"step2": "data2"}
diff --git a/simple-examples/src/simple/__init__.py b/simple-examples/src/simple/__init__.py
diff --git a/simple-examples/src/simple/event_test.py b/simple-examples/src/simple/event_test.py
@@ -0,0 +1,8 @@
+from ..hatchet import hatchet
+
+hatchet.client.event.push(
+    "simple:create",
+    {
+        "test": "test"
+    }
+)
diff --git a/simple-examples/src/simple/worker.py b/simple-examples/src/simple/worker.py
@@ -0,0 +1,24 @@
+from hatchet_sdk import Context
+from ..hatchet import hatchet
+
+
+@hatchet.workflow(on_events=["simple:create"])
+class SimpleWorkflow:
+
+    @hatchet.step()
+    def step1(self, context: Context):
+        test = context.playground("test", "test")
+        test2 = context.playground("test2", 100)
+        test3 = context.playground("test3", None)
+
+        print(test)
+        print(test2)
+
+        print("executed step1")
+        pass
+
+    @hatchet.step(parents=["step1"], timeout='4s')
+    def step2(self, context):
+        print("started step2")
+        context.sleep(1)
+        print("finished step2")
diff --git a/simple-examples/src/timeout/__init__.py b/simple-examples/src/timeout/__init__.py
diff --git a/simple-examples/src/timeout/event_test.py b/simple-examples/src/timeout/event_test.py
@@ -0,0 +1,8 @@
+from ..hatchet import hatchet
+
+hatchet.client.event.push(
+    "timeout:create",
+    {
+        "test": "test"
+    }
+)
diff --git a/simple-examples/src/timeout/worker.py b/simple-examples/src/timeout/worker.py
@@ -0,0 +1,15 @@
+from ..hatchet import hatchet
+
+
+@hatchet.workflow(on_events=["timeout:create"])
+class TimeoutWorkflow:
+
+    @hatchet.step(timeout='4s')
+    def timeout(self, context):
+        try:
+            print("started step2")
+            context.sleep(5)
+            print("finished step2")
+        except Exception as e:
+            print("caught an exception: " + str(e))
+            raise e

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+{`
	`2`	`+ "cSpell.words": ["basicrag", "genai"]`
	`3`	`+}`