How to Use CrawlForge with Dify Workflows

openapi: "3.0.0"
info:
  title: CrawlForge Web Scraping Tools
  version: "1.0.0"
  description: "26 specialized web scraping tools for AI applications"
servers:
  - url: https://crawlforge.dev/api/v1/tools
paths:
  /extract_content:
    post:
      operationId: extractContent
      summary: Extract clean content from a URL (2 credits)
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              required: [url]
              properties:
                url:
                  type: string
                  description: The URL to extract content from
      responses:
        "200":
          description: Extracted content
  /search_web:
    post:
      operationId: searchWeb
      summary: Search the web via Google (5 credits)
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              required: [query]
              properties:
                query:
                  type: string
                  description: Search query
                limit:
                  type: integer
                  description: Max results (default 10)
      responses:
        "200":
          description: Search results
  /fetch_url:
    post:
      operationId: fetchUrl
      summary: Fetch raw page content (1 credit)
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              required: [url]
              properties:
                url:
                  type: string
                  description: The URL to fetch
      responses:
        "200":
          description: Raw page content
  /scrape_structured:
    post:
      operationId: scrapeStructured
      summary: Extract data with CSS selectors (2 credits)
      requestBody:
        required: true
        content:
          application/json:
            schema:
              type: object
              required: [url, selectors]
              properties:
                url:
                  type: string
                selectors:
                  type: object
                  additionalProperties:
                    type: string
      responses:
        "200":
          description: Structured extraction results

// Pseudocode for the Dify workflow (implemented visually in Dify's editor)

// Node 1: Start -- User provides a research topic
// Input: { topic: string }

// Node 2: Search Web (5 credits)
// Tool: CrawlForge search_web
// Input: { query: "{{topic}} latest developments 2026", limit: 5 }
// Output: search_results

// Node 3: Extract Top Results (2 credits each)
// Tool: CrawlForge extract_content
// Loop over: search_results.results[0..2]
// Input: { url: "{{item.link}}" }
// Output: extracted_pages[]

// Node 4: LLM Synthesis
// Model: Claude Sonnet
// Prompt: "Synthesize these sources into a research brief: {{extracted_pages}}"
// Output: research_summary

// Node 5: End -- Return research_summary to user
// Total credits: 5 + (3 * 2) = 11 credits per run

// Dify workflow for daily competitor monitoring

// Node 1: Start (triggered by schedule or API call)
// Input: { urls: ["https://competitor1.com/pricing", "https://competitor2.com/pricing"] }

// Node 2: Batch Extract (2 credits per URL)
// Tool: CrawlForge scrape_structured
// Loop over: urls
// Input: {
//   url: "{{item}}",
//   selectors: {
//     plans: ".pricing-plan h3",
//     prices: ".pricing-plan .price",
//     features: ".pricing-plan .feature-list"
//   }
// }
// Output: pricing_data[]

// Node 3: LLM Analysis
// Model: Claude Haiku (for cost efficiency)
// Prompt: "Compare these pricing pages and highlight any changes: {{pricing_data}}"
// Output: analysis

// Node 4: Conditional -- if changes detected, send notification
// Node 5: End
// Total: 2 * number_of_urls credits per run

// Error handling pattern for Dify workflows

// After each CrawlForge tool node, add a conditional:
// If response.status === 402 -> "Insufficient credits"
//   -> Notify user to top up at crawlforge.dev/pricing
// If response.status === 429 -> "Rate limited"
//   -> Wait 2 seconds, retry the node
// If response.status === 500 -> "Server error"
//   -> Log error, skip this URL, continue workflow