ComfyUI Workflow to API: A Comprehensive Guide in Next.js

ComfyUI is a powerful tool for creating image generation workflows. However, its true potential can be unlocked by converting these workflows into APIs, allowing for dynamic processing of user input and wider application integration. This guide will walk you through the process of transforming your ComfyUI workflow into a functional API using Next.js, a popular React framework for building server-side rendered applications.

Setting Up the API

The first step is to establish a connection with ComfyUI's WebSocket interface. This allows for real-time updates on the workflow's progress. In Next.js, we can use the built-in WebSocket APIs and Node.js modules to achieve this.

// utils/websocket.js
 
import { v4 as uuidv4 } from 'uuid';
 
export function openWebSocketConnection() {
  const serverAddress = '127.0.0.1:8188';
  const clientId = uuidv4();
  const ws = new WebSocket(`ws://${serverAddress}/ws?clientId=${clientId}`);
 
  ws.onopen = () => {
    console.log('WebSocket connection established.');
  };
 
  ws.onclose = () => {
    console.log('WebSocket connection closed.');
  };
 
  return { ws, serverAddress, clientId };
}

Key API Endpoints

Queueing a Prompt

To initiate the workflow, we need to send a prompt to ComfyUI using an HTTP POST request. This can be accomplished using the fetch API.

// utils/api.js
 
export async function queuePrompt(prompt, clientId, serverAddress) {
  const payload = {
    prompt,
    client_id: clientId,
  };
 
  const response = await fetch(`http://${serverAddress}/prompt`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json'
    },
    body: JSON.stringify(payload),
  });
  
  if (!response.ok) {
    throw new Error(`Error queueing prompt: ${response.statusText}`);
  }
 
  return await response.json();
}

Retrieving History

After processing, we can fetch the workflow's history using an HTTP GET request.

// utils/api.js
 
export async function getHistory(promptId, serverAddress) {
  const response = await fetch(`http://${serverAddress}/history/${promptId}`);
  
  if (!response.ok) {
    throw new Error(`Error retrieving history: ${response.statusText}`);
  }
  
  return await response.json();
}

Fetching Generated Images

To retrieve the generated images, we'll need to fetch the image data as a binary array buffer.

// utils/api.js
 
export async function getImage(filename, subfolder, folderType, serverAddress) {
  const params = new URLSearchParams({
    filename,
    subfolder,
    type: folderType,
  });
 
  const response = await fetch(`http://${serverAddress}/view?${params.toString()}`);
 
  if (!response.ok) {
    throw new Error(`Error fetching image: ${response.statusText}`);
  }
 
  return await response.arrayBuffer();
}

Workflow Manipulation

To make your workflow dynamic, you'll need to modify certain aspects of it programmatically. We can use JavaScript's built-in capabilities to process and manipulate the workflow JSON.

// utils/workflow.js
 
import crypto from 'crypto';
 
export function promptToImage(workflow, positivePrompt, negativePrompt = '', savePreviews = false) {
  const prompt = JSON.parse(workflow);
  const idToClassType = {};
 
  for (const [id, details] of Object.entries(prompt)) {
    idToClassType[id] = details.class_type;
  }
 
  const kSampler = Object.keys(idToClassType).find(key => idToClassType[key] === 'KSampler');
 
  // Set a random seed
  prompt[kSampler].inputs.seed = crypto.randomInt(1e14, 1e15);
 
  // Update prompts
  const positiveInputId = prompt[kSampler].inputs.positive[0];
  prompt[positiveInputId].inputs.text = positivePrompt;
 
  if (negativePrompt) {
    const negativeInputId = prompt[kSampler].inputs.negative[0];
    prompt[negativeInputId].inputs.text = negativePrompt;
  }
 
  // Proceed to generate image with the updated prompt
  // generateImageByPrompt(prompt, './output/', savePreviews);
  return prompt; // Return the modified prompt for further processing
}

Tracking Progress

Utilize the WebSocket connection to monitor the workflow's progress. This will allow you to provide real-time updates to the user.

// utils/progress.js
 
export function trackProgress(prompt, ws, promptId) {
  const nodeIds = Object.keys(prompt);
  const finishedNodes = new Set();
 
  ws.onmessage = (event) => {
    const message = JSON.parse(event.data);
 
    if (message.type === 'progress') {
      const data = message.data;
      console.log(`K-Sampler Progress: Step ${data.value} of ${data.max}`);
    } else if (message.type === 'execution_cached' || message.type === 'executing') {
      const data = message.data;
      if (data.node && !finishedNodes.has(data.node)) {
        finishedNodes.add(data.node);
        console.log(`Progress: ${finishedNodes.size}/${nodeIds.length} tasks completed.`);
      }
      if (data.node === null && data.prompt_id === promptId) {
        console.log('Workflow execution completed.');
        ws.close();
      }
    }
  };
}

Handling Generated Images

After the workflow completes, retrieve and save the generated images. We'll use Node.js fs module to save the images to the file system.

// utils/images.js
 
import fs from 'fs';
import path from 'path';
 
export async function getImages(promptId, serverAddress, allowPreview = false) {
  const outputImages = [];
  const history = await getHistory(promptId, serverAddress);
  const promptHistory = history[promptId];
 
  for (const nodeId in promptHistory.outputs) {
    const nodeOutput = promptHistory.outputs[nodeId];
 
    if (nodeOutput.images) {
      for (const image of nodeOutput.images) {
        if ((allowPreview && image.type === 'temp') || image.type === 'output') {
          const imageData = await getImage(image.filename, image.subfolder, image.type, serverAddress);
          outputImages.push({
            imageData: Buffer.from(imageData),
            fileName: image.filename,
            type: image.type
          });
        }
      }
    }
  }
  return outputImages;
}
 
export function saveImages(images, outputPath, savePreviews) {
  images.forEach((item) => {
    const directory = path.join(outputPath, item.type === 'temp' && savePreviews ? 'temp' : '');
    fs.mkdirSync(directory, { recursive: true });
    const filePath = path.join(directory, item.fileName);
    fs.writeFile(filePath, item.imageData, (err) => {
      if (err) {
        console.error(`Failed to save image ${item.fileName}: ${err.message}`);
      } else {
        console.log(`Image ${item.fileName} saved successfully.`);
      }
    });
  });
}

Putting It All Together

Combining all the above functions, we can create an API route in Next.js to handle the entire workflow. Here's an example of how you might set this up.

// pages/api/generate.js
 
import { openWebSocketConnection } from '../../utils/websocket';
import { queuePrompt, getHistory, getImage } from '../../utils/api';
import { promptToImage } from '../../utils/workflow';
import { trackProgress } from '../../utils/progress';
import { getImages, saveImages } from '../../utils/images';
 
export default async function handler(req, res) {
  if (req.method === 'POST') {
    const { positivePrompt, negativePrompt } = req.body;
 
    try {
      const { ws, serverAddress, clientId } = openWebSocketConnection();
 
      // Load your workflow JSON string from a file or database
      const workflow = getYourWorkflowJson();
 
      const modifiedPrompt = promptToImage(workflow, positivePrompt, negativePrompt);
      const promptResponse = await queuePrompt(JSON.stringify(modifiedPrompt), clientId, serverAddress);
      const promptId = promptResponse.prompt_id;
 
      trackProgress(modifiedPrompt, ws, promptId);
 
      ws.onclose = async () => {
        const images = await getImages(promptId, serverAddress, false);
        const outputPath = path.join(process.cwd(), 'public', 'generated-images');
        saveImages(images, outputPath, false);
 
        res.status(200).json({ message: 'Images generated and saved successfully.' });
      };
    } catch (error) {
      console.error('Error during workflow execution:', error);
      res.status(500).json({ error: error.message });
    }
  } else {
    res.setHeader('Allow', ['POST']);
    res.status(405).end(`Method ${req.method} Not Allowed`);
  }
}

Remember to replace getYourWorkflowJson() with the actual way you retrieve your workflow JSON, such as reading from a file or a database.

Or, You Can Use ComfyUIAsAPI.com's Service

If you don't want to deal with the hassle of setting up the API, you can use ComfyUIAsAPI.com's service.

All you need is to upload your ComfyUI workflow .json file and get a ready-to-use API. We also support SDKs for all the popular languages. You just need a few lines of code to integrate it into your project.

Conclusion

By following these steps, you can transform your ComfyUI workflow into a versatile API using Next.js. This approach allows you to integrate your image generation pipeline into various applications seamlessly, enabling dynamic input processing and real-time feedback. Whether you're building a web application, a mobile app, or a complex AI system, this API-centric method provides the flexibility and power to leverage ComfyUI's capabilities in diverse scenarios.