feat: auto-compress clipboard images to avoid API size limits

packages/opencode/package.json

@@ -104,6 +104,7 @@
     "opentui-spinner": "0.0.6",
     "partial-json": "0.1.7",
     "remeda": "catalog:",
+    "sharp": "0.34.5",
     "solid-js": "catalog:",
     "strip-ansi": "7.1.2",
     "tree-sitter-bash": "0.25.0",

packages/opencode/src/cli/cmd/tui/component/prompt/index.tsx

@@ -20,6 +20,7 @@ import { useRenderer } from "@opentui/solid"
 import { Editor } from "@tui/util/editor"
 import { useExit } from "../../context/exit"
 import { Clipboard } from "../../util/clipboard"
+import { Image } from "@/util/image"
 import type { FilePart } from "@opencode-ai/sdk/v2"
 import { TuiEvent } from "../../event"
 import { iife } from "@/util/iife"
@@ -697,6 +698,21 @@ export function Prompt(props: PromptProps) {
   }
 
   async function pasteImage(file: { filename?: string; content: string; mime: string }) {
+    let optimizedContent = file.content
+    let optimizedMime = file.mime
+
+    if (Image.needsCompression(file.content)) {
+      const result = await Image.optimizeForUpload({
+        data: file.content,
+        mime: file.mime,
+      }).catch(() => null)
+
+      if (result) {
+        optimizedContent = result.data
+        optimizedMime = result.mime
+      }
+    }
+
     const currentOffset = input.visualCursor.offset
     const extmarkStart = currentOffset
     const count = store.prompt.parts.filter((x) => x.type === "file").length
@@ -716,9 +732,9 @@ export function Prompt(props: PromptProps) {
 
     const part: Omit<FilePart, "id" | "messageID" | "sessionID"> = {
       type: "file" as const,
-      mime: file.mime,
+      mime: optimizedMime,
       filename: file.filename,
-      url: `data:${file.mime};base64,${file.content}`,
+      url: `data:${optimizedMime};base64,${optimizedContent}`,
       source: {
         type: "file",
         path: file.filename ?? "",

packages/opencode/src/cli/cmd/tui/util/clipboard.ts

@@ -4,13 +4,23 @@ import clipboardy from "clipboardy"
 import { lazy } from "../../../../util/lazy.js"
 import { tmpdir } from "os"
 import path from "path"
+import { Image } from "../../../../util/image.js"
 
 export namespace Clipboard {
   export interface Content {
     data: string
     mime: string
   }
 
+  async function optimizeImage(data: string, mime: string): Promise<Content> {
+    if (!Image.needsCompression(data)) {
+      return { data, mime }
+    }
+
+    const result = await Image.optimizeForUpload({ data, mime }).catch(() => ({ data, mime, compressed: false }))
+    return { data: result.data, mime: result.mime }
+  }
+
   export async function read(): Promise<Content | undefined> {
     const os = platform()
 
@@ -22,7 +32,8 @@ export namespace Clipboard {
           .quiet()
         const file = Bun.file(tmpfile)
         const buffer = await file.arrayBuffer()
-        return { data: Buffer.from(buffer).toString("base64"), mime: "image/png" }
+        const base64 = Buffer.from(buffer).toString("base64")
+        return optimizeImage(base64, "image/png")
       } catch {
       } finally {
         await $`rm -f "${tmpfile}"`.nothrow().quiet()
@@ -36,19 +47,19 @@ export namespace Clipboard {
       if (base64) {
         const imageBuffer = Buffer.from(base64.trim(), "base64")
         if (imageBuffer.length > 0) {
-          return { data: imageBuffer.toString("base64"), mime: "image/png" }
+          return optimizeImage(imageBuffer.toString("base64"), "image/png")
         }
       }
     }
 
     if (os === "linux") {
       const wayland = await $`wl-paste -t image/png`.nothrow().arrayBuffer()
       if (wayland && wayland.byteLength > 0) {
-        return { data: Buffer.from(wayland).toString("base64"), mime: "image/png" }
+        return optimizeImage(Buffer.from(wayland).toString("base64"), "image/png")
       }
       const x11 = await $`xclip -selection clipboard -t image/png -o`.nothrow().arrayBuffer()
       if (x11 && x11.byteLength > 0) {
-        return { data: Buffer.from(x11).toString("base64"), mime: "image/png" }
+        return optimizeImage(Buffer.from(x11).toString("base64"), "image/png")
       }
     }
 

packages/opencode/src/util/image.ts

@@ -0,0 +1,207 @@
+import sharp from "sharp"
+
+const DEFAULT_MAX_BYTES = 4 * 1024 * 1024
+const DEFAULT_QUALITY = 85
+const MAX_DIMENSION = 2048
+
+export namespace Image {
+  export interface CompressOptions {
+    data: string
+    mime: string
+    maxBytes?: number
+    quality?: number
+    allowFormatChange?: boolean
+  }
+
+  export interface CompressResult {
+    data: string
+    mime: string
+    compressed: boolean
+    originalSize: number
+    finalSize: number
+  }
+
+  export interface ImageInfo {
+    width: number
+    height: number
+    format: string
+    hasAlpha: boolean
+  }
+
+  export interface ResizeOptions {
+    data: string
+    maxWidth: number
+    maxHeight: number
+  }
+
+  export interface ResizeResult {
+    data: string
+    width: number
+    height: number
+  }
+
+  export interface OptimizeOptions {
+    data: string
+    mime: string
+    targetBytes?: number
+  }
+
+  export function needsCompression(base64: string, thresholdBytes = DEFAULT_MAX_BYTES): boolean {
+    const sizeBytes = Math.ceil((base64.length * 3) / 4)
+    return sizeBytes > thresholdBytes
+  }
+
+  export async function getInfo(base64: string): Promise<ImageInfo> {
+    const buffer = Buffer.from(base64, "base64")
+    const metadata = await sharp(buffer).metadata()
+
+    return {
+      width: metadata.width ?? 0,
+      height: metadata.height ?? 0,
+      format: metadata.format ?? "unknown",
+      hasAlpha: metadata.hasAlpha ?? false,
+    }
+  }
+
+  export async function resize(options: ResizeOptions): Promise<ResizeResult> {
+    const buffer = Buffer.from(options.data, "base64")
+    const metadata = await sharp(buffer).metadata()
+
+    const currentWidth = metadata.width ?? 0
+    const currentHeight = metadata.height ?? 0
+
+    if (currentWidth <= options.maxWidth && currentHeight <= options.maxHeight) {
+      return {
+        data: options.data,
+        width: currentWidth,
+        height: currentHeight,
+      }
+    }
+
+    const resized = await sharp(buffer)
+      .resize(options.maxWidth, options.maxHeight, {
+        fit: "inside",
+        withoutEnlargement: true,
+      })
+      .toBuffer()
+
+    const newMetadata = await sharp(resized).metadata()
+
+    return {
+      data: resized.toString("base64"),
+      width: newMetadata.width ?? 0,
+      height: newMetadata.height ?? 0,
+    }
+  }
+
+  export function estimateCompressedSize(
+    originalSize: number,
+    format: "jpeg" | "webp" | "png",
+    quality: number,
+  ): number {
+    const qualityFactor = quality / 100
+    switch (format) {
+      case "jpeg":
+        return Math.ceil(originalSize * qualityFactor * 0.3)
+      case "webp":
+        return Math.ceil(originalSize * qualityFactor * 0.25)
+      case "png":
+        return Math.ceil(originalSize * 0.7)
+    }
+  }
+
+  export async function compress(options: CompressOptions): Promise<CompressResult> {
+    if (!options.data) {
+      throw new Error("Image data is required")
+    }
+
+    const buffer = Buffer.from(options.data, "base64")
+    const originalSize = buffer.length
+    const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES
+    const quality = options.quality ?? DEFAULT_QUALITY
+
+    if (originalSize <= maxBytes) {
+      return {
+        data: options.data,
+        mime: options.mime,
+        compressed: false,
+        originalSize,
+        finalSize: originalSize,
+      }
+    }
+
+    const metadata = await sharp(buffer).metadata()
+    const hasAlpha = metadata.hasAlpha ?? false
+    const currentWidth = metadata.width ?? 0
+    const currentHeight = metadata.height ?? 0
+
+    let pipeline = sharp(buffer)
+
+    if (currentWidth > MAX_DIMENSION || currentHeight > MAX_DIMENSION) {
+      pipeline = pipeline.resize(MAX_DIMENSION, MAX_DIMENSION, {
+        fit: "inside",
+        withoutEnlargement: true,
+      })
+    }
+
+    let outputFormat: "jpeg" | "webp" | "png" = "jpeg"
+    let outputMime = "image/jpeg"
+
+    if (hasAlpha) {
+      outputFormat = "webp"
+      outputMime = "image/webp"
+    } else if (options.allowFormatChange !== false) {
+      outputFormat = "jpeg"
+      outputMime = "image/jpeg"
+    } else {
+      outputFormat = "webp"
+      outputMime = "image/webp"
+    }
+
+    let currentQuality = quality
+    let result: Buffer
+
+    while (currentQuality >= 20) {
+      if (outputFormat === "jpeg") {
+        result = await pipeline.clone().jpeg({ quality: currentQuality, mozjpeg: true }).toBuffer()
+      } else if (outputFormat === "webp") {
+        result = await pipeline.clone().webp({ quality: currentQuality }).toBuffer()
+      } else {
+        result = await pipeline.clone().png({ compressionLevel: 9 }).toBuffer()
+      }
+
+      if (result.length <= maxBytes) {
+        return {
+          data: result.toString("base64"),
+          mime: outputMime,
+          compressed: true,
+          originalSize,
+          finalSize: result.length,
+        }
+      }
+
+      currentQuality -= 10
+    }
+
+    result = await pipeline.clone().webp({ quality: 20 }).toBuffer()
+
+    return {
+      data: result.toString("base64"),
+      mime: "image/webp",
+      compressed: true,
+      originalSize,
+      finalSize: result.length,
+    }
+  }
+
+  export async function optimizeForUpload(options: OptimizeOptions): Promise<CompressResult> {
+    const targetBytes = options.targetBytes ?? DEFAULT_MAX_BYTES
+
+    return compress({
+      data: options.data,
+      mime: options.mime,
+      maxBytes: targetBytes,
+      allowFormatChange: true,
+    })
+  }
+}