feat(composite): native Kitty graphics protocol for inline images

Use vaxis NewImage/Resize/Draw which auto-selects Kitty graphics protocol.
Images decode in background, vaxis handles async Kitty protocol encoding.
Removed halfblock fallback — Kitty only.
This commit is contained in:
Mortdecai
2026-04-07 22:12:06 -04:00
parent 8a19e0b95c
commit 8e663b6e63
+62 -131
View File
@@ -3,7 +3,6 @@ package app
import ( import (
"fmt" "fmt"
"image" "image"
"image/color"
"os" "os"
"strconv" "strconv"
"strings" "strings"
@@ -14,8 +13,6 @@ import (
"git.sr.ht/~rjarry/aerc/lib/parse" "git.sr.ht/~rjarry/aerc/lib/parse"
"git.sr.ht/~rjarry/aerc/lib/ui" "git.sr.ht/~rjarry/aerc/lib/ui"
"golang.org/x/image/draw"
_ "image/jpeg" _ "image/jpeg"
_ "image/png" _ "image/png"
@@ -28,7 +25,6 @@ const (
blockText = 0 blockText = 0
blockImage = 1 blockImage = 1
// Max image height in terminal rows
maxImageHeight = 20 maxImageHeight = 20
) )
@@ -38,12 +34,6 @@ type contentBlock struct {
imageIdx int imageIdx int
} }
// halfBlockRow stores pre-computed cells for one row of a halfblock image.
// Each cell uses upper-half-block "▀" with fg=top pixel, bg=bottom pixel.
type halfBlockRow struct {
cells []vaxis.Cell
}
type compositeContent struct { type compositeContent struct {
blocks []contentBlock blocks []contentBlock
images []parse.ImageRef images []parse.ImageRef
@@ -52,7 +42,7 @@ type compositeContent struct {
mu sync.Mutex mu sync.Mutex
decoded map[int]image.Image decoded map[int]image.Image
decErrs map[int]bool decErrs map[int]bool
rendered map[int][]halfBlockRow // pre-rendered halfblock rows per image graphics map[int]vaxis.Image // Kitty graphics handles
invalidate func() invalidate func()
} }
@@ -115,16 +105,16 @@ func newCompositeContent(text string, images []parse.ImageRef, width int, invali
width: width, width: width,
decoded: make(map[int]image.Image), decoded: make(map[int]image.Image),
decErrs: make(map[int]bool), decErrs: make(map[int]bool),
rendered: make(map[int][]halfBlockRow), graphics: make(map[int]vaxis.Image),
invalidate: invalidate, invalidate: invalidate,
} }
go cc.decodeAndRender() go cc.decodeImages()
return cc return cc
} }
// decodeAndRender decodes images and pre-renders them as halfblock cells. // decodeImages decodes all images in background. Vaxis graphic creation
// All rendering happens here (background goroutine), not in Draw(). // and rendering happen in Draw() on the main thread.
func (cc *compositeContent) decodeAndRender() { func (cc *compositeContent) decodeImages() {
defer log.PanicHandler() defer log.PanicHandler()
for i, img := range cc.images { for i, img := range cc.images {
f, err := os.Open(img.Path) f, err := os.Open(img.Path)
@@ -147,103 +137,12 @@ func (cc *compositeContent) decodeAndRender() {
cc.mu.Lock() cc.mu.Lock()
cc.decoded[i] = decoded cc.decoded[i] = decoded
cc.mu.Unlock() cc.mu.Unlock()
// Pre-render as halfblock cells
rows := renderHalfBlock(decoded, cc.width, maxImageHeight)
cc.mu.Lock()
cc.rendered[i] = rows
cc.mu.Unlock()
} }
if cc.invalidate != nil { if cc.invalidate != nil {
cc.invalidate() cc.invalidate()
} }
} }
// renderHalfBlock converts an image into halfblock character rows.
// Each terminal row represents 2 pixel rows using "▀" (upper half block).
// Top pixel → foreground color, bottom pixel → background color.
func renderHalfBlock(img image.Image, maxWidth, maxHeight int) []halfBlockRow {
bounds := img.Bounds()
srcW := bounds.Dx()
srcH := bounds.Dy()
if srcW == 0 || srcH == 0 {
return nil
}
// Calculate target dimensions in cells
// Each cell = 1 char wide, 2 pixel rows tall
targetW := maxWidth
targetH := maxHeight * 2 // pixel rows (2 per cell row)
// Maintain aspect ratio
ratio := float64(srcH) / float64(srcW)
fitH := int(float64(targetW) * ratio)
if fitH < targetH {
targetH = fitH
}
// Don't upscale
if targetW > srcW {
targetW = srcW
}
if targetH > srcH {
targetH = srcH
}
// Ensure even height for halfblock pairing
if targetH%2 != 0 {
targetH++
}
// Resize the image
dst := image.NewRGBA(image.Rect(0, 0, targetW, targetH))
draw.BiLinear.Scale(dst, dst.Rect, img, bounds, draw.Over, nil)
// Convert to halfblock rows
cellRows := targetH / 2
rows := make([]halfBlockRow, cellRows)
for y := 0; y < cellRows; y++ {
cells := make([]vaxis.Cell, targetW)
for x := 0; x < targetW; x++ {
topR, topG, topB, topA := toRGBA(dst.At(x, y*2))
botR, botG, botB, botA := toRGBA(dst.At(x, y*2+1))
switch {
case topA < 50 && botA < 50:
cells[x] = vaxis.Cell{
Character: vaxis.Character{Grapheme: " ", Width: 1},
}
case topA < 50:
cells[x] = vaxis.Cell{
Character: vaxis.Character{Grapheme: "▄", Width: 1},
Style: vaxis.Style{Foreground: vaxis.RGBColor(botR, botG, botB)},
}
case botA < 50:
cells[x] = vaxis.Cell{
Character: vaxis.Character{Grapheme: "▀", Width: 1},
Style: vaxis.Style{Foreground: vaxis.RGBColor(topR, topG, topB)},
}
default:
cells[x] = vaxis.Cell{
Character: vaxis.Character{Grapheme: "▀", Width: 1},
Style: vaxis.Style{
Foreground: vaxis.RGBColor(topR, topG, topB),
Background: vaxis.RGBColor(botR, botG, botB),
},
}
}
}
rows[y] = halfBlockRow{cells: cells}
}
return rows
}
func toRGBA(c color.Color) (uint8, uint8, uint8, uint8) {
pr, pg, pb, pa := c.RGBA()
if pa == 0 {
return uint8(pr), uint8(pg), uint8(pb), 0
}
return uint8((pr * 255) / pa), uint8((pg * 255) / pa), uint8((pb * 255) / pa), uint8(pa >> 8)
}
func (cc *compositeContent) totalHeight() int { func (cc *compositeContent) totalHeight() int {
h := 0 h := 0
for _, b := range cc.blocks { for _, b := range cc.blocks {
@@ -259,17 +158,33 @@ func (cc *compositeContent) totalHeight() int {
func (cc *compositeContent) imageHeight(idx int) int { func (cc *compositeContent) imageHeight(idx int) int {
cc.mu.Lock() cc.mu.Lock()
rows, ok := cc.rendered[idx] img, ok := cc.decoded[idx]
cc.mu.Unlock() cc.mu.Unlock()
if ok && len(rows) > 0 { if ok {
return len(rows) bounds := img.Bounds()
if bounds.Dx() == 0 {
return maxImageHeight
}
ratio := float64(bounds.Dy()) / float64(bounds.Dx())
h := int(float64(cc.width) * ratio / 2.0)
if h < 3 {
h = 3
}
if h > maxImageHeight {
h = maxImageHeight
}
return h
} }
// Placeholder height before rendered return 1 // placeholder
return 1
} }
func (cc *compositeContent) Destroy() { func (cc *compositeContent) Destroy() {
// No vaxis resources to clean up — halfblock uses only cells cc.mu.Lock()
defer cc.mu.Unlock()
for _, g := range cc.graphics {
g.Destroy()
}
cc.graphics = make(map[int]vaxis.Image)
} }
func (cc *compositeContent) CleanupFiles() { func (cc *compositeContent) CleanupFiles() {
@@ -278,7 +193,7 @@ func (cc *compositeContent) CleanupFiles() {
} }
} }
// Draw renders composite content into the UI context. // Draw renders composite content. Uses Kitty graphics protocol via vaxis.
func (cc *compositeContent) Draw(ctx *ui.Context, scroll int, style ...vaxis.Style) { func (cc *compositeContent) Draw(ctx *ui.Context, scroll int, style ...vaxis.Style) {
width := ctx.Width() width := ctx.Width()
height := ctx.Height() height := ctx.Height()
@@ -305,20 +220,20 @@ func (cc *compositeContent) Draw(ctx *ui.Context, scroll int, style ...vaxis.Sty
case blockImage: case blockImage:
imgH := cc.imageHeight(b.imageIdx) imgH := cc.imageHeight(b.imageIdx)
if row+imgH > 0 && row < height { if row+imgH > 0 && row < height {
cc.drawImage(ctx, b.imageIdx, row, width, defaultStyle) cc.drawImage(ctx, b.imageIdx, row, width, imgH, defaultStyle)
} }
row += imgH row += imgH
} }
} }
} }
func (cc *compositeContent) drawImage(ctx *ui.Context, idx, row, width int, style vaxis.Style) { func (cc *compositeContent) drawImage(ctx *ui.Context, idx, row, width, height int, style vaxis.Style) {
cc.mu.Lock() cc.mu.Lock()
rows, rendered := cc.rendered[idx] img, decoded := cc.decoded[idx]
failed := cc.decErrs[idx] failed := cc.decErrs[idx]
cc.mu.Unlock() cc.mu.Unlock()
if !rendered { if !decoded {
alt := "[loading image...]" alt := "[loading image...]"
if failed { if failed {
alt = "[image]" alt = "[image]"
@@ -332,22 +247,38 @@ func (cc *compositeContent) drawImage(ctx *ui.Context, idx, row, width int, styl
return return
} }
win := ctx.Window() // Create vaxis graphic on first render (uses Kitty protocol)
for i, hbRow := range rows { graphic, ok := cc.graphics[idx]
y := row + i if !ok {
if y < 0 { vx := ctx.Window().Vx
continue var err error
} graphic, err = vx.NewImage(img)
if y >= ctx.Height() { if err != nil {
break log.Errorf("composite: NewImage %d: %v", idx, err)
} if row >= 0 && row < ctx.Height() {
for x, cell := range hbRow.cells { alt := "[image]"
if x >= width { if idx < len(cc.images) && cc.images[idx].Alt != "" {
break alt = fmt.Sprintf("[%s]", cc.images[idx].Alt)
}
ctx.Printf(0, row, style, "%s", alt)
} }
win.SetCell(x, y, cell) // Mark as failed so we don't retry every frame
cc.mu.Lock()
cc.decErrs[idx] = true
delete(cc.decoded, idx)
cc.mu.Unlock()
return
} }
cc.graphics[idx] = graphic
graphic.Resize(width, height)
} }
startRow := row
if startRow < 0 {
startRow = 0
}
win := ctx.Window().New(0, startRow, width, height)
graphic.Draw(win)
} }
func truncateLine(s string, width int) string { func truncateLine(s string, width int) string {