init: pristine aerc 0.20.0 source

worker/imap/cache.go

@@ -0,0 +1,212 @@
+package imap
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"git.sr.ht/~rjarry/aerc/lib/log"
+	"git.sr.ht/~rjarry/aerc/lib/parse"
+	"git.sr.ht/~rjarry/aerc/lib/xdg"
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+	"github.com/emersion/go-message"
+	"github.com/emersion/go-message/mail"
+	"github.com/emersion/go-message/textproto"
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+type CachedHeader struct {
+	BodyStructure models.BodyStructure
+	Envelope      models.Envelope
+	InternalDate  time.Time
+	Uid           models.UID
+	Size          uint32
+	Header        []byte
+	Created       time.Time
+}
+
+var (
+	// cacheTag should be updated when changing the cache
+	// structure; this will ensure that the user's cache is cleared and
+	// reloaded when the underlying cache structure changes
+	cacheTag    = []byte("0003")
+	cacheTagKey = []byte("cache.tag")
+)
+
+// initCacheDb opens (or creates) the database for the cache. One database is
+// created per account
+func (w *IMAPWorker) initCacheDb(acct string) {
+	switch {
+	case len(w.config.headersExclude) > 0:
+		headerTag := strings.Join(w.config.headersExclude, "")
+		cacheTag = append(cacheTag, headerTag...)
+	case len(w.config.headers) > 0:
+		headerTag := strings.Join(w.config.headers, "")
+		cacheTag = append(cacheTag, headerTag...)
+	}
+	p := xdg.CachePath("aerc", acct)
+	db, err := leveldb.OpenFile(p, nil)
+	if err != nil {
+		w.cache = nil
+		w.worker.Errorf("failed opening cache db: %v", err)
+		return
+	}
+	w.cache = db
+	w.worker.Debugf("cache db opened: %s", p)
+
+	tag, err := w.cache.Get(cacheTagKey, nil)
+	clearCache := errors.Is(err, leveldb.ErrNotFound) ||
+		!bytes.Equal(tag, cacheTag)
+	switch {
+	case clearCache:
+		w.worker.Infof("current cache tag is '%s' but found '%s'",
+			cacheTag, tag)
+		w.worker.Warnf("tag mismatch: clear cache")
+		w.clearCache()
+		if err = w.cache.Put(cacheTagKey, cacheTag, nil); err != nil {
+			w.worker.Errorf("could not set the current cache tag")
+		}
+	case err != nil:
+		w.worker.Errorf("could not get the cache tag from db")
+	default:
+		if w.config.cacheMaxAge.Hours() > 0 {
+			go w.cleanCache(p)
+		}
+	}
+}
+
+func (w *IMAPWorker) cacheHeader(mi *models.MessageInfo) {
+	key := w.headerKey(mi.Uid)
+	w.worker.Debugf("caching header for message %s", key)
+	hdr := bytes.NewBuffer(nil)
+	err := textproto.WriteHeader(hdr, mi.RFC822Headers.Header.Header)
+	if err != nil {
+		w.worker.Errorf("cannot write header %s: %v", key, err)
+		return
+	}
+	h := &CachedHeader{
+		BodyStructure: *mi.BodyStructure,
+		Envelope:      *mi.Envelope,
+		InternalDate:  mi.InternalDate,
+		Uid:           mi.Uid,
+		Size:          mi.Size,
+		Header:        hdr.Bytes(),
+		Created:       time.Now(),
+	}
+	data := bytes.NewBuffer(nil)
+	enc := gob.NewEncoder(data)
+	err = enc.Encode(h)
+	if err != nil {
+		w.worker.Errorf("cannot encode message %s: %v", key, err)
+		return
+	}
+	err = w.cache.Put(key, data.Bytes(), nil)
+	if err != nil {
+		w.worker.Errorf("cannot write header for message %s: %v", key, err)
+		return
+	}
+}
+
+func (w *IMAPWorker) getCachedHeaders(msg *types.FetchMessageHeaders) []models.UID {
+	w.worker.Tracef("Retrieving headers from cache: %v", msg.Uids)
+	var need []models.UID
+	for _, uid := range msg.Uids {
+		key := w.headerKey(uid)
+		data, err := w.cache.Get(key, nil)
+		if err != nil {
+			need = append(need, uid)
+			continue
+		}
+		ch := &CachedHeader{}
+		dec := gob.NewDecoder(bytes.NewReader(data))
+		err = dec.Decode(ch)
+		if err != nil {
+			w.worker.Errorf("cannot decode cached header %s: %v", key, err)
+			need = append(need, uid)
+			continue
+		}
+		hr := bytes.NewReader(ch.Header)
+		textprotoHeader, err := textproto.ReadHeader(bufio.NewReader(hr))
+		if err != nil {
+			w.worker.Errorf("cannot read cached header %s: %v", key, err)
+			need = append(need, uid)
+			continue
+		}
+
+		hdr := &mail.Header{Header: message.Header{Header: textprotoHeader}}
+		mi := &models.MessageInfo{
+			BodyStructure: &ch.BodyStructure,
+			Envelope:      &ch.Envelope,
+			Flags:         models.SeenFlag, // Always return a SEEN flag
+			Uid:           ch.Uid,
+			RFC822Headers: hdr,
+			Refs:          parse.MsgIDList(hdr, "references"),
+			Size:          ch.Size,
+		}
+		w.worker.PostMessage(&types.MessageInfo{
+			Message:    types.RespondTo(msg),
+			Info:       mi,
+			NeedsFlags: true,
+		}, nil)
+	}
+	return need
+}
+
+func (w *IMAPWorker) headerKey(uid models.UID) []byte {
+	key := fmt.Sprintf("header.%s.%d.%s",
+		w.selected.Name, w.selected.UidValidity, uid)
+	return []byte(key)
+}
+
+// cleanCache removes stale entries from the selected mailbox cachedb
+func (w *IMAPWorker) cleanCache(path string) {
+	defer log.PanicHandler()
+	start := time.Now()
+	var scanned, removed int
+	iter := w.cache.NewIterator(nil, nil)
+	for iter.Next() {
+		if bytes.Equal(iter.Key(), cacheTagKey) {
+			continue
+		}
+		data := iter.Value()
+		ch := &CachedHeader{}
+		dec := gob.NewDecoder(bytes.NewReader(data))
+		err := dec.Decode(ch)
+		if err != nil {
+			w.worker.Errorf("cannot clean database %d: %v",
+				w.selected.UidValidity, err)
+			continue
+		}
+		exp := ch.Created.Add(w.config.cacheMaxAge)
+		if exp.Before(time.Now()) {
+			err = w.cache.Delete(iter.Key(), nil)
+			if err != nil {
+				w.worker.Errorf("cannot clean database %d: %v",
+					w.selected.UidValidity, err)
+				continue
+			}
+			removed++
+		}
+		scanned++
+	}
+	iter.Release()
+	elapsed := time.Since(start)
+	w.worker.Debugf("%s: removed %d/%d expired entries in %s",
+		path, removed, scanned, elapsed)
+}
+
+// clearCache clears the entire cache
+func (w *IMAPWorker) clearCache() {
+	iter := w.cache.NewIterator(nil, nil)
+	for iter.Next() {
+		if err := w.cache.Delete(iter.Key(), nil); err != nil {
+			w.worker.Errorf("error clearing cache: %v", err)
+		}
+	}
+	iter.Release()
+}

worker/imap/checkmail.go

@@ -0,0 +1,76 @@
+package imap
+
+import (
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+	"github.com/emersion/go-imap"
+)
+
+func (w *IMAPWorker) handleCheckMailMessage(msg *types.CheckMail) {
+	items := []imap.StatusItem{
+		imap.StatusMessages,
+		imap.StatusRecent,
+		imap.StatusUnseen,
+		imap.StatusUidNext,
+	}
+	var (
+		statuses  []*imap.MailboxStatus
+		err       error
+		remaining []string
+	)
+	switch {
+	case w.liststatus:
+		w.worker.Tracef("Checking mail with LIST-STATUS")
+		statuses, err = w.client.liststatus.ListStatus("", "*", items, nil)
+		if err != nil {
+			w.worker.PostMessage(&types.Error{
+				Message: types.RespondTo(msg),
+				Error:   err,
+			}, nil)
+			return
+		}
+	default:
+		for _, dir := range msg.Directories {
+			if len(w.worker.Actions()) > 0 {
+				remaining = append(remaining, dir)
+				continue
+			}
+			w.worker.Tracef("Getting status of directory %s", dir)
+			status, err := w.client.Status(dir, items)
+			if err != nil {
+				w.worker.PostMessage(&types.Error{
+					Message: types.RespondTo(msg),
+					Error:   err,
+				}, nil)
+				continue
+			}
+			statuses = append(statuses, status)
+		}
+	}
+	for _, status := range statuses {
+		refetch := false
+		if status.Name == w.selected.Name {
+			if status.UidNext != w.selected.UidNext {
+				refetch = true
+			}
+			w.selected = status
+		}
+		w.worker.PostMessage(&types.DirectoryInfo{
+			Info: &models.DirectoryInfo{
+				Name:   status.Name,
+				Exists: int(status.Messages),
+				Recent: int(status.Recent),
+				Unseen: int(status.Unseen),
+			},
+			Refetch: refetch,
+		}, nil)
+	}
+	if len(remaining) > 0 {
+		w.worker.PostMessage(&types.CheckMailDirectories{
+			Message:     types.RespondTo(msg),
+			Directories: remaining,
+		}, nil)
+		return
+	}
+	w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+}

worker/imap/configure.go

@@ -0,0 +1,187 @@
+package imap
+
+import (
+	"bufio"
+	"fmt"
+	"net/url"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.sr.ht/~rjarry/aerc/lib/xdg"
+	"git.sr.ht/~rjarry/aerc/worker/lib"
+	"git.sr.ht/~rjarry/aerc/worker/middleware"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+	"golang.org/x/oauth2"
+)
+
+func (w *IMAPWorker) handleConfigure(msg *types.Configure) error {
+	w.config.name = msg.Config.Name
+	u, err := url.Parse(msg.Config.Source)
+	if err != nil {
+		return err
+	}
+
+	w.config.scheme = u.Scheme
+	if strings.HasSuffix(w.config.scheme, "+insecure") {
+		w.config.scheme = strings.TrimSuffix(w.config.scheme, "+insecure")
+		w.config.insecure = true
+	}
+
+	if strings.HasSuffix(w.config.scheme, "+oauthbearer") {
+		w.config.scheme = strings.TrimSuffix(w.config.scheme, "+oauthbearer")
+		w.config.oauthBearer.Enabled = true
+		q := u.Query()
+
+		oauth2 := &oauth2.Config{}
+		if q.Get("token_endpoint") != "" {
+			oauth2.ClientID = q.Get("client_id")
+			oauth2.ClientSecret = q.Get("client_secret")
+			oauth2.Scopes = []string{q.Get("scope")}
+			oauth2.Endpoint.TokenURL = q.Get("token_endpoint")
+		}
+		w.config.oauthBearer.OAuth2 = oauth2
+	}
+
+	if strings.HasSuffix(w.config.scheme, "+xoauth2") {
+		w.config.scheme = strings.TrimSuffix(w.config.scheme, "+xoauth2")
+		w.config.xoauth2.Enabled = true
+		q := u.Query()
+
+		oauth2 := &oauth2.Config{}
+		if q.Get("token_endpoint") != "" {
+			oauth2.ClientID = q.Get("client_id")
+			oauth2.ClientSecret = q.Get("client_secret")
+			oauth2.Scopes = []string{q.Get("scope")}
+			oauth2.Endpoint.TokenURL = q.Get("token_endpoint")
+		}
+		w.config.xoauth2.OAuth2 = oauth2
+	}
+
+	w.config.addr = u.Host
+	if !strings.ContainsRune(w.config.addr, ':') {
+		w.config.addr += ":" + w.config.scheme
+	}
+
+	w.config.user = u.User
+	w.config.folders = msg.Config.Folders
+	w.config.headers = msg.Config.Headers
+	w.config.headersExclude = msg.Config.HeadersExclude
+
+	w.config.idle_timeout = 10 * time.Second
+	w.config.idle_debounce = 10 * time.Millisecond
+
+	w.config.connection_timeout = 30 * time.Second
+	w.config.keepalive_period = 0 * time.Second
+	w.config.keepalive_probes = 3
+	w.config.keepalive_interval = 3
+
+	w.config.reconnect_maxwait = 30 * time.Second
+
+	w.config.cacheEnabled = false
+	w.config.cacheMaxAge = 30 * 24 * time.Hour // 30 days
+
+	for key, value := range msg.Config.Params {
+		switch key {
+		case "idle-timeout":
+			val, err := time.ParseDuration(value)
+			if err != nil || val < 0 {
+				return fmt.Errorf(
+					"invalid idle-timeout value %v: %w",
+					value, err)
+			}
+			w.config.idle_timeout = val
+		case "idle-debounce":
+			val, err := time.ParseDuration(value)
+			if err != nil || val < 0 {
+				return fmt.Errorf(
+					"invalid idle-debounce value %v: %w",
+					value, err)
+			}
+			w.config.idle_debounce = val
+		case "reconnect-maxwait":
+			val, err := time.ParseDuration(value)
+			if err != nil || val < 0 {
+				return fmt.Errorf(
+					"invalid reconnect-maxwait value %v: %w",
+					value, err)
+			}
+			w.config.reconnect_maxwait = val
+		case "connection-timeout":
+			val, err := time.ParseDuration(value)
+			if err != nil || val < 0 {
+				return fmt.Errorf(
+					"invalid connection-timeout value %v: %w",
+					value, err)
+			}
+			w.config.connection_timeout = val
+		case "keepalive-period":
+			val, err := time.ParseDuration(value)
+			if err != nil || val < 0 {
+				return fmt.Errorf(
+					"invalid keepalive-period value %v: %w",
+					value, err)
+			}
+			w.config.keepalive_period = val
+		case "keepalive-probes":
+			val, err := strconv.Atoi(value)
+			if err != nil || val < 0 {
+				return fmt.Errorf(
+					"invalid keepalive-probes value %v: %w",
+					value, err)
+			}
+			w.config.keepalive_probes = val
+		case "keepalive-interval":
+			val, err := time.ParseDuration(value)
+			if err != nil || val < 0 {
+				return fmt.Errorf(
+					"invalid keepalive-interval value %v: %w",
+					value, err)
+			}
+			w.config.keepalive_interval = int(val.Seconds())
+		case "cache-headers":
+			cache, err := strconv.ParseBool(value)
+			if err != nil {
+				// Return an error here because the user tried to set header
+				// caching, and we want them to know they didn't set it right -
+				// one way or the other
+				return fmt.Errorf("invalid cache-headers value %v: %w", value, err)
+			}
+			w.config.cacheEnabled = cache
+		case "cache-max-age":
+			val, err := time.ParseDuration(value)
+			if err != nil || val < 0 {
+				return fmt.Errorf("invalid cache-max-age value %v: %w", value, err)
+			}
+			w.config.cacheMaxAge = val
+		case "use-gmail-ext":
+			val, err := strconv.ParseBool(value)
+			if err != nil {
+				return fmt.Errorf("invalid use-gmail-ext value %v: %w", value, err)
+			}
+			w.config.useXGMEXT = val
+		}
+	}
+	if w.config.cacheEnabled {
+		w.initCacheDb(msg.Config.Name)
+	}
+	w.idler = newIdler(w.config, w.worker, w.executeIdle)
+	w.observer = newObserver(w.config, w.worker)
+
+	if name, ok := msg.Config.Params["folder-map"]; ok {
+		file := xdg.ExpandHome(name)
+		f, err := os.Open(file)
+		if err != nil {
+			return err
+		}
+		defer f.Close()
+		fmap, order, err := lib.ParseFolderMap(bufio.NewReader(f))
+		if err != nil {
+			return err
+		}
+		w.worker = middleware.NewFolderMapper(w.worker, fmap, order)
+	}
+
+	return nil
+}

worker/imap/connect.go

@@ -0,0 +1,196 @@
+package imap
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"time"
+
+	"git.sr.ht/~rjarry/aerc/lib"
+	"git.sr.ht/~rjarry/aerc/lib/log"
+	"github.com/emersion/go-imap"
+	"github.com/emersion/go-imap/client"
+	"github.com/emersion/go-sasl"
+)
+
+// connect establishes a new tcp connection to the imap server, logs in and
+// selects the default inbox. If no error is returned, the imap client will be
+// in the imap.SelectedState.
+func (w *IMAPWorker) connect() (*client.Client, error) {
+	var (
+		conn *net.TCPConn
+		err  error
+		c    *client.Client
+	)
+
+	conn, err = newTCPConn(w.config.addr, w.config.connection_timeout)
+	if conn == nil || err != nil {
+		return nil, err
+	}
+
+	if w.config.connection_timeout > 0 {
+		end := time.Now().Add(w.config.connection_timeout)
+		err = conn.SetDeadline(end)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if w.config.keepalive_period > 0 {
+		err = w.setKeepaliveParameters(conn)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	serverName, _, _ := net.SplitHostPort(w.config.addr)
+	tlsConfig := &tls.Config{ServerName: serverName}
+
+	switch w.config.scheme {
+	case "imap":
+		c, err = client.New(conn)
+		if err != nil {
+			return nil, err
+		}
+		if !w.config.insecure {
+			if err = c.StartTLS(tlsConfig); err != nil {
+				return nil, err
+			}
+		}
+	case "imaps":
+		if w.config.insecure {
+			tlsConfig.InsecureSkipVerify = true
+		}
+		tlsConn := tls.Client(conn, tlsConfig)
+		c, err = client.New(tlsConn)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("Unknown IMAP scheme %s", w.config.scheme)
+	}
+
+	c.ErrorLog = log.ErrorLogger()
+
+	if w.config.user != nil {
+		username := w.config.user.Username()
+
+		// TODO: 2nd parameter false if no password is set. ask for it
+		// if unset.
+		password, _ := w.config.user.Password()
+
+		if w.config.oauthBearer.Enabled {
+			if err := w.config.oauthBearer.Authenticate(
+				username, password, c); err != nil {
+				return nil, err
+			}
+		} else if w.config.xoauth2.Enabled {
+			if err := w.config.xoauth2.Authenticate(
+				username, password, w.config.name, c); err != nil {
+				return nil, err
+			}
+		} else if plain, err := c.SupportAuth("PLAIN"); err != nil {
+			return nil, err
+		} else if plain {
+			auth := sasl.NewPlainClient("", username, password)
+			if err := c.Authenticate(auth); err != nil {
+				return nil, err
+			}
+		} else if err := c.Login(username, password); err != nil {
+			return nil, err
+		}
+	}
+
+	if _, err := c.Select(imap.InboxName, false); err != nil {
+		return nil, err
+	}
+
+	info := make(chan *imap.MailboxInfo, 1)
+	if err := c.List("", "", info); err != nil {
+		return nil, fmt.Errorf("failed to retrieve delimiter: %w", err)
+	}
+	if mailboxinfo := <-info; mailboxinfo != nil {
+		w.delimiter = mailboxinfo.Delimiter
+	}
+	if w.delimiter == "" {
+		// just in case some implementation does not follow standards
+		w.delimiter = "/"
+	}
+
+	return c, nil
+}
+
+// newTCPConn establishes a new tcp connection. Timeout will ensure that the
+// function does not hang when there is no connection. If there is a timeout,
+// but a valid connection is eventually returned, ensure that it is properly
+// closed.
+func newTCPConn(addr string, timeout time.Duration) (*net.TCPConn, error) {
+	errTCPTimeout := fmt.Errorf("tcp connection timeout")
+
+	type tcpConn struct {
+		conn *net.TCPConn
+		err  error
+	}
+
+	done := make(chan tcpConn)
+	go func() {
+		defer log.PanicHandler()
+
+		newConn, err := net.Dial("tcp", addr)
+		if err != nil {
+			done <- tcpConn{nil, err}
+			return
+		}
+		done <- tcpConn{newConn.(*net.TCPConn), nil}
+	}()
+
+	select {
+	case <-time.After(timeout):
+		go func() {
+			defer log.PanicHandler()
+			if tcpResult := <-done; tcpResult.conn != nil {
+				tcpResult.conn.Close()
+			}
+		}()
+		return nil, errTCPTimeout
+	case tcpResult := <-done:
+		if tcpResult.conn == nil || tcpResult.err != nil {
+			return nil, tcpResult.err
+		}
+		return tcpResult.conn, nil
+	}
+}
+
+// Set additional keepalive parameters.
+// Uses new interfaces introduced in Go1.11, which let us get connection's file
+// descriptor, without blocking, and therefore without uncontrolled spawning of
+// threads (not goroutines, actual threads).
+func (w *IMAPWorker) setKeepaliveParameters(conn *net.TCPConn) error {
+	err := conn.SetKeepAlive(true)
+	if err != nil {
+		return err
+	}
+	// Idle time before sending a keepalive probe
+	err = conn.SetKeepAlivePeriod(w.config.keepalive_period)
+	if err != nil {
+		return err
+	}
+	rawConn, e := conn.SyscallConn()
+	if e != nil {
+		return e
+	}
+	err = rawConn.Control(func(fdPtr uintptr) {
+		fd := int(fdPtr)
+		// Max number of probes before failure
+		err := lib.SetTcpKeepaliveProbes(fd, w.config.keepalive_probes)
+		if err != nil {
+			w.worker.Errorf("cannot set tcp keepalive probes: %v", err)
+		}
+		// Wait time after an unsuccessful probe
+		err = lib.SetTcpKeepaliveInterval(fd, w.config.keepalive_interval)
+		if err != nil {
+			w.worker.Errorf("cannot set tcp keepalive interval: %v", err)
+		}
+	})
+	return err
+}

worker/imap/create.go

@@ -0,0 +1,19 @@
+package imap
+
+import (
+	"git.sr.ht/~rjarry/aerc/worker/types"
+)
+
+func (imapw *IMAPWorker) handleCreateDirectory(msg *types.CreateDirectory) {
+	if err := imapw.client.Create(msg.Directory); err != nil {
+		if msg.Quiet {
+			return
+		}
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+	} else {
+		imapw.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+	}
+}

worker/imap/extensions/liststatus.go

@@ -0,0 +1,149 @@
+package extensions
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/emersion/go-imap"
+	"github.com/emersion/go-imap/client"
+	"github.com/emersion/go-imap/responses"
+	"github.com/emersion/go-imap/utf7"
+)
+
+// A LIST-STATUS client
+type ListStatusClient struct {
+	c *client.Client
+}
+
+func NewListStatusClient(c *client.Client) *ListStatusClient {
+	return &ListStatusClient{c}
+}
+
+// SupportListStatus checks if the server supports the LIST-STATUS extension.
+func (c *ListStatusClient) SupportListStatus() (bool, error) {
+	return c.c.Support("LIST-STATUS")
+}
+
+// ListStatus performs a LIST-STATUS command, listing mailboxes and also
+// retrieving the requested status items. A nil channel can be passed in order
+// to only retrieve the STATUS responses
+func (c *ListStatusClient) ListStatus(
+	ref string,
+	name string,
+	items []imap.StatusItem,
+	ch chan *imap.MailboxInfo,
+) ([]*imap.MailboxStatus, error) {
+	if ch != nil {
+		defer close(ch)
+	}
+
+	if c.c.State() != imap.AuthenticatedState && c.c.State() != imap.SelectedState {
+		return nil, client.ErrNotLoggedIn
+	}
+
+	cmd := &ListStatusCommand{
+		Reference: ref,
+		Mailbox:   name,
+		Items:     items,
+	}
+	res := &ListStatusResponse{Mailboxes: ch}
+
+	status, err := c.c.Execute(cmd, res)
+	if err != nil {
+		return nil, err
+	}
+	return res.Statuses, status.Err()
+}
+
+// ListStatusCommand is a LIST command, as defined in RFC 3501 section 6.3.8. If
+// Subscribed is set to true, LSUB will be used instead. Mailbox statuses will
+// be returned if Items is not nil
+type ListStatusCommand struct {
+	Reference string
+	Mailbox   string
+
+	Subscribed bool
+	Items      []imap.StatusItem
+}
+
+func (cmd *ListStatusCommand) Command() *imap.Command {
+	name := "LIST"
+	if cmd.Subscribed {
+		name = "LSUB"
+	}
+
+	enc := utf7.Encoding.NewEncoder()
+	ref, _ := enc.String(cmd.Reference)
+	mailbox, _ := enc.String(cmd.Mailbox)
+
+	items := make([]string, len(cmd.Items))
+	if cmd.Items != nil {
+		for i, item := range cmd.Items {
+			items[i] = string(item)
+		}
+	}
+
+	args := fmt.Sprintf("RETURN (STATUS (%s))", strings.Join(items, " "))
+	return &imap.Command{
+		Name:      name,
+		Arguments: []interface{}{ref, mailbox, imap.RawString(args)},
+	}
+}
+
+// A LIST-STATUS response
+type ListStatusResponse struct {
+	Mailboxes  chan *imap.MailboxInfo
+	Subscribed bool
+	Statuses   []*imap.MailboxStatus
+}
+
+func (r *ListStatusResponse) Name() string {
+	if r.Subscribed {
+		return "LSUB"
+	} else {
+		return "LIST"
+	}
+}
+
+func (r *ListStatusResponse) Handle(resp imap.Resp) error {
+	name, _, ok := imap.ParseNamedResp(resp)
+	if !ok {
+		return responses.ErrUnhandled
+	}
+	switch name {
+	case "LIST":
+		if r.Mailboxes == nil {
+			return nil
+		}
+		res := responses.List{Mailboxes: r.Mailboxes}
+		return res.Handle(resp)
+	case "STATUS":
+		res := responses.Status{
+			Mailbox: new(imap.MailboxStatus),
+		}
+		err := res.Handle(resp)
+		if err != nil {
+			return err
+		}
+		r.Statuses = append(r.Statuses, res.Mailbox)
+	default:
+		return responses.ErrUnhandled
+	}
+
+	return nil
+}
+
+func (r *ListStatusResponse) WriteTo(w *imap.Writer) error {
+	respName := r.Name()
+
+	for mbox := range r.Mailboxes {
+		fields := []interface{}{imap.RawString(respName)}
+		fields = append(fields, mbox.Format()...)
+
+		resp := imap.NewUntaggedResp(fields)
+		if err := resp.WriteTo(w); err != nil {
+			return err
+		}
+	}
+	return nil
+}

worker/imap/extensions/xgmext/client.go

@@ -0,0 +1,101 @@
+package xgmext
+
+import (
+	"errors"
+	"fmt"
+
+	"git.sr.ht/~rjarry/aerc/lib/log"
+	"git.sr.ht/~rjarry/aerc/models"
+	"github.com/emersion/go-imap"
+	"github.com/emersion/go-imap/client"
+	"github.com/emersion/go-imap/commands"
+	"github.com/emersion/go-imap/responses"
+)
+
+type handler struct {
+	client *client.Client
+}
+
+func NewHandler(c *client.Client) *handler {
+	return &handler{client: c}
+}
+
+func (h handler) FetchEntireThreads(requested []models.UID) ([]models.UID, error) {
+	threadIds, err := h.fetchThreadIds(requested)
+	if err != nil {
+		return nil,
+			fmt.Errorf("failed to fetch thread IDs: %w", err)
+	}
+	uids, err := h.searchUids(threadIds)
+	if err != nil {
+		return nil,
+			fmt.Errorf("failed to search for thread IDs: %w", err)
+	}
+	return uids, nil
+}
+
+func (h handler) fetchThreadIds(uids []models.UID) ([]string, error) {
+	messages := make(chan *imap.Message)
+	done := make(chan error)
+
+	thriditem := imap.FetchItem("X-GM-THRID")
+	items := []imap.FetchItem{
+		thriditem,
+	}
+
+	m := make(map[string]struct{}, len(uids))
+	go func() {
+		defer log.PanicHandler()
+		for msg := range messages {
+			if msg == nil {
+				continue
+			}
+			item, ok := msg.Items[thriditem].(string)
+			if ok {
+				m[item] = struct{}{}
+			}
+		}
+		done <- nil
+	}()
+
+	var set imap.SeqSet
+	for _, uid := range uids {
+		set.AddNum(models.UidToUint32(uid))
+	}
+	err := h.client.UidFetch(&set, items, messages)
+	<-done
+
+	thrid := make([]string, 0, len(m))
+	for id := range m {
+		thrid = append(thrid, id)
+	}
+	return thrid, err
+}
+
+func (h handler) searchUids(thrid []string) ([]models.UID, error) {
+	if len(thrid) == 0 {
+		return nil, errors.New("no thread IDs provided")
+	}
+	return h.runSearch(NewThreadIDSearch(thrid))
+}
+
+func (h handler) RawSearch(rawSearch string) ([]models.UID, error) {
+	return h.runSearch(NewRawSearch(rawSearch))
+}
+
+func (h handler) runSearch(cmd imap.Commander) ([]models.UID, error) {
+	if h.client.State() != imap.SelectedState {
+		return nil, errors.New("no mailbox selected")
+	}
+	cmd = &commands.Uid{Cmd: cmd}
+	res := new(responses.Search)
+	status, err := h.client.Execute(cmd, res)
+	if err != nil {
+		return nil, fmt.Errorf("imap execute failed: %w", err)
+	}
+	var uids []models.UID
+	for _, i := range res.Ids {
+		uids = append(uids, models.Uint32ToUid(i))
+	}
+	return uids, status.Err()
+}

worker/imap/extensions/xgmext/search.go

@@ -0,0 +1,74 @@
+package xgmext
+
+import "github.com/emersion/go-imap"
+
+type threadIDSearch struct {
+	Charset   string
+	ThreadIDs []string
+}
+
+// NewThreadIDSearch return an imap.Command to search UIDs for the provided
+// thread IDs using the X-GM-EXT-1 (Gmail extension)
+func NewThreadIDSearch(threadIDs []string) *threadIDSearch {
+	return &threadIDSearch{
+		Charset:   "UTF-8",
+		ThreadIDs: threadIDs,
+	}
+}
+
+func (cmd *threadIDSearch) Command() *imap.Command {
+	const threadSearchKey = "X-GM-THRID"
+
+	var args []interface{}
+	if cmd.Charset != "" {
+		args = append(args, imap.RawString("CHARSET"))
+		args = append(args, imap.RawString(cmd.Charset))
+	}
+
+	// we want to produce a search query that looks like this:
+	// SEARCH CHARSET UTF-8 OR OR X-GM-THRID 1771431779961568536 \
+	// X-GM-THRID 1765355745646219617 X-GM-THRID 1771500774375286796
+	for i := 0; i < len(cmd.ThreadIDs)-1; i++ {
+		args = append(args, imap.RawString("OR"))
+	}
+
+	for _, thrid := range cmd.ThreadIDs {
+		args = append(args, imap.RawString(threadSearchKey))
+		args = append(args, imap.RawString(thrid))
+	}
+
+	return &imap.Command{
+		Name:      "SEARCH",
+		Arguments: args,
+	}
+}
+
+type rawSearch struct {
+	Charset string
+	Search  string
+}
+
+func NewRawSearch(search string) *rawSearch {
+	return &rawSearch{
+		Charset: "UTF-8",
+		Search:  search,
+	}
+}
+
+func (cmd *rawSearch) Command() *imap.Command {
+	const key = "X-GM-RAW"
+
+	var args []interface{}
+	if cmd.Charset != "" {
+		args = append(args, imap.RawString("CHARSET"))
+		args = append(args, imap.RawString(cmd.Charset))
+	}
+
+	args = append(args, imap.RawString(key))
+	args = append(args, imap.RawString(cmd.Search))
+
+	return &imap.Command{
+		Name:      "SEARCH",
+		Arguments: args,
+	}
+}

worker/imap/extensions/xgmext/search_test.go

@@ -0,0 +1,76 @@
+package xgmext_test
+
+import (
+	"bytes"
+	"testing"
+
+	"git.sr.ht/~rjarry/aerc/worker/imap/extensions/xgmext"
+	"github.com/emersion/go-imap"
+)
+
+func TestXGMEXT_ThreadIDSearch(t *testing.T) {
+	tests := []struct {
+		name string
+		ids  []string
+		want string
+	}{
+		{
+			name: "search for single id",
+			ids:  []string{"1234"},
+			want: "* SEARCH CHARSET UTF-8 X-GM-THRID 1234\r\n",
+		},
+		{
+			name: "search for multiple id",
+			ids:  []string{"1234", "5678", "2345"},
+			want: "* SEARCH CHARSET UTF-8 OR OR X-GM-THRID 1234 X-GM-THRID 5678 X-GM-THRID 2345\r\n",
+		},
+	}
+	for _, test := range tests {
+		cmd := xgmext.NewThreadIDSearch(test.ids).Command()
+		var buf bytes.Buffer
+		err := cmd.WriteTo(imap.NewWriter(&buf))
+		if err != nil {
+			t.Errorf("failed to write command: %v", err)
+		}
+		if got := buf.String(); got != test.want {
+			t.Errorf("test '%s' failed: got: '%s', but wanted: '%s'",
+				test.name, got, test.want)
+		}
+	}
+}
+
+func TestXGMEXT_RawSearch(t *testing.T) {
+	tests := []struct {
+		name   string
+		search string
+		want   string
+	}{
+		{
+			name:   "search messages from mailing list",
+			search: "list:info@example.com",
+			want:   "* SEARCH CHARSET UTF-8 X-GM-RAW list:info@example.com\r\n",
+		},
+		{
+			name:   "search for an exact phrase",
+			search: "\"good morning\"",
+			want:   "* SEARCH CHARSET UTF-8 X-GM-RAW \"good morning\"\r\n",
+		},
+		{
+			name:   "group multiple search terms together",
+			search: "subject:(dinner movie)",
+			want:   "* SEARCH CHARSET UTF-8 X-GM-RAW subject:(dinner movie)\r\n",
+		},
+	}
+	for _, test := range tests {
+		cmd := xgmext.NewRawSearch(test.search).Command()
+		var buf bytes.Buffer
+		err := cmd.WriteTo(imap.NewWriter(&buf))
+		if err != nil {
+			t.Errorf("failed to write command: %v", err)
+		}
+		if got := buf.String(); got != test.want {
+			t.Errorf("test '%s' failed: got: '%s', but wanted: '%s'",
+				test.name, got, test.want)
+		}
+	}
+}

worker/imap/extensions/xgmext/terms.go

@@ -0,0 +1,46 @@
+package xgmext
+
+var Terms = []string{
+	"from:",
+	"to:",
+	"cc:",
+	"bcc:",
+	"subject:",
+	"label:",
+	"deliveredto:",
+	"category:primary",
+	"category:social",
+	"category:promotions",
+	"category:updates",
+	"category:forums",
+	"category:reservations",
+	"category:purchases",
+	"has:",
+	"has:attachment",
+	"has:drive",
+	"has:document",
+	"has:spreadsheet",
+	"has:presentation",
+	"has:youtube",
+	"list:",
+	"filename:",
+	"in:",
+	"is:",
+	"is:important",
+	"is:read",
+	"is:unread",
+	"is:starred",
+	"after:",
+	"before:",
+	"older:",
+	"newer:",
+	"older_than:",
+	"newer_than:",
+	"size:",
+	"larger:",
+	"smaller:",
+	"rfc822msgid:",
+	"OR",
+	"AND",
+	"AROUND",
+}

worker/imap/fetch.go

@@ -0,0 +1,292 @@
+package imap
+
+import (
+	"bufio"
+	"fmt"
+
+	"github.com/emersion/go-imap"
+	"github.com/emersion/go-message"
+	_ "github.com/emersion/go-message/charset"
+	"github.com/emersion/go-message/mail"
+	"github.com/emersion/go-message/textproto"
+
+	"git.sr.ht/~rjarry/aerc/lib/log"
+	"git.sr.ht/~rjarry/aerc/lib/parse"
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+)
+
+func (imapw *IMAPWorker) handleFetchMessageHeaders(
+	msg *types.FetchMessageHeaders,
+) {
+	if msg.Context.Err() != nil {
+		imapw.worker.PostMessage(&types.Cancelled{
+			Message: types.RespondTo(msg),
+		}, nil)
+		return
+	}
+	toFetch := msg.Uids
+	if imapw.config.cacheEnabled && imapw.cache != nil {
+		toFetch = imapw.getCachedHeaders(msg)
+	}
+	if len(toFetch) == 0 {
+		imapw.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)},
+			nil)
+		return
+	}
+	imapw.worker.Tracef("Fetching message headers: %v", toFetch)
+	hdrBodyPart := imap.BodyPartName{
+		Specifier: imap.HeaderSpecifier,
+	}
+	switch {
+	case len(imapw.config.headersExclude) > 0:
+		hdrBodyPart.NotFields = true
+		hdrBodyPart.Fields = imapw.config.headersExclude
+	case len(imapw.config.headers) > 0:
+		hdrBodyPart.Fields = imapw.config.headers
+	}
+	section := &imap.BodySectionName{
+		BodyPartName: hdrBodyPart,
+		Peek:         true,
+	}
+
+	items := []imap.FetchItem{
+		imap.FetchBodyStructure,
+		imap.FetchEnvelope,
+		imap.FetchInternalDate,
+		imap.FetchFlags,
+		imap.FetchUid,
+		imap.FetchRFC822Size,
+		section.FetchItem(),
+	}
+	imapw.handleFetchMessages(msg, toFetch, items,
+		func(_msg *imap.Message) error {
+			if len(_msg.Body) == 0 {
+				// ignore duplicate messages with only flag updates
+				return nil
+			}
+			reader := _msg.GetBody(section)
+			if reader == nil {
+				return fmt.Errorf("failed to find part: %v", section)
+			}
+			textprotoHeader, err := textproto.ReadHeader(bufio.NewReader(reader))
+			if err != nil {
+				return fmt.Errorf("failed to read part header: %w", err)
+			}
+			header := &mail.Header{Header: message.Header{Header: textprotoHeader}}
+			info := &models.MessageInfo{
+				BodyStructure: translateBodyStructure(_msg.BodyStructure),
+				Envelope:      translateEnvelope(_msg.Envelope),
+				Flags:         translateImapFlags(_msg.Flags),
+				InternalDate:  _msg.InternalDate,
+				RFC822Headers: header,
+				Refs:          parse.MsgIDList(header, "references"),
+				Size:          _msg.Size,
+				Uid:           models.Uint32ToUid(_msg.Uid),
+			}
+			imapw.worker.PostMessage(&types.MessageInfo{
+				Message: types.RespondTo(msg),
+				Info:    info,
+			}, nil)
+			if imapw.config.cacheEnabled && imapw.cache != nil {
+				imapw.cacheHeader(info)
+			}
+			return nil
+		})
+}
+
+func (imapw *IMAPWorker) handleFetchMessageBodyPart(
+	msg *types.FetchMessageBodyPart,
+) {
+	imapw.worker.Tracef("Fetching message %d part: %v", msg.Uid, msg.Part)
+
+	var partHeaderSection imap.BodySectionName
+	partHeaderSection.Peek = true
+	if len(msg.Part) > 0 {
+		partHeaderSection.Specifier = imap.MIMESpecifier
+	} else {
+		partHeaderSection.Specifier = imap.HeaderSpecifier
+	}
+	partHeaderSection.Path = msg.Part
+
+	var partBodySection imap.BodySectionName
+	if len(msg.Part) > 0 {
+		partBodySection.Specifier = imap.EntireSpecifier
+	} else {
+		partBodySection.Specifier = imap.TextSpecifier
+	}
+	partBodySection.Path = msg.Part
+	partBodySection.Peek = true
+
+	items := []imap.FetchItem{
+		imap.FetchEnvelope,
+		imap.FetchUid,
+		imap.FetchBodyStructure,
+		imap.FetchFlags,
+		partHeaderSection.FetchItem(),
+		partBodySection.FetchItem(),
+	}
+	imapw.handleFetchMessages(msg, []models.UID{msg.Uid}, items,
+		func(_msg *imap.Message) error {
+			if len(_msg.Body) == 0 {
+				// ignore duplicate messages with only flag updates
+				return nil
+			}
+			body := _msg.GetBody(&partHeaderSection)
+			if body == nil {
+				return fmt.Errorf("failed to find part: %v", partHeaderSection)
+			}
+			h, err := textproto.ReadHeader(bufio.NewReader(body))
+			if err != nil {
+				return fmt.Errorf("failed to read part header: %w", err)
+			}
+
+			part, err := message.New(message.Header{Header: h},
+				_msg.GetBody(&partBodySection))
+			if message.IsUnknownCharset(err) {
+				imapw.worker.Warnf("unknown charset encountered "+
+					"for uid %d", _msg.Uid)
+			} else if err != nil {
+				return fmt.Errorf("failed to create message reader: %w", err)
+			}
+
+			imapw.worker.PostMessage(&types.MessageBodyPart{
+				Message: types.RespondTo(msg),
+				Part: &models.MessageBodyPart{
+					Reader: part.Body,
+					Uid:    models.Uint32ToUid(_msg.Uid),
+				},
+			}, nil)
+			// Update flags (to mark message as read)
+			imapw.worker.PostMessage(&types.MessageInfo{
+				Message: types.RespondTo(msg),
+				Info: &models.MessageInfo{
+					Flags: translateImapFlags(_msg.Flags),
+					Uid:   models.Uint32ToUid(_msg.Uid),
+				},
+			}, nil)
+			return nil
+		})
+}
+
+func (imapw *IMAPWorker) handleFetchFullMessages(
+	msg *types.FetchFullMessages,
+) {
+	imapw.worker.Tracef("Fetching full messages: %v", msg.Uids)
+	section := &imap.BodySectionName{
+		Peek: true,
+	}
+	items := []imap.FetchItem{
+		imap.FetchEnvelope,
+		imap.FetchFlags,
+		imap.FetchUid,
+		section.FetchItem(),
+	}
+	imapw.handleFetchMessages(msg, msg.Uids, items,
+		func(_msg *imap.Message) error {
+			if len(_msg.Body) == 0 {
+				// ignore duplicate messages with only flag updates
+				return nil
+			}
+			r := _msg.GetBody(section)
+			if r == nil {
+				return fmt.Errorf("could not get section %#v", section)
+			}
+			imapw.worker.PostMessage(&types.FullMessage{
+				Message: types.RespondTo(msg),
+				Content: &models.FullMessage{
+					Reader: bufio.NewReader(r),
+					Uid:    models.Uint32ToUid(_msg.Uid),
+				},
+			}, nil)
+			// Update flags (to mark message as read)
+			imapw.worker.PostMessage(&types.MessageInfo{
+				Message: types.RespondTo(msg),
+				Info: &models.MessageInfo{
+					Flags: translateImapFlags(_msg.Flags),
+					Uid:   models.Uint32ToUid(_msg.Uid),
+				},
+			}, nil)
+			return nil
+		})
+}
+
+func (imapw *IMAPWorker) handleFetchMessageFlags(msg *types.FetchMessageFlags) {
+	items := []imap.FetchItem{
+		imap.FetchFlags,
+		imap.FetchUid,
+	}
+	if msg.Context.Err() != nil {
+		imapw.worker.PostMessage(&types.Cancelled{
+			Message: types.RespondTo(msg),
+		}, nil)
+		return
+	}
+	imapw.handleFetchMessages(msg, msg.Uids, items,
+		func(_msg *imap.Message) error {
+			imapw.worker.PostMessage(&types.MessageInfo{
+				Message: types.RespondTo(msg),
+				Info: &models.MessageInfo{
+					Flags: translateImapFlags(_msg.Flags),
+					Uid:   models.Uint32ToUid(_msg.Uid),
+				},
+			}, nil)
+			return nil
+		})
+}
+
+func (imapw *IMAPWorker) handleFetchMessages(
+	msg types.WorkerMessage, uids []models.UID, items []imap.FetchItem,
+	procFunc func(*imap.Message) error,
+) {
+	messages := make(chan *imap.Message)
+	done := make(chan struct{})
+
+	missingUids := make(map[models.UID]bool)
+	for _, uid := range uids {
+		missingUids[uid] = true
+	}
+
+	go func() {
+		defer log.PanicHandler()
+
+		for _msg := range messages {
+			delete(missingUids, models.Uint32ToUid(_msg.Uid))
+			err := procFunc(_msg)
+			if err != nil {
+				log.Errorf("failed to process message <%d>: %v", _msg.Uid, err)
+				imapw.worker.PostMessage(&types.MessageInfo{
+					Message: types.RespondTo(msg),
+					Info: &models.MessageInfo{
+						Uid:   models.Uint32ToUid(_msg.Uid),
+						Error: err,
+					},
+				}, nil)
+			}
+		}
+		close(done)
+	}()
+
+	set := toSeqSet(uids)
+	if err := imapw.client.UidFetch(set, items, messages); err != nil {
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+		return
+	}
+	<-done
+
+	for uid := range missingUids {
+		imapw.worker.PostMessage(&types.MessageInfo{
+			Message: types.RespondTo(msg),
+			Info: &models.MessageInfo{
+				Uid:   uid,
+				Error: fmt.Errorf("invalid response from server (detailed error in log)"),
+			},
+		}, nil)
+	}
+
+	imapw.worker.PostMessage(
+		&types.Done{Message: types.RespondTo(msg)}, nil)
+}

worker/imap/flags.go

@@ -0,0 +1,156 @@
+package imap
+
+import (
+	"github.com/emersion/go-imap"
+	"github.com/emersion/go-imap/client"
+
+	"git.sr.ht/~rjarry/aerc/lib/log"
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+)
+
+// drainUpdates will drain the updates channel. For some operations, the imap
+// server will send unilateral messages. If they arrive while another operation
+// is in progress, the buffered updates channel can fill up and cause a freeze
+// of the entire backend. Avoid this by draining the updates channel and only
+// process the Message and Expunge updates.
+//
+// To stop the draining, close the returned struct.
+func (imapw *IMAPWorker) drainUpdates() *drainCloser {
+	done := make(chan struct{})
+	go func() {
+		defer log.PanicHandler()
+		for {
+			select {
+			case update := <-imapw.updates:
+				switch update.(type) {
+				case *client.MessageUpdate,
+					*client.ExpungeUpdate:
+					imapw.handleImapUpdate(update)
+				}
+			case <-done:
+				return
+			}
+		}
+	}()
+	return &drainCloser{done}
+}
+
+type drainCloser struct {
+	done chan struct{}
+}
+
+func (d *drainCloser) Close() error {
+	close(d.done)
+	return nil
+}
+
+func (imapw *IMAPWorker) handleDeleteMessages(msg *types.DeleteMessages) {
+	drain := imapw.drainUpdates()
+	defer drain.Close()
+
+	item := imap.FormatFlagsOp(imap.AddFlags, true)
+	flags := []interface{}{imap.DeletedFlag}
+	uids := toSeqSet(msg.Uids)
+	if err := imapw.client.UidStore(uids, item, flags, nil); err != nil {
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+		return
+	}
+	if err := imapw.client.Expunge(nil); err != nil {
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+	} else {
+		imapw.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+	}
+}
+
+func (imapw *IMAPWorker) handleAnsweredMessages(msg *types.AnsweredMessages) {
+	item := imap.FormatFlagsOp(imap.AddFlags, false)
+	flags := []interface{}{imap.AnsweredFlag}
+	if !msg.Answered {
+		item = imap.FormatFlagsOp(imap.RemoveFlags, false)
+	}
+	imapw.handleStoreOps(msg, msg.Uids, item, flags,
+		func(_msg *imap.Message) error {
+			imapw.worker.PostMessage(&types.MessageInfo{
+				Message: types.RespondTo(msg),
+				Info: &models.MessageInfo{
+					Flags: translateImapFlags(_msg.Flags),
+					Uid:   models.Uint32ToUid(_msg.Uid),
+				},
+			}, nil)
+			return nil
+		})
+}
+
+func (imapw *IMAPWorker) handleFlagMessages(msg *types.FlagMessages) {
+	flags := []interface{}{flagToImap[msg.Flags]}
+	item := imap.FormatFlagsOp(imap.AddFlags, false)
+	if !msg.Enable {
+		item = imap.FormatFlagsOp(imap.RemoveFlags, false)
+	}
+	imapw.handleStoreOps(msg, msg.Uids, item, flags,
+		func(_msg *imap.Message) error {
+			imapw.worker.PostMessage(&types.MessageInfo{
+				Message: types.RespondTo(msg),
+				Info: &models.MessageInfo{
+					Flags: translateImapFlags(_msg.Flags),
+					Uid:   models.Uint32ToUid(_msg.Uid),
+				},
+			}, nil)
+			return nil
+		})
+}
+
+func (imapw *IMAPWorker) handleStoreOps(
+	msg types.WorkerMessage, uids []models.UID, item imap.StoreItem, flag interface{},
+	procFunc func(*imap.Message) error,
+) {
+	messages := make(chan *imap.Message)
+	done := make(chan error)
+
+	go func() {
+		defer log.PanicHandler()
+
+		var reterr error
+		for _msg := range messages {
+			err := procFunc(_msg)
+			if err != nil {
+				if reterr == nil {
+					reterr = err
+				}
+				// drain the channel upon error
+				for range messages {
+				}
+			}
+		}
+		done <- reterr
+	}()
+
+	emitErr := func(err error) {
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+	}
+
+	set := toSeqSet(uids)
+	if err := imapw.client.UidStore(set, item, flag, messages); err != nil {
+		emitErr(err)
+		return
+	}
+	if err := <-done; err != nil {
+		emitErr(err)
+		return
+	}
+	imapw.worker.PostAction(&types.CheckMail{
+		Directories: []string{imapw.selected.Name},
+	}, nil)
+	imapw.worker.PostMessage(
+		&types.Done{Message: types.RespondTo(msg)}, nil)
+}

worker/imap/idler.go

@@ -0,0 +1,132 @@
+package imap
+
+import (
+	"fmt"
+	"time"
+
+	"git.sr.ht/~rjarry/aerc/lib/log"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+	"github.com/emersion/go-imap"
+)
+
+var errIdleTimeout = fmt.Errorf("idle timeout")
+
+// idler manages the idle mode of the imap server. Enter idle mode if there's
+// no other task and leave idle mode when a new task arrives. Idle mode is only
+// used when the client is ready and connected. After a connection loss, make
+// sure that idling returns gracefully and the worker remains responsive.
+type idler struct {
+	client    *imapClient
+	debouncer *time.Timer
+	debounce  time.Duration
+	timeout   time.Duration
+	worker    types.WorkerInteractor
+	stop      chan struct{}
+	start     chan struct{}
+	done      chan error
+}
+
+func newIdler(cfg imapConfig, w types.WorkerInteractor, startIdler chan struct{}) *idler {
+	return &idler{
+		debouncer: nil,
+		debounce:  cfg.idle_debounce,
+		timeout:   cfg.idle_timeout,
+		worker:    w,
+		stop:      make(chan struct{}),
+		start:     startIdler,
+		done:      make(chan error),
+	}
+}
+
+func (i *idler) SetClient(c *imapClient) {
+	i.client = c
+}
+
+func (i *idler) ready() bool {
+	return (i.client != nil && i.client.State() == imap.SelectedState)
+}
+
+func (i *idler) Start() {
+	if !i.ready() {
+		return
+	}
+
+	select {
+	case <-i.stop:
+		// stop channel is nil (probably after a debounce), we don't
+		// want to close it
+	default:
+		close(i.stop)
+	}
+
+	// create new stop channel
+	i.stop = make(chan struct{})
+
+	// clear done channel
+	clearing := true
+	for clearing {
+		select {
+		case <-i.done:
+			continue
+		default:
+			clearing = false
+		}
+	}
+
+	i.worker.Tracef("idler (start): start idle after debounce")
+	i.debouncer = time.AfterFunc(i.debounce, func() {
+		i.start <- struct{}{}
+		i.worker.Tracef("idler (start): started")
+	})
+}
+
+func (i *idler) Execute() {
+	if !i.ready() {
+		return
+	}
+
+	// we need to call client.Idle in a goroutine since it is blocking call
+	// and we still want to receive messages
+	go func() {
+		defer log.PanicHandler()
+
+		start := time.Now()
+		err := i.client.Idle(i.stop, nil)
+		if err != nil {
+			i.worker.Errorf("idle returned error: %v", err)
+		}
+		i.worker.Tracef("idler (execute): idleing for %s", time.Since(start))
+
+		i.done <- err
+	}()
+}
+
+func (i *idler) Stop() error {
+	if !i.ready() {
+		return nil
+	}
+
+	select {
+	case <-i.stop:
+		i.worker.Debugf("idler (stop): idler already stopped?")
+		return nil
+	default:
+		close(i.stop)
+	}
+
+	if i.debouncer != nil {
+		if i.debouncer.Stop() {
+			i.worker.Tracef("idler (stop): debounced")
+			return nil
+		}
+	}
+
+	select {
+	case err := <-i.done:
+		i.worker.Tracef("idler (stop): idle stopped: %v", err)
+		return err
+	case <-time.After(i.timeout):
+		i.worker.Errorf("idler (stop): cannot stop idle (timeout)")
+		return errIdleTimeout
+	}
+}

worker/imap/imap.go

@@ -0,0 +1,119 @@
+package imap
+
+import (
+	"strings"
+
+	"github.com/emersion/go-imap"
+
+	"git.sr.ht/~rjarry/aerc/models"
+	"github.com/emersion/go-message/charset"
+	"github.com/emersion/go-message/mail"
+)
+
+func init() {
+	imap.CharsetReader = charset.Reader
+}
+
+func toSeqSet(uids []models.UID) *imap.SeqSet {
+	set := new(imap.SeqSet)
+	for _, uid := range uids {
+		set.AddNum(models.UidToUint32(uid))
+	}
+	return set
+}
+
+func translateBodyStructure(bs *imap.BodyStructure) *models.BodyStructure {
+	if bs == nil {
+		return nil
+	}
+	var parts []*models.BodyStructure
+	for _, part := range bs.Parts {
+		parts = append(parts, translateBodyStructure(part))
+	}
+
+	// TODO: is that all?
+
+	return &models.BodyStructure{
+		MIMEType:          bs.MIMEType,
+		MIMESubType:       bs.MIMESubType,
+		Params:            bs.Params,
+		Description:       bs.Description,
+		Encoding:          bs.Encoding,
+		Parts:             parts,
+		Disposition:       bs.Disposition,
+		DispositionParams: bs.DispositionParams,
+	}
+}
+
+func translateEnvelope(e *imap.Envelope) *models.Envelope {
+	if e == nil {
+		return nil
+	}
+
+	return &models.Envelope{
+		Date:      e.Date,
+		Subject:   e.Subject,
+		From:      translateAddresses(e.From),
+		ReplyTo:   translateAddresses(e.ReplyTo),
+		To:        translateAddresses(e.To),
+		Cc:        translateAddresses(e.Cc),
+		Bcc:       translateAddresses(e.Bcc),
+		MessageId: translateMessageID(e.MessageId),
+		InReplyTo: translateMessageID(e.InReplyTo),
+	}
+}
+
+func translateMessageID(messageID string) string {
+	// Strip away unwanted characters, go-message expects the message id
+	// without brackets, spaces, tabs and new lines.
+	return strings.Trim(messageID, "<> \t\r\n")
+}
+
+func translateAddresses(addrs []*imap.Address) []*mail.Address {
+	var converted []*mail.Address
+	for _, addr := range addrs {
+		converted = append(converted, &mail.Address{
+			Name:    addr.PersonalName,
+			Address: addr.Address(),
+		})
+	}
+	return converted
+}
+
+var imapToFlag = map[string]models.Flags{
+	imap.SeenFlag:     models.SeenFlag,
+	imap.RecentFlag:   models.RecentFlag,
+	imap.AnsweredFlag: models.AnsweredFlag,
+	imap.DeletedFlag:  models.DeletedFlag,
+	imap.FlaggedFlag:  models.FlaggedFlag,
+	imap.DraftFlag:    models.DraftFlag,
+}
+
+var flagToImap = map[models.Flags]string{
+	models.SeenFlag:     imap.SeenFlag,
+	models.RecentFlag:   imap.RecentFlag,
+	models.AnsweredFlag: imap.AnsweredFlag,
+	models.DeletedFlag:  imap.DeletedFlag,
+	models.FlaggedFlag:  imap.FlaggedFlag,
+	models.DraftFlag:    imap.DraftFlag,
+}
+
+func translateImapFlags(imapFlags []string) models.Flags {
+	var flags models.Flags
+	for _, imapFlag := range imapFlags {
+		if flag, ok := imapToFlag[imapFlag]; ok {
+			flags |= flag
+		}
+	}
+	return flags
+}
+
+func translateFlags(flags models.Flags) []string {
+	var imapFlags []string
+	for flag, imapFlag := range flagToImap {
+		if flags.Has(flag) {
+			imapFlags = append(imapFlags, imapFlag)
+		}
+	}
+	return imapFlags
+}

worker/imap/imap_test.go

@@ -0,0 +1,51 @@
+package imap
+
+import (
+	"testing"
+	"time"
+
+	"git.sr.ht/~rjarry/aerc/models"
+	"github.com/emersion/go-message/mail"
+
+	"github.com/emersion/go-imap"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTranslateEnvelope(t *testing.T) {
+	date, _ := time.Parse("2010-01-31", "1992-10-24")
+	givenAddress := imap.Address{
+		PersonalName: "PERSONAL_NAME",
+		AtDomainList: "AT_DOMAIN_LIST",
+		MailboxName:  "MAILBOX_NAME",
+		HostName:     "HOST_NAME",
+	}
+	givenMessageID := " \r\n\r  \t <initial-message-id@with-leading-space>\t\r"
+	given := imap.Envelope{
+		Date:      date,
+		Subject:   "Test Subject",
+		From:      []*imap.Address{&givenAddress},
+		ReplyTo:   []*imap.Address{&givenAddress},
+		To:        []*imap.Address{&givenAddress},
+		Cc:        []*imap.Address{&givenAddress},
+		Bcc:       []*imap.Address{&givenAddress},
+		MessageId: givenMessageID,
+		InReplyTo: givenMessageID,
+	}
+	expectedMessageID := "initial-message-id@with-leading-space"
+	expectedAddress := mail.Address{
+		Name:    "PERSONAL_NAME",
+		Address: "MAILBOX_NAME@HOST_NAME",
+	}
+	expected := models.Envelope{
+		Date:      date,
+		Subject:   "Test Subject",
+		From:      []*mail.Address{&expectedAddress},
+		ReplyTo:   []*mail.Address{&expectedAddress},
+		To:        []*mail.Address{&expectedAddress},
+		Cc:        []*mail.Address{&expectedAddress},
+		Bcc:       []*mail.Address{&expectedAddress},
+		MessageId: expectedMessageID,
+		InReplyTo: expectedMessageID,
+	}
+	assert.Equal(t, &expected, translateEnvelope(&given))
+}

worker/imap/list.go

@@ -0,0 +1,144 @@
+package imap
+
+import (
+	"strings"
+
+	"github.com/emersion/go-imap"
+
+	"git.sr.ht/~rjarry/aerc/lib/log"
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+)
+
+func (imapw *IMAPWorker) handleListDirectories(msg *types.ListDirectories) {
+	mailboxes := make(chan *imap.MailboxInfo)
+	imapw.worker.Tracef("Listing mailboxes")
+	done := make(chan interface{})
+
+	go func() {
+		defer log.PanicHandler()
+
+		for mbox := range mailboxes {
+			if !canOpen(mbox) {
+				// no need to pass this to handlers if it can't be opened
+				continue
+			}
+			dir := &models.Directory{
+				Name: mbox.Name,
+			}
+			for _, attr := range mbox.Attributes {
+				attr = strings.TrimPrefix(attr, "\\")
+				attr = strings.ToLower(attr)
+				role, ok := models.Roles[attr]
+				if !ok {
+					continue
+				}
+				dir.Role = role
+			}
+			if mbox.Name == "INBOX" {
+				dir.Role = models.InboxRole
+			}
+			imapw.worker.PostMessage(&types.Directory{
+				Message: types.RespondTo(msg),
+				Dir:     dir,
+			}, nil)
+		}
+		done <- nil
+	}()
+
+	switch {
+	case imapw.liststatus:
+		items := []imap.StatusItem{
+			imap.StatusMessages,
+			imap.StatusRecent,
+			imap.StatusUnseen,
+		}
+		statuses, err := imapw.client.liststatus.ListStatus(
+			"",
+			"*",
+			items,
+			mailboxes,
+		)
+		if err != nil {
+			<-done
+			imapw.worker.PostMessage(&types.Error{
+				Message: types.RespondTo(msg),
+				Error:   err,
+			}, nil)
+			return
+
+		}
+		for _, status := range statuses {
+			imapw.worker.PostMessage(&types.DirectoryInfo{
+				Info: &models.DirectoryInfo{
+					Name:   status.Name,
+					Exists: int(status.Messages),
+					Recent: int(status.Recent),
+					Unseen: int(status.Unseen),
+				},
+			}, nil)
+		}
+	default:
+		err := imapw.client.List("", "*", mailboxes)
+		if err != nil {
+			<-done
+			imapw.worker.PostMessage(&types.Error{
+				Message: types.RespondTo(msg),
+				Error:   err,
+			}, nil)
+			return
+		}
+	}
+	<-done
+	imapw.worker.PostMessage(
+		&types.Done{Message: types.RespondTo(msg)}, nil)
+}
+
+const NonExistentAttr = "\\NonExistent"
+
+func canOpen(mbox *imap.MailboxInfo) bool {
+	for _, attr := range mbox.Attributes {
+		if attr == imap.NoSelectAttr ||
+			attr == NonExistentAttr {
+			return false
+		}
+	}
+	return true
+}
+
+func (imapw *IMAPWorker) handleSearchDirectory(msg *types.SearchDirectory) {
+	emitError := func(err error) {
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+	}
+
+	imapw.worker.Tracef("Executing search")
+	criteria := translateSearch(msg.Criteria)
+
+	if msg.Context.Err() != nil {
+		imapw.worker.PostMessage(&types.Cancelled{
+			Message: types.RespondTo(msg),
+		}, nil)
+		return
+	}
+
+	uids, err := imapw.client.UidSearch(criteria)
+	if err != nil {
+		emitError(err)
+		return
+	}
+
+	if msg.Context.Err() != nil {
+		imapw.worker.PostMessage(&types.Cancelled{
+			Message: types.RespondTo(msg),
+		}, nil)
+		return
+	}
+
+	imapw.worker.PostMessage(&types.SearchResults{
+		Message: types.RespondTo(msg),
+		Uids:    models.Uint32ToUidList(uids),
+	}, nil)
+}

worker/imap/movecopy.go

@@ -0,0 +1,68 @@
+package imap
+
+import (
+	"io"
+
+	"git.sr.ht/~rjarry/aerc/worker/types"
+)
+
+func (imapw *IMAPWorker) handleCopyMessages(msg *types.CopyMessages) {
+	uids := toSeqSet(msg.Uids)
+	if err := imapw.client.UidCopy(uids, msg.Destination); err != nil {
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+	} else {
+		imapw.worker.PostMessage(&types.MessagesCopied{
+			Message:     types.RespondTo(msg),
+			Destination: msg.Destination,
+			Uids:        msg.Uids,
+		}, nil)
+		imapw.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+	}
+}
+
+type appendLiteral struct {
+	io.Reader
+	Length int
+}
+
+func (m appendLiteral) Len() int {
+	return m.Length
+}
+
+func (imapw *IMAPWorker) handleAppendMessage(msg *types.AppendMessage) {
+	if err := imapw.client.Append(msg.Destination, translateFlags(msg.Flags), msg.Date,
+		&appendLiteral{
+			Reader: msg.Reader,
+			Length: msg.Length,
+		}); err != nil {
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+	} else {
+		imapw.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+	}
+}
+
+func (imapw *IMAPWorker) handleMoveMessages(msg *types.MoveMessages) {
+	drain := imapw.drainUpdates()
+	defer drain.Close()
+
+	uids := toSeqSet(msg.Uids)
+	if err := imapw.client.UidMove(uids, msg.Destination); err != nil {
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+	} else {
+		imapw.worker.PostMessage(&types.MessagesMoved{
+			Message:     types.RespondTo(msg),
+			Destination: msg.Destination,
+			Uids:        msg.Uids,
+		}, nil)
+		imapw.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+	}
+}

worker/imap/observer.go

@@ -0,0 +1,149 @@
+package imap
+
+import (
+	"fmt"
+	"math"
+	"sync"
+	"time"
+
+	"git.sr.ht/~rjarry/aerc/lib/log"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+	"github.com/emersion/go-imap"
+)
+
+// observer monitors the loggedOut channel of the imap client. If the logout
+// signal is received, the observer will emit a connection error to the ui in
+// order to start the reconnect cycle.
+type observer struct {
+	sync.Mutex
+	config        imapConfig
+	client        *imapClient
+	worker        types.WorkerInteractor
+	done          chan struct{}
+	autoReconnect bool
+	retries       int
+	running       bool
+}
+
+func newObserver(cfg imapConfig, w types.WorkerInteractor) *observer {
+	return &observer{config: cfg, worker: w, done: make(chan struct{})}
+}
+
+func (o *observer) SetClient(c *imapClient) {
+	o.Stop()
+	o.Lock()
+	o.client = c
+	o.Unlock()
+	o.Start()
+	o.retries = 0
+}
+
+func (o *observer) SetAutoReconnect(auto bool) {
+	o.autoReconnect = auto
+}
+
+func (o *observer) AutoReconnect() bool {
+	return o.autoReconnect
+}
+
+func (o *observer) isClientConnected() bool {
+	o.Lock()
+	defer o.Unlock()
+	return o.client != nil && o.client.State() == imap.SelectedState
+}
+
+func (o *observer) EmitIfNotConnected() bool {
+	if !o.isClientConnected() {
+		o.emit("imap client not connected: attempt reconnect")
+		return true
+	}
+	return false
+}
+
+func (o *observer) IsRunning() bool {
+	return o.running
+}
+
+func (o *observer) Start() {
+	if o.running {
+		return
+	}
+	if o.client == nil {
+		return
+	}
+	if o.EmitIfNotConnected() {
+		return
+	}
+	go func() {
+		defer log.PanicHandler()
+		select {
+		case <-o.client.LoggedOut():
+			if o.autoReconnect {
+				o.emit("logged out")
+			} else {
+				o.log("ignore logout (auto-reconnect off)")
+			}
+		case <-o.done:
+			break
+		}
+		o.running = false
+		o.log("stopped")
+	}()
+	o.running = true
+	o.log("started")
+}
+
+func (o *observer) Stop() {
+	if o.client == nil {
+		return
+	}
+	if o.done != nil {
+		close(o.done)
+	}
+	o.done = make(chan struct{})
+	o.running = false
+}
+
+func (o *observer) DelayedReconnect() error {
+	var wait time.Duration
+	var reterr error
+
+	if o.retries > 0 {
+		backoff := int(math.Pow(1.8, float64(o.retries)))
+		var err error
+		wait, err = time.ParseDuration(fmt.Sprintf("%ds", backoff))
+		if err != nil {
+			return err
+		}
+		if wait > o.config.reconnect_maxwait {
+			wait = o.config.reconnect_maxwait
+		}
+
+		reterr = fmt.Errorf("reconnect in %v", wait)
+	} else {
+		reterr = fmt.Errorf("reconnect")
+	}
+
+	go func() {
+		defer log.PanicHandler()
+		<-time.After(wait)
+		o.emit(reterr.Error())
+	}()
+
+	o.retries++
+	return reterr
+}
+
+func (o *observer) emit(errMsg string) {
+	o.worker.PostMessage(&types.Done{
+		Message: types.RespondTo(&types.Disconnect{}),
+	}, nil)
+	o.worker.PostMessage(&types.ConnError{
+		Error: fmt.Errorf("%s", errMsg),
+	}, nil)
+}
+
+func (o *observer) log(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	o.worker.Tracef("observer (%p) [running:%t] %s", o, o.running, msg)
+}

worker/imap/open.go

@@ -0,0 +1,203 @@
+package imap
+
+import (
+	"sort"
+
+	sortthread "github.com/emersion/go-imap-sortthread"
+
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+)
+
+func (imapw *IMAPWorker) handleOpenDirectory(msg *types.OpenDirectory) {
+	imapw.worker.Debugf("Opening %s", msg.Directory)
+
+	sel, err := imapw.client.Select(msg.Directory, false)
+	if err != nil {
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+		return
+	}
+	select {
+	case <-msg.Context.Done():
+		imapw.worker.PostMessage(&types.Cancelled{Message: types.RespondTo(msg)}, nil)
+	default:
+		imapw.selected = sel
+		imapw.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+	}
+}
+
+func (imapw *IMAPWorker) handleFetchDirectoryContents(
+	msg *types.FetchDirectoryContents,
+) {
+	if msg.Context.Err() != nil {
+		imapw.worker.PostMessage(&types.Cancelled{
+			Message: types.RespondTo(msg),
+		}, nil)
+		return
+	}
+	imapw.worker.Tracef("Fetching UID list")
+
+	searchCriteria := translateSearch(msg.Filter)
+	sortCriteria := translateSortCriterions(msg.SortCriteria)
+	hasSortCriteria := len(sortCriteria) > 0
+
+	var err error
+	var uids []uint32
+
+	// If the server supports the SORT extension, do the sorting server side
+	switch {
+	case imapw.caps.Sort && hasSortCriteria:
+		uids, err = imapw.client.sort.UidSort(sortCriteria, searchCriteria)
+		if err != nil {
+			imapw.worker.PostMessage(&types.Error{
+				Message: types.RespondTo(msg),
+				Error:   err,
+			}, nil)
+			return
+		}
+		// copy in reverse as msgList displays backwards
+		for i, j := 0, len(uids)-1; i < j; i, j = i+1, j-1 {
+			uids[i], uids[j] = uids[j], uids[i]
+		}
+	default:
+		if hasSortCriteria {
+			imapw.worker.Warnf("SORT is not supported but requested: list messages by UID")
+		}
+		uids, err = imapw.client.UidSearch(searchCriteria)
+		if err != nil {
+			imapw.worker.PostMessage(&types.Error{
+				Message: types.RespondTo(msg),
+				Error:   err,
+			}, nil)
+			return
+		}
+	}
+	if msg.Context.Err() != nil {
+		imapw.worker.PostMessage(&types.Cancelled{
+			Message: types.RespondTo(msg),
+		}, nil)
+		return
+	}
+	imapw.worker.Tracef("Found %d UIDs", len(uids))
+	if msg.Filter == nil {
+		// Only initialize if we are not filtering
+		imapw.seqMap.Initialize(uids)
+	}
+
+	imapw.worker.PostMessage(&types.DirectoryContents{
+		Message: types.RespondTo(msg),
+		Uids:    models.Uint32ToUidList(uids),
+	}, nil)
+	imapw.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+}
+
+type sortFieldMapT map[types.SortField]sortthread.SortField
+
+// caution, incomplete mapping
+var sortFieldMap sortFieldMapT = sortFieldMapT{
+	types.SortArrival: sortthread.SortArrival,
+	types.SortCc:      sortthread.SortCc,
+	types.SortDate:    sortthread.SortDate,
+	types.SortFrom:    sortthread.SortFrom,
+	types.SortSize:    sortthread.SortSize,
+	types.SortSubject: sortthread.SortSubject,
+	types.SortTo:      sortthread.SortTo,
+}
+
+func translateSortCriterions(
+	cs []*types.SortCriterion,
+) []sortthread.SortCriterion {
+	result := make([]sortthread.SortCriterion, 0, len(cs))
+	for _, c := range cs {
+		if f, ok := sortFieldMap[c.Field]; ok {
+			result = append(result, sortthread.SortCriterion{Field: f, Reverse: c.Reverse})
+		}
+	}
+	return result
+}
+
+func (imapw *IMAPWorker) handleDirectoryThreaded(
+	msg *types.FetchDirectoryThreaded,
+) {
+	if msg.Context.Err() != nil {
+		imapw.worker.PostMessage(&types.Cancelled{
+			Message: types.RespondTo(msg),
+		}, nil)
+		return
+	}
+	imapw.worker.Tracef("Fetching threaded UID list")
+
+	searchCriteria := translateSearch(msg.Filter)
+	threads, err := imapw.client.thread.UidThread(imapw.threadAlgorithm,
+		searchCriteria)
+	if err != nil {
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+		return
+	}
+	aercThreads, count := convertThreads(threads, nil)
+	sort.Sort(types.ByUID(aercThreads))
+	imapw.worker.Tracef("Found %d threaded messages", count)
+	if msg.Filter == nil {
+		// Only initialize if we are not filtering
+		var uids []uint32
+		for i := len(aercThreads) - 1; i >= 0; i-- {
+			aercThreads[i].Walk(func(t *types.Thread, level int, currentErr error) error { //nolint:errcheck // error indicates skipped threads
+				uids = append(uids, models.UidToUint32(t.Uid))
+				return nil
+			})
+		}
+		imapw.seqMap.Initialize(uids)
+	}
+	if msg.Context.Err() != nil {
+		imapw.worker.PostMessage(&types.Cancelled{
+			Message: types.RespondTo(msg),
+		}, nil)
+		return
+	}
+	imapw.worker.PostMessage(&types.DirectoryThreaded{
+		Message: types.RespondTo(msg),
+		Threads: aercThreads,
+	}, nil)
+	imapw.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+}
+
+func convertThreads(threads []*sortthread.Thread, parent *types.Thread) ([]*types.Thread, int) {
+	if threads == nil {
+		return nil, 0
+	}
+	conv := make([]*types.Thread, len(threads))
+	count := 0
+
+	for i := 0; i < len(threads); i++ {
+		t := threads[i]
+		conv[i] = &types.Thread{
+			Uid: models.Uint32ToUid(t.Id),
+		}
+
+		// Set the first child node
+		children, childCount := convertThreads(t.Children, conv[i])
+		if len(children) > 0 {
+			conv[i].FirstChild = children[0]
+		}
+
+		// Set the parent node
+		if parent != nil {
+			conv[i].Parent = parent
+
+			// elements of threads are siblings
+			if i > 0 {
+				conv[i].PrevSibling = conv[i-1]
+				conv[i-1].NextSibling = conv[i]
+			}
+		}
+
+		count += childCount + 1
+	}
+	return conv, count
+}

worker/imap/remove.go

@@ -0,0 +1,19 @@
+package imap
+
+import (
+	"git.sr.ht/~rjarry/aerc/worker/types"
+)
+
+func (imapw *IMAPWorker) handleRemoveDirectory(msg *types.RemoveDirectory) {
+	if err := imapw.client.Delete(msg.Directory); err != nil {
+		if msg.Quiet {
+			return
+		}
+		imapw.worker.PostMessage(&types.Error{
+			Message: types.RespondTo(msg),
+			Error:   err,
+		}, nil)
+	} else {
+		imapw.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+	}
+}

worker/imap/search.go

@@ -0,0 +1,52 @@
+package imap
+
+import (
+	"strings"
+
+	"github.com/emersion/go-imap"
+
+	"git.sr.ht/~rjarry/aerc/worker/types"
+	"git.sr.ht/~rjarry/go-opt/v2"
+)
+
+func translateSearch(c *types.SearchCriteria) *imap.SearchCriteria {
+	criteria := imap.NewSearchCriteria()
+	if c == nil {
+		return criteria
+	}
+	criteria.WithFlags = translateFlags(c.WithFlags)
+	criteria.WithoutFlags = translateFlags(c.WithoutFlags)
+
+	if !c.StartDate.IsZero() {
+		criteria.SentSince = c.StartDate
+	}
+	if !c.StartDate.IsZero() {
+		criteria.SentBefore = c.EndDate
+	}
+	for k, v := range c.Headers {
+		criteria.Header[k] = v
+	}
+	for _, f := range c.From {
+		criteria.Header.Add("From", f)
+	}
+	for _, t := range c.To {
+		criteria.Header.Add("To", t)
+	}
+	for _, c := range c.Cc {
+		criteria.Header.Add("Cc", c)
+	}
+	terms := opt.LexArgs(strings.Join(c.Terms, " "))
+	if terms.Count() > 0 {
+		switch {
+		case c.SearchAll:
+			criteria.Text = terms.Args()
+		case c.SearchBody:
+			criteria.Body = terms.Args()
+		default:
+			for _, term := range terms.Args() {
+				criteria.Header.Add("Subject", term)
+			}
+		}
+	}
+	return criteria
+}

worker/imap/seqmap.go

@@ -0,0 +1,77 @@
+package imap
+
+import (
+	"sort"
+	"sync"
+)
+
+type SeqMap struct {
+	lock sync.Mutex
+	// map of IMAP sequence numbers to message UIDs
+	m []uint32
+}
+
+// Initialize sets the initial seqmap of the mailbox
+func (s *SeqMap) Initialize(uids []uint32) {
+	s.lock.Lock()
+	s.m = make([]uint32, len(uids))
+	copy(s.m, uids)
+	s.sort()
+	s.lock.Unlock()
+}
+
+func (s *SeqMap) Size() int {
+	s.lock.Lock()
+	size := len(s.m)
+	s.lock.Unlock()
+	return size
+}
+
+// Get returns the UID of the given seqnum
+func (s *SeqMap) Get(seqnum uint32) (uint32, bool) {
+	if int(seqnum) > s.Size() || seqnum < 1 {
+		return 0, false
+	}
+	s.lock.Lock()
+	uid := s.m[seqnum-1]
+	s.lock.Unlock()
+	return uid, true
+}
+
+// Put adds a UID to the slice. Put should only be used to add new messages
+// into the slice
+func (s *SeqMap) Put(uid uint32) {
+	s.lock.Lock()
+	for _, n := range s.m {
+		if n == uid {
+			// We already have this UID, don't insert it.
+			s.lock.Unlock()
+			return
+		}
+	}
+	s.m = append(s.m, uid)
+	s.sort()
+	s.lock.Unlock()
+}
+
+// Pop removes seqnum from the SeqMap. seqnum must be a valid seqnum, ie
+// [1:size of mailbox]
+func (s *SeqMap) Pop(seqnum uint32) (uint32, bool) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+	if int(seqnum) > len(s.m) || seqnum < 1 {
+		return 0, false
+	}
+	uid := s.m[seqnum-1]
+	s.m = append(s.m[:seqnum-1], s.m[seqnum:]...)
+	return uid, true
+}
+
+// sort sorts the slice in ascending UID order. See:
+// https://datatracker.ietf.org/doc/html/rfc3501#section-2.3.1.2
+func (s *SeqMap) sort() {
+	// Always be sure the SeqMap is sorted
+	sort.Slice(s.m, func(i, j int) bool {
+		return s.m[i] < s.m[j]
+	})
+}

worker/imap/seqmap_test.go

@@ -0,0 +1,85 @@
+package imap
+
+import (
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSeqMap(t *testing.T) {
+	var seqmap SeqMap
+	var uid uint32
+	var found bool
+	assert := assert.New(t)
+
+	assert.Equal(0, seqmap.Size())
+
+	_, found = seqmap.Get(42)
+	assert.Equal(false, found)
+
+	_, found = seqmap.Pop(0)
+	assert.Equal(false, found)
+
+	uids := []uint32{1337, 42, 1107}
+	seqmap.Initialize(uids)
+	assert.Equal(3, seqmap.Size())
+	// Original list should remain unsorted
+	assert.Equal([]uint32{1337, 42, 1107}, uids)
+
+	_, found = seqmap.Pop(0)
+	assert.Equal(false, found)
+
+	uid, found = seqmap.Get(1)
+	assert.Equal(42, int(uid))
+	assert.Equal(true, found)
+
+	uid, found = seqmap.Pop(1)
+	assert.Equal(42, int(uid))
+	assert.Equal(true, found)
+	assert.Equal(2, seqmap.Size())
+
+	uid, found = seqmap.Get(1)
+	assert.Equal(1107, int(uid))
+
+	// Repeated puts of the same UID shouldn't change the size
+	seqmap.Put(1231)
+	assert.Equal(3, seqmap.Size())
+	seqmap.Put(1231)
+	assert.Equal(3, seqmap.Size())
+
+	uid, found = seqmap.Get(2)
+	assert.Equal(1231, int(uid))
+
+	_, found = seqmap.Pop(1)
+	assert.Equal(true, found)
+	assert.Equal(2, seqmap.Size())
+
+	seqmap.Initialize(nil)
+	assert.Equal(0, seqmap.Size())
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		seqmap.Initialize([]uint32{42, 1337})
+	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for _, found := seqmap.Pop(1); !found; _, found = seqmap.Pop(1) {
+			time.Sleep(1 * time.Millisecond)
+		}
+	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for _, found := seqmap.Pop(1); !found; _, found = seqmap.Pop(1) {
+			time.Sleep(1 * time.Millisecond)
+		}
+	}()
+	wg.Wait()
+
+	assert.Equal(0, seqmap.Size())
+}

worker/imap/worker.go

@@ -0,0 +1,397 @@
+package imap
+
+import (
+	"fmt"
+	"net/url"
+	"time"
+
+	"github.com/emersion/go-imap"
+	sortthread "github.com/emersion/go-imap-sortthread"
+	"github.com/emersion/go-imap/client"
+	"github.com/pkg/errors"
+	"github.com/syndtr/goleveldb/leveldb"
+
+	"git.sr.ht/~rjarry/aerc/lib"
+	"git.sr.ht/~rjarry/aerc/models"
+	"git.sr.ht/~rjarry/aerc/worker/handlers"
+	"git.sr.ht/~rjarry/aerc/worker/imap/extensions"
+	"git.sr.ht/~rjarry/aerc/worker/middleware"
+	"git.sr.ht/~rjarry/aerc/worker/types"
+)
+
+func init() {
+	handlers.RegisterWorkerFactory("imap", NewIMAPWorker)
+	handlers.RegisterWorkerFactory("imaps", NewIMAPWorker)
+}
+
+var (
+	errUnsupported      = fmt.Errorf("unsupported command")
+	errClientNotReady   = fmt.Errorf("client not ready")
+	errNotConnected     = fmt.Errorf("not connected")
+	errAlreadyConnected = fmt.Errorf("already connected")
+)
+
+type imapClient struct {
+	*client.Client
+	thread     *sortthread.ThreadClient
+	sort       *sortthread.SortClient
+	liststatus *extensions.ListStatusClient
+}
+
+type imapConfig struct {
+	name              string
+	scheme            string
+	insecure          bool
+	addr              string
+	user              *url.Userinfo
+	headers           []string
+	headersExclude    []string
+	folders           []string
+	oauthBearer       lib.OAuthBearer
+	xoauth2           lib.Xoauth2
+	idle_timeout      time.Duration
+	idle_debounce     time.Duration
+	reconnect_maxwait time.Duration
+	// tcp connection parameters
+	connection_timeout time.Duration
+	keepalive_period   time.Duration
+	keepalive_probes   int
+	keepalive_interval int
+	cacheEnabled       bool
+	cacheMaxAge        time.Duration
+	useXGMEXT          bool
+}
+
+type IMAPWorker struct {
+	config imapConfig
+
+	client    *imapClient
+	selected  *imap.MailboxStatus
+	updates   chan client.Update
+	worker    types.WorkerInteractor
+	seqMap    SeqMap
+	delimiter string
+
+	idler    *idler
+	observer *observer
+	cache    *leveldb.DB
+
+	caps *models.Capabilities
+
+	threadAlgorithm sortthread.ThreadAlgorithm
+	liststatus      bool
+
+	executeIdle chan struct{}
+}
+
+func NewIMAPWorker(worker *types.Worker) (types.Backend, error) {
+	return &IMAPWorker{
+		updates:     make(chan client.Update, 50),
+		worker:      worker,
+		selected:    &imap.MailboxStatus{},
+		idler:       nil, // will be set in configure()
+		observer:    nil, // will be set in configure()
+		caps:        &models.Capabilities{},
+		executeIdle: make(chan struct{}),
+	}, nil
+}
+
+func (w *IMAPWorker) newClient(c *client.Client) {
+	c.Updates = nil
+	w.client = &imapClient{
+		c,
+		sortthread.NewThreadClient(c),
+		sortthread.NewSortClient(c),
+		extensions.NewListStatusClient(c),
+	}
+	if w.idler != nil {
+		w.idler.SetClient(w.client)
+		c.Updates = w.updates
+	}
+	if w.observer != nil {
+		w.observer.SetClient(w.client)
+	}
+	sort, err := w.client.sort.SupportSort()
+	if err == nil && sort {
+		w.caps.Sort = true
+		w.worker.Debugf("Server Capability found: Sort")
+	}
+	for _, alg := range []sortthread.ThreadAlgorithm{sortthread.References, sortthread.OrderedSubject} {
+		ok, err := w.client.Support(fmt.Sprintf("THREAD=%s", string(alg)))
+		if err == nil && ok {
+			w.threadAlgorithm = alg
+			w.caps.Thread = true
+			w.worker.Debugf("Server Capability found: Thread (algorithm: %s)", string(alg))
+			break
+		}
+	}
+	lStatus, err := w.client.liststatus.SupportListStatus()
+	if err == nil && lStatus {
+		w.liststatus = true
+		w.caps.Extensions = append(w.caps.Extensions, "LIST-STATUS")
+		w.worker.Debugf("Server Capability found: LIST-STATUS")
+	}
+	xgmext, err := w.client.Support("X-GM-EXT-1")
+	if err == nil && xgmext && w.config.useXGMEXT {
+		w.caps.Extensions = append(w.caps.Extensions, "X-GM-EXT-1")
+		w.worker.Debugf("Server Capability found: X-GM-EXT-1")
+		w.worker = middleware.NewGmailWorker(w.worker, w.client.Client)
+	}
+	if err == nil && !xgmext && w.config.useXGMEXT {
+		w.worker.Infof("X-GM-EXT-1 requested, but it is not supported")
+	}
+}
+
+func (w *IMAPWorker) handleMessage(msg types.WorkerMessage) error {
+	var reterr error // will be returned at the end, needed to support idle
+
+	// when client is nil allow only certain messages to be handled
+	if w.client == nil {
+		switch msg.(type) {
+		case *types.Connect, *types.Reconnect, *types.Disconnect, *types.Configure:
+		default:
+			return errClientNotReady
+		}
+	}
+
+	// set connection timeout for calls to imap server
+	if w.client != nil {
+		w.client.Timeout = w.config.connection_timeout
+	}
+
+	switch msg := msg.(type) {
+	case *types.Unsupported:
+		// No-op
+	case *types.Configure:
+		reterr = w.handleConfigure(msg)
+	case *types.Connect:
+		if w.client != nil && w.client.State() == imap.SelectedState {
+			if !w.observer.AutoReconnect() {
+				w.observer.SetAutoReconnect(true)
+				w.observer.EmitIfNotConnected()
+			}
+			reterr = errAlreadyConnected
+			break
+		}
+
+		w.observer.SetAutoReconnect(true)
+		c, err := w.connect()
+		if err != nil {
+			w.observer.EmitIfNotConnected()
+			reterr = err
+			break
+		}
+
+		w.newClient(c)
+
+		w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+	case *types.Reconnect:
+		if !w.observer.AutoReconnect() {
+			reterr = fmt.Errorf("auto-reconnect is disabled; run connect to enable it")
+			break
+		}
+		c, err := w.connect()
+		if err != nil {
+			errReconnect := w.observer.DelayedReconnect()
+			reterr = errors.Wrap(errReconnect, err.Error())
+			break
+		}
+
+		w.newClient(c)
+
+		w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+	case *types.Disconnect:
+		w.observer.SetAutoReconnect(false)
+		w.observer.Stop()
+
+		if w.client == nil || (w.client != nil && w.client.State() != imap.SelectedState) {
+			reterr = errNotConnected
+			break
+		}
+
+		if err := w.client.Logout(); err != nil {
+			w.terminate()
+			reterr = err
+			break
+		}
+		w.worker.PostMessage(&types.Done{Message: types.RespondTo(msg)}, nil)
+	case *types.ListDirectories:
+		w.handleListDirectories(msg)
+	case *types.OpenDirectory:
+		w.handleOpenDirectory(msg)
+	case *types.FetchDirectoryContents:
+		w.handleFetchDirectoryContents(msg)
+	case *types.FetchDirectoryThreaded:
+		w.handleDirectoryThreaded(msg)
+	case *types.CreateDirectory:
+		w.handleCreateDirectory(msg)
+	case *types.RemoveDirectory:
+		w.handleRemoveDirectory(msg)
+	case *types.FetchMessageHeaders:
+		w.handleFetchMessageHeaders(msg)
+	case *types.FetchMessageBodyPart:
+		w.handleFetchMessageBodyPart(msg)
+	case *types.FetchFullMessages:
+		w.handleFetchFullMessages(msg)
+	case *types.FetchMessageFlags:
+		w.handleFetchMessageFlags(msg)
+	case *types.DeleteMessages:
+		w.handleDeleteMessages(msg)
+	case *types.FlagMessages:
+		w.handleFlagMessages(msg)
+	case *types.AnsweredMessages:
+		w.handleAnsweredMessages(msg)
+	case *types.CopyMessages:
+		w.handleCopyMessages(msg)
+	case *types.MoveMessages:
+		w.handleMoveMessages(msg)
+	case *types.AppendMessage:
+		w.handleAppendMessage(msg)
+	case *types.SearchDirectory:
+		w.handleSearchDirectory(msg)
+	case *types.CheckMail:
+		w.handleCheckMailMessage(msg)
+	default:
+		reterr = errUnsupported
+	}
+
+	// we don't want idle to timeout, so set timeout to zero
+	if w.client != nil {
+		w.client.Timeout = 0
+	}
+
+	return reterr
+}
+
+func (w *IMAPWorker) handleImapUpdate(update client.Update) {
+	w.worker.Tracef("(= %T", update)
+	switch update := update.(type) {
+	case *client.MailboxUpdate:
+		w.worker.PostAction(&types.CheckMail{
+			Directories: []string{update.Mailbox.Name},
+		}, nil)
+	case *client.MessageUpdate:
+		msg := update.Message
+		if msg.Uid == 0 {
+			if uid, found := w.seqMap.Get(msg.SeqNum); !found {
+				w.worker.Errorf("MessageUpdate unknown seqnum: %d", msg.SeqNum)
+				return
+			} else {
+				msg.Uid = uid
+			}
+		}
+		if int(msg.SeqNum) > w.seqMap.Size() {
+			w.seqMap.Put(msg.Uid)
+		}
+		w.worker.PostMessage(&types.MessageInfo{
+			Info: &models.MessageInfo{
+				BodyStructure: translateBodyStructure(msg.BodyStructure),
+				Envelope:      translateEnvelope(msg.Envelope),
+				Flags:         translateImapFlags(msg.Flags),
+				InternalDate:  msg.InternalDate,
+				Uid:           models.Uint32ToUid(msg.Uid),
+			},
+		}, nil)
+	case *client.ExpungeUpdate:
+		if uid, found := w.seqMap.Pop(update.SeqNum); !found {
+			w.worker.Errorf("ExpungeUpdate unknown seqnum: %d", update.SeqNum)
+		} else {
+			w.worker.PostMessage(&types.MessagesDeleted{
+				Uids: []models.UID{models.Uint32ToUid(uid)},
+			}, nil)
+		}
+	}
+}
+
+func (w *IMAPWorker) terminate() {
+	if w.observer != nil {
+		w.observer.Stop()
+		w.observer.SetClient(nil)
+	}
+
+	if w.client != nil {
+		w.client.Updates = nil
+		if err := w.client.Terminate(); err != nil {
+			w.worker.Errorf("could not terminate connection: %v", err)
+		}
+	}
+
+	w.client = nil
+	w.selected = &imap.MailboxStatus{}
+
+	if w.idler != nil {
+		w.idler.SetClient(nil)
+	}
+}
+
+func (w *IMAPWorker) stopIdler() error {
+	if w.idler == nil {
+		return nil
+	}
+
+	if err := w.idler.Stop(); err != nil {
+		w.terminate()
+		w.observer.EmitIfNotConnected()
+		w.worker.Errorf("idler stopped with error:%v", err)
+		return err
+	}
+
+	return nil
+}
+
+func (w *IMAPWorker) startIdler() {
+	if w.idler == nil {
+		return
+	}
+
+	w.idler.Start()
+}
+
+func (w *IMAPWorker) Run() {
+	for {
+		select {
+		case msg := <-w.worker.Actions():
+
+			if err := w.stopIdler(); err != nil {
+				w.worker.PostMessage(&types.Error{
+					Message: types.RespondTo(msg),
+					Error:   err,
+				}, nil)
+				break
+			}
+			w.worker.Tracef("ready to handle %T", msg)
+
+			msg = w.worker.ProcessAction(msg)
+
+			if err := w.handleMessage(msg); errors.Is(err, errUnsupported) {
+				w.worker.PostMessage(&types.Unsupported{
+					Message: types.RespondTo(msg),
+				}, nil)
+			} else if err != nil {
+				w.worker.PostMessage(&types.Error{
+					Message: types.RespondTo(msg),
+					Error:   err,
+				}, nil)
+			}
+
+			w.startIdler()
+
+		case update := <-w.updates:
+			w.handleImapUpdate(update)
+
+		case <-w.executeIdle:
+			w.idler.Execute()
+		}
+	}
+}
+
+func (w *IMAPWorker) Capabilities() *models.Capabilities {
+	return w.caps
+}
+
+func (w *IMAPWorker) PathSeparator() string {
+	if w.delimiter == "" {
+		return "/"
+	}
+	return w.delimiter
+}