feat: index extended filesystem attribute user.xdg.tags (#14)

* wip: adding xattr usser.xdg.tags support

* feat: config setting for xattr indexing

* fix: actually index/search properly

Author: darkliquid

cmd/dsearch/main.go

@@ -65,6 +65,7 @@ var (
 	searchExifLatMax      float64
 	searchExifLonMin      float64
 	searchExifLonMax      float64
+	searchXattrTags       string
 )
 
 var rootCmd = &cobra.Command{
@@ -200,6 +201,7 @@ func init() {
 	searchCmd.Flags().Float64Var(&searchExifLatMax, "exif-lat-max", 0, "maximum GPS latitude")
 	searchCmd.Flags().Float64Var(&searchExifLonMin, "exif-lon-min", 0, "minimum GPS longitude")
 	searchCmd.Flags().Float64Var(&searchExifLonMax, "exif-lon-max", 0, "maximum GPS longitude")
+	searchCmd.Flags().StringVar(&searchXattrTags, "xattr-tags", "", "tags in user.xdg.tags xattr")
 
 	indexFilesCmd.Flags().IntVar(&filesLimit, "limit", 100, "maximum number of files to list")
 
@@ -383,6 +385,7 @@ func runSearch(cmd *cobra.Command, args []string) error {
 		ExifLatMax:      searchExifLatMax,
 		ExifLonMin:      searchExifLonMin,
 		ExifLonMax:      searchExifLonMax,
+		XattrTags:       searchXattrTags,
 	}
 
 	result, err := client.SearchWithOptions(clientOpts)
@@ -442,6 +445,7 @@ func runSearch(cmd *cobra.Command, args []string) error {
 		ExifLatMax:      searchExifLatMax,
 		ExifLonMin:      searchExifLonMin,
 		ExifLonMax:      searchExifLonMax,
+		XattrTags:       searchXattrTags,
 	}
 
 	result, err = idx.SearchWithOptions(indexerOpts)

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/danielgtaylor/huma/v2 v2.35.0
 	github.com/fsnotify/fsnotify v1.9.0
 	github.com/go-chi/chi/v5 v5.2.5
+	github.com/pkg/xattr v0.4.12
 	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
 	github.com/spf13/cobra v1.10.2
 	go.etcd.io/bbolt v1.4.3

go.sum

@@ -96,6 +96,8 @@ github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
 github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
 github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
 github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
+github.com/pkg/xattr v0.4.12 h1:rRTkSyFNTRElv6pkA3zpjHpQ90p/OdHQC1GmGh1aTjM=
+github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
@@ -123,6 +125,7 @@ golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
 golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
 golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
 golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
 golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=

internal/api/handlers.go

@@ -57,6 +57,7 @@ type SearchInput struct {
 	ExifLatMax      float64  `query:"exif_lat_max" doc:"Maximum GPS latitude" example:"41.0"`
 	ExifLonMin      float64  `query:"exif_lon_min" doc:"Minimum GPS longitude" example:"-74.0"`
 	ExifLonMax      float64  `query:"exif_lon_max" doc:"Maximum GPS longitude" example:"-73.0"`
+	XattrTags       string   `query:"xattr_tags" doc:"Tags" example:"+must,should,-must-not"`
 }
 
 type SearchOutput struct {
@@ -122,6 +123,7 @@ func RegisterHandlers(srv *Server, api huma.API) {
 			ExifLatMax:      input.ExifLatMax,
 			ExifLonMin:      input.ExifLonMin,
 			ExifLonMax:      input.ExifLonMax,
+			XattrTags:       input.XattrTags,
 		}
 
 		result, err := srv.Indexer.SearchWithOptions(opts)

internal/client/client.go

@@ -133,6 +133,7 @@ type SearchOptions struct {
 	ExifLatMax      float64
 	ExifLonMin      float64
 	ExifLonMax      float64
+	XattrTags       string
 }
 
 func Search(query string, limit int) (*bleve.SearchResult, error) {
@@ -224,6 +225,9 @@ func SearchWithOptions(opts *SearchOptions) (*bleve.SearchResult, error) {
 	if opts.ExifLonMax != 0 {
 		params["exif_lon_max"] = opts.ExifLonMax
 	}
+	if opts.XattrTags != "" {
+		params["xattr_tags"] = opts.XattrTags
+	}
 
 	result, err := sendRequest("search", params)
 	if err != nil {

internal/config/config.go

@@ -13,25 +13,27 @@ import (
 )
 
 type IndexPath struct {
-	Path          string   `toml:"path"`
-	MaxDepth      int      `toml:"max_depth"`
-	ExcludeHidden bool     `toml:"exclude_hidden"`
-	ExcludeDirs   []string `toml:"exclude_dirs"`
-	ExtractExif   bool     `toml:"extract_exif"`
-	Watch         *bool    `toml:"watch,omitempty"` // nil = true (default), false = skip fsnotify
+	Path             string   `toml:"path"`
+	MaxDepth         int      `toml:"max_depth"`
+	ExcludeHidden    bool     `toml:"exclude_hidden"`
+	ExcludeDirs      []string `toml:"exclude_dirs"`
+	ExtractExif      bool     `toml:"extract_exif"`
+	ExtractXattrTags bool     `toml:"extract_xattr_tags"`
+	Watch            *bool    `toml:"watch,omitempty"` // nil = true (default), false = skip fsnotify
 
 	excludeDirsMap   map[string]bool
 	excludeDirsRegex []*regexp.Regexp
 }
 
 type Config struct {
-	IndexPath     string      `toml:"index_path"`
-	ListenAddr    string      `toml:"listen_addr"`
-	MaxFileBytes  int64       `toml:"max_file_bytes"`
-	WorkerCount   int         `toml:"worker_count"`
-	IndexPaths    []IndexPath `toml:"index_paths"`
-	TextExts      []string    `toml:"text_extensions"`
-	IndexAllFiles bool        `toml:"index_all_files"`
+	IndexPath      string      `toml:"index_path"`
+	ListenAddr     string      `toml:"listen_addr"`
+	MaxFileBytes   int64       `toml:"max_file_bytes"`
+	WorkerCount    int         `toml:"worker_count"`
+	IndexPaths     []IndexPath `toml:"index_paths"`
+	TextExts       []string    `toml:"text_extensions"`
+	IndexAllFiles  bool        `toml:"index_all_files"`
+	IndexXattrTags bool        `toml:"index_xattr_tags"`
 
 	RootDir       string   `toml:"root_dir,omitempty"`
 	MaxDepth      int      `toml:"max_depth,omitempty"`
@@ -121,11 +123,12 @@ func Default() *Config {
 	}
 
 	cfg := &Config{
-		IndexPath:     getDefaultIndexPath(),
-		ListenAddr:    ":43654",
-		MaxFileBytes:  2 * 1024 * 1024,
-		WorkerCount:   workerCount,
-		IndexAllFiles: true,
+		IndexPath:      getDefaultIndexPath(),
+		ListenAddr:     ":43654",
+		MaxFileBytes:   2 * 1024 * 1024,
+		WorkerCount:    workerCount,
+		IndexAllFiles:  true,
+		IndexXattrTags: true,
 		IndexPaths: []IndexPath{
 			{
 				Path:          home,

internal/indexer/indexer.go

@@ -1,13 +1,16 @@
 package indexer
 
 import (
+	"bytes"
 	"crypto/sha256"
+	"encoding/csv"
 	"encoding/hex"
 	"fmt"
 	"io"
 	"mime"
 	"os"
 	"path/filepath"
+	"slices"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -25,6 +28,7 @@ import (
 	_ "github.com/blevesearch/bleve/v2/analysis/tokenizer/single"
 	"github.com/blevesearch/bleve/v2/mapping"
 	query "github.com/blevesearch/bleve/v2/search/query"
+	"github.com/pkg/xattr"
 	"github.com/rwcarlsen/goexif/exif"
 )
 
@@ -47,6 +51,7 @@ type Document struct {
 	ExifFNumber    float64   `json:"exif_fnumber,omitempty"`
 	ExifExposure   string    `json:"exif_exposure,omitempty"`
 	ExifFocalLen   float64   `json:"exif_focal_length,omitempty"`
+	XattrTags      []string  `json:"xattr_tags,omitempty"`
 }
 
 type Indexer struct {
@@ -85,6 +90,7 @@ type SearchOptions struct {
 	ExifLatMax      float64  `json:"exif_lat_max,omitempty"`
 	ExifLonMin      float64  `json:"exif_lon_min,omitempty"`
 	ExifLonMax      float64  `json:"exif_lon_max,omitempty"`
+	XattrTags       string   `json:"xattr_tags,omitempty"`
 }
 
 func New(cfg *config.Config) (*Indexer, error) {
@@ -306,6 +312,10 @@ func buildIndexMapping() mapping.IndexMapping {
 	exifFocalField.Store = true
 	docMapping.AddFieldMappingsAt("exif_focal_length", exifFocalField)
 
+	xattrTagsField := bleve.NewKeywordFieldMapping()
+	xattrTagsField.Store = true
+	docMapping.AddFieldMappingsAt("xattr_tags", xattrTagsField)
+
 	m.DefaultMapping = docMapping
 	return m
 }
@@ -389,9 +399,26 @@ func (i *Indexer) readDocument(path string, info os.FileInfo) (*Document, error)
 		i.extractExifData(path, doc)
 	}
 
+	if i.config.IndexXattrTags {
+		i.extractXattrTags(path, doc)
+	}
+
 	return doc, nil
 }
 
+func (i *Indexer) extractXattrTags(path string, doc *Document) {
+	tags, err := xattr.Get(path, "user.xdg.tags")
+	if err != nil || len(tags) == 0 {
+		return
+	}
+	parsedTags, _ := csv.NewReader(bytes.NewReader(tags)).Read()
+	if len(parsedTags) > 0 {
+		doc.XattrTags = parsedTags
+		slices.Sort(doc.XattrTags)
+		doc.XattrTags = slices.Compact(doc.XattrTags)
+	}
+}
+
 func isImageFile(contentType string) bool {
 	return strings.HasPrefix(contentType, "image/")
 }
@@ -652,6 +679,37 @@ func (i *Indexer) SearchWithOptions(opts *SearchOptions) (*bleve.SearchResult, e
 		filters = append(filters, lonQuery)
 	}
 
+	if i.config.IndexXattrTags && opts.XattrTags != "" {
+		tags, _ := csv.NewReader(strings.NewReader(opts.XattrTags)).Read()
+		if len(tags) > 0 {
+			tagsQuery := bleve.NewBooleanQuery()
+			for _, tag := range tags {
+				if len(tag) == 0 {
+					continue
+				}
+
+				addFn := tagsQuery.AddShould
+				switch tag[0] {
+				case '-':
+					tag = tag[1:]
+					addFn = tagsQuery.AddMustNot
+				case '+':
+					tag = tag[1:]
+					addFn = tagsQuery.AddMust
+				}
+
+				if len(tag) == 0 {
+					continue
+				}
+
+				tagQuery := bleve.NewTermQuery(tag)
+				tagQuery.SetField("xattr_tags")
+				addFn(tagQuery)
+			}
+			filters = append(filters, tagsQuery)
+		}
+	}
+
 	// Combine main query with filters
 	var finalQuery query.Query
 	if len(filters) > 0 {