109 lines
2.6 KiB
Go
109 lines
2.6 KiB
Go
// knowledge.go – Listet und löscht Einträge in der Qdrant-Wissensdatenbank
|
||
package brain
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"sort"
|
||
|
||
pb "github.com/qdrant/go-client/qdrant"
|
||
"google.golang.org/grpc/metadata"
|
||
|
||
"my-brain-importer/internal/config"
|
||
)
|
||
|
||
// ListSources gibt alle eindeutigen Quellen in der Wissensdatenbank zurück.
|
||
// Limit begrenzt die Anzahl der zu scrollenden Punkte (0 = Standard 1000).
|
||
func ListSources(limit uint32) ([]string, error) {
|
||
if limit == 0 {
|
||
limit = 1000
|
||
}
|
||
|
||
ctx := context.Background()
|
||
ctx = metadata.AppendToOutgoingContext(ctx, "api-key", config.Cfg.Qdrant.APIKey)
|
||
|
||
conn := config.NewQdrantConn()
|
||
defer conn.Close()
|
||
|
||
pointsClient := pb.NewPointsClient(conn)
|
||
|
||
seen := map[string]bool{}
|
||
var offset *pb.PointId
|
||
|
||
for {
|
||
req := &pb.ScrollPoints{
|
||
CollectionName: config.Cfg.Qdrant.Collection,
|
||
WithPayload: &pb.WithPayloadSelector{
|
||
SelectorOptions: &pb.WithPayloadSelector_Include{
|
||
Include: &pb.PayloadIncludeSelector{Fields: []string{"source"}},
|
||
},
|
||
},
|
||
Limit: uint32Ptr(250),
|
||
}
|
||
if offset != nil {
|
||
req.Offset = offset
|
||
}
|
||
|
||
result, err := pointsClient.Scroll(ctx, req)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("Scroll fehlgeschlagen: %w", err)
|
||
}
|
||
|
||
for _, pt := range result.Result {
|
||
if src := pt.Payload["source"].GetStringValue(); src != "" {
|
||
seen[src] = true
|
||
}
|
||
}
|
||
|
||
if result.NextPageOffset == nil || uint32(len(seen)) >= limit {
|
||
break
|
||
}
|
||
offset = result.NextPageOffset
|
||
}
|
||
|
||
sources := make([]string, 0, len(seen))
|
||
for s := range seen {
|
||
sources = append(sources, s)
|
||
}
|
||
sort.Strings(sources)
|
||
return sources, nil
|
||
}
|
||
|
||
// DeleteBySource löscht alle Punkte mit dem gegebenen Quellennamen aus Qdrant.
|
||
// Gibt Anzahl gelöschter Punkte zurück (Qdrant liefert keine genaue Zahl — gibt 0 zurück wenn erfolgreich).
|
||
func DeleteBySource(source string) error {
|
||
ctx := context.Background()
|
||
ctx = metadata.AppendToOutgoingContext(ctx, "api-key", config.Cfg.Qdrant.APIKey)
|
||
|
||
conn := config.NewQdrantConn()
|
||
defer conn.Close()
|
||
|
||
pointsClient := pb.NewPointsClient(conn)
|
||
|
||
_, err := pointsClient.Delete(ctx, &pb.DeletePoints{
|
||
CollectionName: config.Cfg.Qdrant.Collection,
|
||
Points: &pb.PointsSelector{
|
||
PointsSelectorOneOf: &pb.PointsSelector_Filter{
|
||
Filter: &pb.Filter{
|
||
Must: []*pb.Condition{
|
||
{
|
||
ConditionOneOf: &pb.Condition_Field{
|
||
Field: &pb.FieldCondition{
|
||
Key: "source",
|
||
Match: &pb.Match{
|
||
MatchValue: &pb.Match_Keyword{Keyword: source},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
},
|
||
Wait: boolPtr(true),
|
||
})
|
||
return err
|
||
}
|
||
|
||
func uint32Ptr(v uint32) *uint32 { return &v }
|