feat(seed): asynchronous generation for large file sets

Extract seed generation into fs.GenerateSeedArtifacts and add a SeedGenerateTask manager. Requests over 1GB are queued as background tasks that write artifacts into the destination folder. Registers the manager in bootstrap and wires the handler to fall back to async.

internal/bootstrap/task.go

@@ -49,4 +49,5 @@ func InitTaskManager() {
 	op.RegisterSettingChangingCallback(func() {
 		fs.ArchiveContentUploadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressUploadThreadsNum, conf.Conf.Tasks.DecompressUpload.Workers)))
 	})
+	fs.SeedGenerateTaskManager = tache.NewManager[*fs.SeedGenerateTask](tache.WithWorks(setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers)), tache.WithMaxRetry(conf.Conf.Tasks.Upload.MaxRetry)) //seed generation will not support persist
 }

internal/fs/seed_generate.go

@@ -0,0 +1,522 @@
+package fs
+
+import (
+	"bytes"
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+	"io"
+	stdpath "path"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/OpenListTeam/OpenList/v4/internal/conf"
+	"github.com/OpenListTeam/OpenList/v4/internal/errs"
+	"github.com/OpenListTeam/OpenList/v4/internal/model"
+	"github.com/OpenListTeam/OpenList/v4/internal/op"
+	"github.com/OpenListTeam/OpenList/v4/internal/setting"
+	"github.com/OpenListTeam/OpenList/v4/internal/stream"
+	"github.com/OpenListTeam/OpenList/v4/internal/task"
+	"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
+	"github.com/OpenListTeam/OpenList/v4/pkg/torrent"
+	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
+	"github.com/OpenListTeam/OpenList/v4/server/common"
+	"github.com/OpenListTeam/tache"
+	"github.com/pkg/errors"
+)
+
+// MaxSeedGenerateSyncSize bounds synchronous seed generation (1GB). Larger
+// requests are turned into an asynchronous task.
+const MaxSeedGenerateSyncSize = 1 * 1024 * 1024 * 1024
+
+// SeedGenerateNeedsAsync reports whether the given files must be generated
+// asynchronously because they exceed the synchronous size limit. It resolves
+// each path and sums the file sizes, returning the first error encountered.
+func SeedGenerateNeedsAsync(ctx context.Context, user *model.User, paths []string) (bool, error) {
+	var total int64
+	for _, requestedPath := range paths {
+		fullPath, err := user.JoinPath(requestedPath)
+		if err != nil {
+			return false, err
+		}
+		storage, actualPath, err := op.GetStorageAndActualPath(fullPath)
+		if err != nil {
+			return false, err
+		}
+		obj, err := op.Get(ctx, storage, actualPath)
+		if err != nil {
+			return false, fmt.Errorf("seed path must be a readable file: %s", requestedPath)
+		}
+		if obj.IsDir() {
+			return false, fmt.Errorf("seed path must be a readable file: %s", requestedPath)
+		}
+		total += obj.GetSize()
+		if total > MaxSeedGenerateSyncSize {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+// SeedHashSelection controls whole-file and piece hash inclusion.
+type SeedHashSelection struct {
+	Whole  bool `json:"whole"`
+	Pieces bool `json:"pieces"`
+}
+
+// SeedHashMatrix controls the optional hash metadata stored in a seed.
+type SeedHashMatrix struct {
+	MD5    SeedHashSelection `json:"md5"`
+	SHA1   SeedHashSelection `json:"sha1"`
+	SHA256 SeedHashSelection `json:"sha256"`
+}
+
+// SeedGenerateParams carries a fully-resolved seed generation request, free of
+// any HTTP transport concerns so it can run synchronously or as a task.
+type SeedGenerateParams struct {
+	Paths               []string
+	Formats             []string
+	Name                string
+	Comment             string
+	FileComments        map[string]string
+	HashMatrix          SeedHashMatrix
+	PieceSize           int64
+	Trackers            []string
+	Channels            []torrent.SeedChannel
+	OutputPath          string
+	IncludeShare        bool
+	IncludeDirectSource bool
+	ShareFiles          []string
+	DirectFiles         []string
+}
+
+// SeedArtifact is one generated seed container.
+type SeedArtifact struct {
+	Format   string `json:"format"`
+	Name     string `json:"name"`
+	FileName string `json:"file_name"`
+	SeedData string `json:"seed_data"`
+	Size     int    `json:"size"`
+	Path     string `json:"path,omitempty"`
+}
+
+func seedFormats(params SeedGenerateParams) ([]string, error) {
+	formats := append([]string(nil), params.Formats...)
+	if len(formats) == 1 && strings.TrimSpace(formats[0]) == "" {
+		formats[0] = setting.GetStr(conf.SeedDefaultFormat, "oss")
+	}
+	if len(formats) > 3 {
+		return nil, fmt.Errorf("at most three seed formats may be generated")
+	}
+	result := make([]string, 0, len(formats))
+	seen := make(map[string]struct{}, len(formats))
+	for _, rawFormat := range formats {
+		format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(rawFormat), "."))
+		if format == "bt" {
+			format = "torrent"
+		}
+		if format != "oss" && format != "torrent" && format != "cas" {
+			return nil, fmt.Errorf("unsupported seed format %q", rawFormat)
+		}
+		if _, exists := seen[format]; exists {
+			continue
+		}
+		seen[format] = struct{}{}
+		result = append(result, format)
+	}
+	return result, nil
+}
+
+func seedMatrixEmpty(matrix SeedHashMatrix) bool {
+	return !matrix.MD5.Whole && !matrix.MD5.Pieces && !matrix.SHA1.Whole && !matrix.SHA1.Pieces && !matrix.SHA256.Whole && !matrix.SHA256.Pieces
+}
+
+func loadSeedDefaultMatrix() SeedHashMatrix {
+	var matrix SeedHashMatrix
+	if raw := strings.TrimSpace(setting.GetStr(conf.SeedDefaultMatrix)); raw != "" {
+		_ = json.Unmarshal([]byte(raw), &matrix)
+	}
+	return matrix
+}
+
+func normalizedSeedMatrix(matrix SeedHashMatrix, formats []string) SeedHashMatrix {
+	if seedMatrixEmpty(matrix) {
+		matrix = loadSeedDefaultMatrix()
+	}
+	if seedMatrixEmpty(matrix) {
+		matrix = SeedHashMatrix{
+			MD5: SeedHashSelection{Whole: true, Pieces: true}, SHA1: SeedHashSelection{Whole: true, Pieces: true},
+			SHA256: SeedHashSelection{Whole: true, Pieces: true},
+		}
+	}
+	for _, format := range formats {
+		switch format {
+		case "torrent":
+			matrix.SHA1 = SeedHashSelection{Whole: true, Pieces: true}
+		case "cas":
+			matrix.MD5 = SeedHashSelection{Whole: true, Pieces: true}
+		}
+	}
+	return matrix
+}
+
+func canReuseListedHashes(hashInfo utils.HashInfo, matrix SeedHashMatrix) bool {
+	if matrix.MD5.Pieces || matrix.SHA1.Pieces || matrix.SHA256.Pieces {
+		return false
+	}
+	if matrix.MD5.Whole && hashInfo.GetHash(utils.MD5) == "" {
+		return false
+	}
+	if matrix.SHA1.Whole && hashInfo.GetHash(utils.SHA1) == "" {
+		return false
+	}
+	if matrix.SHA256.Whole && hashInfo.GetHash(utils.SHA256) == "" {
+		return false
+	}
+	return true
+}
+
+func applySeedMatrix(file *torrent.SeedFile, matrix SeedHashMatrix) {
+	if !matrix.MD5.Whole {
+		file.Hashes.MD5 = ""
+	}
+	if !matrix.SHA1.Whole {
+		file.Hashes.SHA1 = ""
+	}
+	if !matrix.SHA256.Whole {
+		file.Hashes.SHA256 = ""
+	}
+	if file.Hashes.Pieces == nil {
+		return
+	}
+	if !matrix.MD5.Pieces {
+		file.Hashes.Pieces.MD5 = nil
+	}
+	if !matrix.SHA1.Pieces {
+		file.Hashes.Pieces.SHA1 = nil
+	}
+	if !matrix.SHA256.Pieces {
+		file.Hashes.Pieces.SHA256 = nil
+	}
+	if len(file.Hashes.Pieces.MD5) == 0 && len(file.Hashes.Pieces.SHA1) == 0 && len(file.Hashes.Pieces.SHA256) == 0 {
+		file.Hashes.Pieces = nil
+	}
+}
+
+func encodeGeneratedSeed(seed *torrent.Seed, format string, standardPieces []byte) ([]byte, error) {
+	if format != "torrent" {
+		return torrent.EncodeSeed(seed, format)
+	}
+	t := &torrent.Torrent{
+		Info:         torrent.TorrentInfo{Name: seed.Name, PieceLength: seed.PieceSize, Pieces: standardPieces},
+		Comment:      seed.Comment,
+		CreatedBy:    seed.CreatedBy,
+		CreationDate: time.Now().Unix(),
+		OpenList:     seed,
+	}
+	if len(seed.Trackers) > 0 {
+		t.Announce = seed.Trackers[0]
+		for _, tracker := range seed.Trackers {
+			t.AnnounceList = append(t.AnnounceList, []string{tracker})
+		}
+	}
+	if len(seed.Files) == 1 {
+		file := seed.Files[0]
+		t.Info.Name = stdpath.Base(file.Path)
+		t.Info.Length = file.Size
+		t.Info.MD5Sum = file.Hashes.MD5
+		if file.CASSliceMD5 != "" {
+			t.SetCASInfo(&torrent.CASInfo{
+				FileMD5: strings.ToUpper(file.Hashes.MD5), SliceMD5: strings.ToUpper(file.CASSliceMD5),
+				SliceSize: torrent.DefaultPieceSize, Cloud: "189",
+			})
+		} else if seed.PieceSize == torrent.DefaultPieceSize && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 {
+			t.SetCASInfo(torrent.BuildCASInfoFromMD5s(file.Hashes.MD5, file.Hashes.Pieces.MD5, torrent.DefaultPieceSize))
+		}
+	} else {
+		for _, file := range seed.Files {
+			t.Info.Files = append(t.Info.Files, torrent.TorrentFile{Length: file.Size, Path: strings.Split(file.Path, "/"), MD5Sum: file.Hashes.MD5})
+		}
+	}
+	return t.Encode()
+}
+
+// GenerateSeedArtifacts reads each file once while computing the requested
+// hashes, then emits one or more seed containers. It has no HTTP dependency and
+// can be driven synchronously or from a background task.
+func GenerateSeedArtifacts(ctx context.Context, user *model.User, params SeedGenerateParams) ([]SeedArtifact, *torrent.Seed, error) {
+	if len(params.Paths) == 0 || len(params.Paths) > torrent.DefaultMaxSeedFiles {
+		return nil, nil, fmt.Errorf("invalid seed file count")
+	}
+	formats, err := seedFormats(params)
+	if err != nil {
+		return nil, nil, err
+	}
+	matrix := normalizedSeedMatrix(params.HashMatrix, formats)
+	pieceSize := params.PieceSize
+	if pieceSize <= 0 {
+		pieceSize = torrent.DefaultPieceSize
+	}
+	if slices.Contains(formats, "cas") {
+		pieceSize = torrent.DefaultPieceSize
+	}
+	seedName := strings.TrimSpace(params.Name)
+	if seedName == "" {
+		if len(params.Paths) == 1 {
+			seedName = stdpath.Base(params.Paths[0])
+		} else {
+			seedName = "OpenList Seed"
+		}
+	}
+	seed := torrent.NewSeed(seedName, "OpenList", pieceSize)
+	seed.Comment = params.Comment
+	seed.Trackers = params.Trackers
+	seed.Channels = params.Channels
+	shareSet := make(map[string]bool, len(params.ShareFiles))
+	for _, p := range params.ShareFiles {
+		if strings.TrimSpace(p) != "" {
+			shareSet[p] = true
+		}
+	}
+	directSet := make(map[string]bool, len(params.DirectFiles))
+	for _, p := range params.DirectFiles {
+		if strings.TrimSpace(p) != "" {
+			directSet[p] = true
+		}
+	}
+	useGlobalShare := len(shareSet) == 0 && params.IncludeShare
+	useGlobalDirect := len(directSet) == 0 && params.IncludeDirectSource
+	hasShare := useGlobalShare || len(shareSet) > 0
+	hasDirect := useGlobalDirect || len(directSet) > 0
+	if hasShare && !user.CanShare() {
+		return nil, nil, errs.PermissionDenied
+	}
+	if hasDirect && setting.GetBool(conf.SignAll) && !hasShare {
+		return nil, nil, fmt.Errorf("direct sources require an automatic share when global signing is enabled")
+	}
+	if (hasShare || hasDirect) && strings.TrimSpace(setting.GetStr(conf.SeedSiteURL)) == "" {
+		return nil, nil, fmt.Errorf("seed_site_url must be configured before embedding download sources")
+	}
+	globalHasher := torrent.NewHashWriter(pieceSize, pieceSize)
+	fullPaths := make([]string, 0, len(params.Paths))
+	var total int64
+	for _, requestedPath := range params.Paths {
+		fullPath, err := user.JoinPath(requestedPath)
+		if err != nil {
+			return nil, nil, err
+		}
+		meta, err := op.GetNearestMeta(fullPath)
+		if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) {
+			return nil, nil, err
+		}
+		if !common.CanRead(user, meta, fullPath) {
+			return nil, nil, errs.PermissionDenied
+		}
+		storage, actualPath, err := op.GetStorageAndActualPath(fullPath)
+		if err != nil {
+			return nil, nil, err
+		}
+		obj, err := op.Get(ctx, storage, actualPath)
+		if err != nil || obj.IsDir() {
+			return nil, nil, fmt.Errorf("seed path must be a readable file: %s", requestedPath)
+		}
+		total += obj.GetSize()
+		modified := ""
+		if !obj.ModTime().IsZero() {
+			modified = obj.ModTime().UTC().Format(time.RFC3339)
+		}
+		seedPath := stdpath.Base(requestedPath)
+		if len(params.Paths) > 1 {
+			seedPath = strings.TrimPrefix(stdpath.Clean(requestedPath), "/")
+		}
+
+		hashInfo := obj.GetHash()
+		if canReuseListedHashes(hashInfo, matrix) {
+			seedFile := torrent.SeedFile{
+				Path:     seedPath,
+				Size:     obj.GetSize(),
+				Modified: modified,
+				Hashes: torrent.SeedHashes{
+					MD5:    strings.ToLower(hashInfo.GetHash(utils.MD5)),
+					SHA1:   strings.ToLower(hashInfo.GetHash(utils.SHA1)),
+					SHA256: strings.ToLower(hashInfo.GetHash(utils.SHA256)),
+				},
+			}
+			applySeedMatrix(&seedFile, matrix)
+			if comment := strings.TrimSpace(params.FileComments[requestedPath]); comment != "" {
+				seedFile.Comment = comment
+			} else if comment := strings.TrimSpace(params.FileComments[obj.GetName()]); comment != "" {
+				seedFile.Comment = comment
+			}
+			if useGlobalDirect || directSet[requestedPath] {
+				baseURL := strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/")
+				seedFile.Sources = []torrent.SeedSource{{Type: "openlist-direct", URL: baseURL + utils.EncodePath("/d"+fullPath)}}
+			}
+			seed.Files = append(seed.Files, seedFile)
+			fullPaths = append(fullPaths, fullPath)
+			continue
+		}
+
+		link, _, err := op.Link(ctx, storage, actualPath, model.LinkArgs{})
+		if err != nil {
+			return nil, nil, fmt.Errorf("storage cannot stream %s: %v", requestedPath, err)
+		}
+		rangeReader, err := stream.GetRangeReaderFromLink(obj.GetSize(), link)
+		if err != nil {
+			return nil, nil, fmt.Errorf("storage cannot stream %s", requestedPath)
+		}
+		rc, err := rangeReader.RangeRead(ctx, http_range.Range{Length: obj.GetSize()})
+		if err != nil {
+			return nil, nil, err
+		}
+		fileHasher := torrent.NewHashWriter(pieceSize, pieceSize)
+		n, copyErr := io.Copy(io.MultiWriter(globalHasher, fileHasher), rc)
+		_ = rc.Close()
+		if copyErr != nil {
+			return nil, nil, fmt.Errorf("read %s: %w", requestedPath, copyErr)
+		}
+		if n != obj.GetSize() {
+			return nil, nil, fmt.Errorf("read %s: got %d of %d bytes", requestedPath, n, obj.GetSize())
+		}
+		fileHasher.Finish()
+		seedFile := fileHasher.BuildSeedFile(seedPath, modified)
+		applySeedMatrix(&seedFile, matrix)
+		if comment := strings.TrimSpace(params.FileComments[requestedPath]); comment != "" {
+			seedFile.Comment = comment
+		} else if comment := strings.TrimSpace(params.FileComments[obj.GetName()]); comment != "" {
+			seedFile.Comment = comment
+		}
+		if useGlobalDirect || directSet[requestedPath] {
+			baseURL := strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/")
+			seedFile.Sources = []torrent.SeedSource{{Type: "openlist-direct", URL: baseURL + utils.EncodePath("/d"+fullPath)}}
+		}
+		seed.Files = append(seed.Files, seedFile)
+		fullPaths = append(fullPaths, fullPath)
+	}
+	globalHasher.Finish()
+	createdShares := make([]string, 0, len(seed.Files))
+	keepCreatedShares := false
+	defer func() {
+		if !keepCreatedShares {
+			for _, createdID := range createdShares {
+				_ = op.DeleteSharing(createdID)
+			}
+		}
+	}()
+	if hasShare {
+		for index, fullPath := range fullPaths {
+			if !useGlobalShare && !shareSet[params.Paths[index]] {
+				continue
+			}
+			sharing := &model.Sharing{
+				SharingDB: &model.SharingDB{Remark: "Transfer seed source"},
+				Files:     []string{fullPath}, Creator: user,
+			}
+			shareID, createErr := op.CreateSharing(sharing)
+			if createErr != nil {
+				return nil, nil, fmt.Errorf("create seed share: %w", createErr)
+			}
+			createdShares = append(createdShares, shareID)
+			seed.Files[index].Sources = []torrent.SeedSource{{
+				Type: "openlist-share", URL: strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/") + "/sd/" + shareID,
+				ShareID: shareID,
+			}}
+		}
+	}
+	if err := torrent.ValidateSeed(seed, torrent.DefaultParseLimits()); err != nil {
+		return nil, nil, err
+	}
+	outputPath := strings.TrimSpace(params.OutputPath)
+	artifacts := make([]SeedArtifact, 0, len(formats))
+	seenFormats := make(map[string]struct{}, len(formats))
+	for _, requestedFormat := range formats {
+		format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(requestedFormat), "."))
+		if format == "bt" {
+			format = "torrent"
+		}
+		if _, exists := seenFormats[format]; exists {
+			continue
+		}
+		seenFormats[format] = struct{}{}
+		data, err := encodeGeneratedSeed(seed, format, globalHasher.GetPieceHashes())
+		if err != nil {
+			return nil, nil, fmt.Errorf("generate %s seed: %w", format, err)
+		}
+		fileName := stdpath.Base(seed.Name) + "." + format
+		artifact := SeedArtifact{
+			Format:   format,
+			Name:     fileName,
+			FileName: fileName,
+			SeedData: base64.StdEncoding.EncodeToString(data),
+			Size:     len(data),
+		}
+		if outputPath != "" {
+			dstDir, err := user.JoinPath(outputPath)
+			if err != nil {
+				return nil, nil, err
+			}
+			meta, metaErr := op.GetNearestMeta(dstDir)
+			if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) {
+				return nil, nil, metaErr
+			}
+			if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, dstDir)) || !common.CanWrite(user, meta, dstDir) {
+				return nil, nil, errs.PermissionDenied
+			}
+			fileStream := &stream.FileStream{
+				Ctx:    ctx,
+				Obj:    &model.Object{Name: fileName, Size: int64(len(data)), Modified: time.Now()},
+				Reader: bytes.NewReader(data), Mimetype: "application/octet-stream",
+			}
+			if err = PutDirectly(ctx, dstDir, fileStream); err != nil {
+				return nil, nil, err
+			}
+			artifact.Path = stdpath.Join(outputPath, fileName)
+		}
+		artifacts = append(artifacts, artifact)
+	}
+	keepCreatedShares = true
+	return artifacts, seed, nil
+}
+
+// SeedGenerateTask generates seed containers asynchronously, writing them into
+// params.OutputPath so they appear in the target folder once done.
+type SeedGenerateTask struct {
+	task.TaskExtension
+	params SeedGenerateParams
+}
+
+func (t *SeedGenerateTask) GetName() string {
+	if len(t.params.Paths) == 1 {
+		return fmt.Sprintf("generate seed for %s", stdpath.Base(t.params.Paths[0]))
+	}
+	return fmt.Sprintf("generate seed for %d files", len(t.params.Paths))
+}
+
+func (t *SeedGenerateTask) GetStatus() string {
+	return "generating seed"
+}
+
+func (t *SeedGenerateTask) Run() error {
+	t.ClearEndTime()
+	t.SetStartTime(time.Now())
+	defer func() { t.SetEndTime(time.Now()) }()
+	_, _, err := GenerateSeedArtifacts(t.Ctx(), t.Creator, t.params)
+	return err
+}
+
+var SeedGenerateTaskManager *tache.Manager[*SeedGenerateTask]
+
+// AddSeedGenerateTask schedules an asynchronous seed generation task.
+func AddSeedGenerateTask(ctx context.Context, user *model.User, params SeedGenerateParams) (task.TaskExtensionInfo, error) {
+	t := &SeedGenerateTask{
+		TaskExtension: task.TaskExtension{
+			Creator: user,
+			ApiUrl:  common.GetApiUrl(ctx),
+		},
+		params: params,
+	}
+	SeedGenerateTaskManager.Add(t)
+	return t, nil
+}

server/handles/torrent.go

@@ -723,36 +723,6 @@ func DiagnoseSeed(c *gin.Context) {
 	common.SuccessResp(c, seedDiagnostics(seed))
 }
 
-func normalizedSeedFormats(req SeedGenerateReq) ([]string, error) {
-	formats := append([]string(nil), req.Formats...)
-	if len(formats) == 0 {
-		formats = []string{req.Format}
-	}
-	if len(formats) == 1 && strings.TrimSpace(formats[0]) == "" {
-		formats[0] = setting.GetStr(conf.SeedDefaultFormat, "oss")
-	}
-	if len(formats) > 3 {
-		return nil, fmt.Errorf("at most three seed formats may be generated")
-	}
-	result := make([]string, 0, len(formats))
-	seen := make(map[string]struct{}, len(formats))
-	for _, rawFormat := range formats {
-		format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(rawFormat), "."))
-		if format == "bt" {
-			format = "torrent"
-		}
-		if format != "oss" && format != "torrent" && format != "cas" {
-			return nil, fmt.Errorf("unsupported seed format %q", rawFormat)
-		}
-		if _, exists := seen[format]; exists {
-			continue
-		}
-		seen[format] = struct{}{}
-		result = append(result, format)
-	}
-	return result, nil
-}
-
 func seedMatrixEmpty(matrix SeedHashMatrix) bool {
 	return !matrix.MD5.Whole && !matrix.MD5.Pieces && !matrix.SHA1.Whole && !matrix.SHA1.Pieces && !matrix.SHA256.Whole && !matrix.SHA256.Pieces
 }
@@ -787,27 +757,6 @@ func normalizedSeedMatrix(matrix SeedHashMatrix, formats []string) SeedHashMatri
 	return matrix
 }
 
-// canReuseListedHashes reports whether a file's hashes can be sourced entirely
-// from the storage driver's file listing (no download) for the given matrix.
-// Piece hashes are never available from listings, so any piece requirement forces
-// a streamed download. Whole-file hashes are reused only when the driver actually
-// provides each one the matrix requests.
-func canReuseListedHashes(hashInfo utils.HashInfo, matrix SeedHashMatrix) bool {
-	if matrix.MD5.Pieces || matrix.SHA1.Pieces || matrix.SHA256.Pieces {
-		return false
-	}
-	if matrix.MD5.Whole && hashInfo.GetHash(utils.MD5) == "" {
-		return false
-	}
-	if matrix.SHA1.Whole && hashInfo.GetHash(utils.SHA1) == "" {
-		return false
-	}
-	if matrix.SHA256.Whole && hashInfo.GetHash(utils.SHA256) == "" {
-		return false
-	}
-	return true
-}
-
 func applySeedMatrix(file *torrent.SeedFile, matrix SeedHashMatrix) {
 	if !matrix.MD5.Whole {
 		file.Hashes.MD5 = ""
@@ -835,7 +784,33 @@ func applySeedMatrix(file *torrent.SeedFile, matrix SeedHashMatrix) {
 	}
 }
 
+// toSeedGenerateParams converts the HTTP request into the transport-agnostic
+// params consumed by fs.GenerateSeedArtifacts.
+func toSeedGenerateParams(req SeedGenerateReq) fs.SeedGenerateParams {
+	return fs.SeedGenerateParams{
+		Paths:        req.Paths,
+		Formats:      req.Formats,
+		Name:         req.Name,
+		Comment:      req.Comment,
+		FileComments: req.FileComments,
+		HashMatrix: fs.SeedHashMatrix{
+			MD5:    fs.SeedHashSelection(req.HashMatrix.MD5),
+			SHA1:   fs.SeedHashSelection(req.HashMatrix.SHA1),
+			SHA256: fs.SeedHashSelection(req.HashMatrix.SHA256),
+		},
+		PieceSize:           req.PieceSize,
+		Trackers:            req.Trackers,
+		Channels:            req.Channels,
+		OutputPath:          req.OutputPath,
+		IncludeShare:        req.IncludeShare,
+		IncludeDirectSource: req.IncludeDirectSource,
+		ShareFiles:          req.ShareFiles,
+		DirectFiles:         req.DirectFiles,
+	}
+}
+
 // GenerateSeedForPaths reads each file once while calculating complete hashes.
+// Requests exceeding the synchronous size limit are queued as a background task.
 func GenerateSeedForPaths(c *gin.Context) {
 	user := c.Request.Context().Value(conf.UserKey).(*model.User)
 	var req SeedGenerateReq
@@ -847,307 +822,45 @@ func GenerateSeedForPaths(c *gin.Context) {
 		common.ErrorStrResp(c, "invalid seed file count", 400)
 		return
 	}
-	formats, err := normalizedSeedFormats(req)
+	if len(req.Formats) == 0 {
+		req.Formats = []string{req.Format}
+	}
+	params := toSeedGenerateParams(req)
+	async, err := fs.SeedGenerateNeedsAsync(c.Request.Context(), user, req.Paths)
 	if err != nil {
 		common.ErrorResp(c, err, 400)
 		return
 	}
-	matrix := normalizedSeedMatrix(req.HashMatrix, formats)
-	pieceSize := req.PieceSize
-	if pieceSize <= 0 {
-		pieceSize = torrent.DefaultPieceSize
-	}
-	if slices.Contains(formats, "cas") {
-		pieceSize = torrent.DefaultPieceSize
-	}
-	seedName := strings.TrimSpace(req.Name)
-	if seedName == "" {
-		if len(req.Paths) == 1 {
-			seedName = stdpath.Base(req.Paths[0])
-		} else {
-			seedName = "OpenList Seed"
+	if async {
+		outputPath := strings.TrimSpace(req.OutputPath)
+		if outputPath == "" {
+			outputPath = strings.TrimSpace(req.SavePath)
 		}
-	}
-	seed := torrent.NewSeed(seedName, "OpenList", pieceSize)
-	seed.Comment = req.Comment
-	seed.Trackers = req.Trackers
-	seed.Channels = req.Channels
-	// Per-file share/direct selection. When either list is empty, fall back to the
-	// legacy global flags so existing callers keep working.
-	shareSet := make(map[string]bool, len(req.ShareFiles))
-	for _, p := range req.ShareFiles {
-		if strings.TrimSpace(p) != "" {
-			shareSet[p] = true
+		if outputPath == "" {
+			common.ErrorStrResp(c, "asynchronous seed generation requires an output_path", 400)
+			return
 		}
-	}
-	directSet := make(map[string]bool, len(req.DirectFiles))
-	for _, p := range req.DirectFiles {
-		if strings.TrimSpace(p) != "" {
-			directSet[p] = true
+		params.OutputPath = outputPath
+		taskInfo, addErr := fs.AddSeedGenerateTask(c.Request.Context(), user, params)
+		if addErr != nil {
+			common.ErrorResp(c, addErr, 500)
+			return
 		}
-	}
-	useGlobalShare := len(shareSet) == 0 && req.IncludeShare
-	useGlobalDirect := len(directSet) == 0 && req.IncludeDirectSource
-	hasShare := useGlobalShare || len(shareSet) > 0
-	hasDirect := useGlobalDirect || len(directSet) > 0
-	if hasShare && !user.CanShare() {
-		common.ErrorResp(c, errs.PermissionDenied, 403)
+		common.SuccessResp(c, gin.H{"task": getTaskInfo(taskInfo), "async": true})
 		return
 	}
-	if hasDirect && setting.GetBool(conf.SignAll) && !hasShare {
-		common.ErrorStrResp(c, "direct sources require an automatic share when global signing is enabled", 400)
+	artifacts, seed, genErr := fs.GenerateSeedArtifacts(c.Request.Context(), user, params)
+	if genErr != nil {
+		status := 400
+		if errors.Is(genErr, errs.PermissionDenied) {
+			status = 403
+		}
+		common.ErrorResp(c, genErr, status)
 		return
 	}
-	if (hasShare || hasDirect) && strings.TrimSpace(setting.GetStr(conf.SeedSiteURL)) == "" {
-		common.ErrorStrResp(c, "seed_site_url must be configured before embedding download sources", 400)
-		return
-	}
-	globalHasher := torrent.NewHashWriter(pieceSize, pieceSize)
-	fullPaths := make([]string, 0, len(req.Paths))
-	var total int64
-	for _, requestedPath := range req.Paths {
-		fullPath, err := user.JoinPath(requestedPath)
-		if err != nil {
-			common.ErrorResp(c, err, 403)
-			return
-		}
-		meta, err := op.GetNearestMeta(fullPath)
-		if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) {
-			common.ErrorResp(c, err, 500, true)
-			return
-		}
-		if !common.CanRead(user, meta, fullPath) {
-			common.ErrorResp(c, errs.PermissionDenied, 403)
-			return
-		}
-		storage, actualPath, err := op.GetStorageAndActualPath(fullPath)
-		if err != nil {
-			common.ErrorResp(c, err, 400)
-			return
-		}
-		obj, err := op.Get(c.Request.Context(), storage, actualPath)
-		if err != nil || obj.IsDir() {
-			common.ErrorResp(c, fmt.Errorf("seed path must be a readable file: %s", requestedPath), 400)
-			return
-		}
-		if obj.GetSize() < 0 || total > maxTorrentGenFileSize-obj.GetSize() {
-			common.ErrorStrResp(c, "synchronous seed generation is limited to 1GB", 400)
-			return
-		}
-		total += obj.GetSize()
-		modified := ""
-		if !obj.ModTime().IsZero() {
-			modified = obj.ModTime().UTC().Format(time.RFC3339)
-		}
-		seedPath := stdpath.Base(requestedPath)
-		if len(req.Paths) > 1 {
-			seedPath = strings.TrimPrefix(stdpath.Clean(requestedPath), "/")
-		}
-
-		// Reuse whole-file hashes from the storage driver's listing when the
-		// matrix needs no piece hashes. This skips the download entirely.
-		hashInfo := obj.GetHash()
-		if canReuseListedHashes(hashInfo, matrix) {
-			seedFile := torrent.SeedFile{
-				Path:     seedPath,
-				Size:     obj.GetSize(),
-				Modified: modified,
-				Hashes: torrent.SeedHashes{
-					MD5:    strings.ToLower(hashInfo.GetHash(utils.MD5)),
-					SHA1:   strings.ToLower(hashInfo.GetHash(utils.SHA1)),
-					SHA256: strings.ToLower(hashInfo.GetHash(utils.SHA256)),
-				},
-			}
-			applySeedMatrix(&seedFile, matrix)
-			if comment := strings.TrimSpace(req.FileComments[requestedPath]); comment != "" {
-				seedFile.Comment = comment
-			} else if comment := strings.TrimSpace(req.FileComments[obj.GetName()]); comment != "" {
-				seedFile.Comment = comment
-			}
-			if useGlobalDirect || directSet[requestedPath] {
-				baseURL := strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/")
-				seedFile.Sources = []torrent.SeedSource{{Type: "openlist-direct", URL: baseURL + utils.EncodePath("/d"+fullPath)}}
-			}
-			seed.Files = append(seed.Files, seedFile)
-			fullPaths = append(fullPaths, fullPath)
-			continue
-		}
-
-		link, _, err := op.Link(c.Request.Context(), storage, actualPath, model.LinkArgs{})
-		if err != nil {
-			common.ErrorResp(c, fmt.Errorf("storage cannot stream %s: %v", requestedPath, err), 400)
-			return
-		}
-		rangeReader, err := stream.GetRangeReaderFromLink(obj.GetSize(), link)
-		if err != nil {
-			common.ErrorResp(c, fmt.Errorf("storage cannot stream %s", requestedPath), 400)
-			return
-		}
-		rc, err := rangeReader.RangeRead(c.Request.Context(), http_range.Range{Length: obj.GetSize()})
-		if err != nil {
-			common.ErrorResp(c, err, 500)
-			return
-		}
-		fileHasher := torrent.NewHashWriter(pieceSize, pieceSize)
-		n, copyErr := io.Copy(io.MultiWriter(globalHasher, fileHasher), rc)
-		_ = rc.Close()
-		if copyErr != nil {
-			common.ErrorResp(c, fmt.Errorf("read %s: %w", requestedPath, copyErr), 500)
-			return
-		}
-		if n != obj.GetSize() {
-			common.ErrorResp(c, fmt.Errorf("read %s: got %d of %d bytes", requestedPath, n, obj.GetSize()), 500)
-			return
-		}
-		fileHasher.Finish()
-		seedFile := fileHasher.BuildSeedFile(seedPath, modified)
-		applySeedMatrix(&seedFile, matrix)
-		if comment := strings.TrimSpace(req.FileComments[requestedPath]); comment != "" {
-			seedFile.Comment = comment
-		} else if comment := strings.TrimSpace(req.FileComments[obj.GetName()]); comment != "" {
-			seedFile.Comment = comment
-		}
-		if useGlobalDirect || directSet[requestedPath] {
-			baseURL := strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/")
-			seedFile.Sources = []torrent.SeedSource{{Type: "openlist-direct", URL: baseURL + utils.EncodePath("/d"+fullPath)}}
-		}
-		seed.Files = append(seed.Files, seedFile)
-		fullPaths = append(fullPaths, fullPath)
-	}
-	globalHasher.Finish()
-	createdShares := make([]string, 0, len(seed.Files))
-	keepCreatedShares := false
-	defer func() {
-		if !keepCreatedShares {
-			for _, createdID := range createdShares {
-				_ = op.DeleteSharing(createdID)
-			}
-		}
-	}()
-	if hasShare {
-		for index, fullPath := range fullPaths {
-			if !useGlobalShare && !shareSet[req.Paths[index]] {
-				continue
-			}
-			sharing := &model.Sharing{
-				SharingDB: &model.SharingDB{Remark: "Transfer seed source"},
-				Files:     []string{fullPath}, Creator: user,
-			}
-			shareID, createErr := op.CreateSharing(sharing)
-			if createErr != nil {
-				common.ErrorResp(c, fmt.Errorf("create seed share: %w", createErr), 500)
-				return
-			}
-			createdShares = append(createdShares, shareID)
-			seed.Files[index].Sources = []torrent.SeedSource{{
-				Type: "openlist-share", URL: strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/") + "/sd/" + shareID,
-				ShareID: shareID,
-			}}
-		}
-	}
-	if err := torrent.ValidateSeed(seed, torrent.DefaultParseLimits()); err != nil {
-		common.ErrorResp(c, err, 400)
-		return
-	}
-	outputPath := strings.TrimSpace(req.OutputPath)
-	if outputPath == "" {
-		outputPath = strings.TrimSpace(req.SavePath)
-	}
-	artifacts := make([]gin.H, 0, len(formats))
-	seenFormats := make(map[string]struct{}, len(formats))
-	for _, requestedFormat := range formats {
-		format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(requestedFormat), "."))
-		if format == "bt" {
-			format = "torrent"
-		}
-		if _, exists := seenFormats[format]; exists {
-			continue
-		}
-		seenFormats[format] = struct{}{}
-		data, err := encodeGeneratedSeed(seed, format, globalHasher.GetPieceHashes())
-		if err != nil {
-			common.ErrorResp(c, fmt.Errorf("generate %s seed: %w", format, err), 400)
-			return
-		}
-		fileName := stdpath.Base(seed.Name) + "." + format
-		artifact := gin.H{
-			"format":    format,
-			"name":      fileName,
-			"file_name": fileName,
-			"data":      base64.StdEncoding.EncodeToString(data),
-			"seed_data": base64.StdEncoding.EncodeToString(data),
-			"size":      len(data),
-		}
-		if outputPath != "" {
-			dstDir, err := user.JoinPath(outputPath)
-			if err != nil {
-				common.ErrorResp(c, err, 403)
-				return
-			}
-			meta, metaErr := op.GetNearestMeta(dstDir)
-			if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) {
-				common.ErrorResp(c, metaErr, 500, true)
-				return
-			}
-			if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, dstDir)) || !common.CanWrite(user, meta, dstDir) {
-				common.ErrorResp(c, errs.PermissionDenied, 403)
-				return
-			}
-			fileStream := &stream.FileStream{
-				Ctx:    c.Request.Context(),
-				Obj:    &model.Object{Name: fileName, Size: int64(len(data)), Modified: time.Now()},
-				Reader: bytes.NewReader(data), Mimetype: "application/octet-stream",
-			}
-			if err = fs.PutDirectly(c.Request.Context(), dstDir, fileStream); err != nil {
-				common.ErrorResp(c, err, 500)
-				return
-			}
-			artifact["path"] = stdpath.Join(outputPath, fileName)
-		}
-		artifacts = append(artifacts, artifact)
-	}
-	keepCreatedShares = true
 	common.SuccessResp(c, gin.H{"artifacts": artifacts, "seed": seed})
 }
 
-func encodeGeneratedSeed(seed *torrent.Seed, format string, standardPieces []byte) ([]byte, error) {
-	if format != "torrent" {
-		return torrent.EncodeSeed(seed, format)
-	}
-	t := &torrent.Torrent{
-		Info:         torrent.TorrentInfo{Name: seed.Name, PieceLength: seed.PieceSize, Pieces: standardPieces},
-		Comment:      seed.Comment,
-		CreatedBy:    seed.CreatedBy,
-		CreationDate: time.Now().Unix(),
-		OpenList:     seed,
-	}
-	if len(seed.Trackers) > 0 {
-		t.Announce = seed.Trackers[0]
-		for _, tracker := range seed.Trackers {
-			t.AnnounceList = append(t.AnnounceList, []string{tracker})
-		}
-	}
-	if len(seed.Files) == 1 {
-		file := seed.Files[0]
-		t.Info.Name = stdpath.Base(file.Path)
-		t.Info.Length = file.Size
-		t.Info.MD5Sum = file.Hashes.MD5
-		if file.CASSliceMD5 != "" {
-			t.SetCASInfo(&torrent.CASInfo{
-				FileMD5: strings.ToUpper(file.Hashes.MD5), SliceMD5: strings.ToUpper(file.CASSliceMD5),
-				SliceSize: torrent.DefaultPieceSize, Cloud: "189",
-			})
-		} else if seed.PieceSize == torrent.DefaultPieceSize && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 {
-			t.SetCASInfo(torrent.BuildCASInfoFromMD5s(file.Hashes.MD5, file.Hashes.Pieces.MD5, torrent.DefaultPieceSize))
-		}
-	} else {
-		for _, file := range seed.Files {
-			t.Info.Files = append(t.Info.Files, torrent.TorrentFile{Length: file.Size, Path: strings.Split(file.Path, "/"), MD5Sum: file.Hashes.MD5})
-		}
-	}
-	return t.Encode()
-}
-
 // loadSeedDefaultTrackers returns the configured default tracker list, one per line.
 func loadSeedDefaultTrackers() []string {
 	raw := strings.TrimSpace(setting.GetStr(conf.SeedDefaultTrackers))