mirror of
https://github.com/OpenListTeam/OpenList.git
synced 2026-10-10 21:13:10 +08:00
Compare commits
28 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5be5bc7873 | |||
| 966164d3a0 | |||
| f828938095 | |||
| fe575686f1 | |||
| e73e69f05e | |||
| 6d17d37ee7 | |||
| f933d59ec4 | |||
| 79c1d9d721 | |||
| 6f2ba09df9 | |||
| 624fdd24e9 | |||
| 06423083c7 | |||
| 4903bf61a9 | |||
| d29baa0eaf | |||
| d684e45d63 | |||
| ac1192a8d9 | |||
| 172ef17421 | |||
| 85be4214ee | |||
| e10ebd2694 | |||
| 8467a3abe0 | |||
| aa42d9e5fd | |||
| 596e284fb7 | |||
| 2ed55487df | |||
| 2e6dd00d91 | |||
| 09b150a0ef | |||
| 238dbb66ec | |||
| 9c7ad84242 | |||
| a5e5048555 | |||
| b0f6919f86 |
@@ -0,0 +1,74 @@
|
||||
package _115
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
// RapidHashAlgos 返回 115 支持的秒传哈希算法(SHA1)
|
||||
func (d *Pan115) RapidHashAlgos() []utils.HashType {
|
||||
return []utils.HashType{*utils.SHA1}
|
||||
}
|
||||
|
||||
// RapidHashNeedsPieces 115 不需要分片哈希
|
||||
func (d *Pan115) RapidHashNeedsPieces() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// RapidUploadByHashes 使用种子中的 SHA1 哈希尝试秒传。
|
||||
//
|
||||
// 115 的秒传协议除了整文件 SHA1,还需要文件头部 128KB 的 SHA1(pre_hash),
|
||||
// 因此当内容源可用时会打开它来计算前置哈希;内容不可用时无法秒传。
|
||||
func (d *Pan115) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
|
||||
fullHash := strings.ToUpper(req.Whole.GetHash(utils.SHA1))
|
||||
if len(fullHash) != utils.SHA1.Width {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
if req.Open == nil {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
|
||||
src, err := req.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer src.Close()
|
||||
|
||||
const PreHashSize int64 = 128 * utils.KB
|
||||
hashSize := PreHashSize
|
||||
if req.Size < PreHashSize {
|
||||
hashSize = req.Size
|
||||
}
|
||||
reader, err := src.RangeRead(http_range.Range{Start: 0, Length: hashSize})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
preHash, err := utils.HashReader(utils.SHA1, reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
preHash = strings.ToUpper(preHash)
|
||||
|
||||
fastInfo, err := d.rapidUpload(req.Size, req.Name, dstDir.GetID(), preHash, fullHash, src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
matched, err := fastInfo.Ok()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !matched {
|
||||
return nil, errs.ErrRapidUploadFailed
|
||||
}
|
||||
f, err := d.getNewFileByPickCode(fastInfo.PickCode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
@@ -171,11 +171,22 @@ func (d *Open115) Get(ctx context.Context, path string) (model.Obj, error) {
|
||||
path = stdpath.Join(d.parentPath, path)
|
||||
resp, err := d.client.GetFolderInfoByPath(ctx, path)
|
||||
if err != nil {
|
||||
// SDK-level "object not found" (empty array response from API)
|
||||
if errors.Is(err, sdk.ErrObjectNotFound) {
|
||||
return d.getFromParent(ctx, path, "")
|
||||
}
|
||||
// API-level error response (State=false), treat as not found
|
||||
// since this is a path-lookup that can't resolve the target
|
||||
var apiErr *sdk.Error
|
||||
if errors.As(err, &apiErr) {
|
||||
return nil, errs.ObjectNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
// Handle null/empty response (e.g., API returns null data for non-existent path)
|
||||
if resp.FileID == "" {
|
||||
return nil, errs.ObjectNotFound
|
||||
}
|
||||
obj := &Obj{
|
||||
Fid: resp.FileID,
|
||||
Fn: resp.FileName,
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
package _123
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/base"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/go-resty/resty/v2"
|
||||
)
|
||||
|
||||
// RapidHashAlgos 返回 123 云盘支持的秒传哈希算法(MD5)
|
||||
func (d *Pan123) RapidHashAlgos() []utils.HashType {
|
||||
return []utils.HashType{*utils.MD5}
|
||||
}
|
||||
|
||||
// RapidHashNeedsPieces 123 云盘不需要分片哈希
|
||||
func (d *Pan123) RapidHashNeedsPieces() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// RapidUploadByHashes 使用种子中的 MD5(Etag)哈希尝试秒传。
|
||||
//
|
||||
// 123 的秒传即「上传请求返回 reuse=true」,无需真正传输内容。
|
||||
func (d *Pan123) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
|
||||
etag := req.Whole.GetHash(utils.MD5)
|
||||
if len(etag) < utils.MD5.Width {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
|
||||
duplicate := 0
|
||||
if overwrite {
|
||||
duplicate = 2
|
||||
}
|
||||
data := base.Json{
|
||||
"driveId": 0,
|
||||
"duplicate": duplicate,
|
||||
"etag": strings.ToLower(etag),
|
||||
"fileName": req.Name,
|
||||
"parentFileId": dstDir.GetID(),
|
||||
"size": req.Size,
|
||||
"type": 0,
|
||||
}
|
||||
|
||||
var resp UploadResp
|
||||
_, err := d.Request(UploadRequest, http.MethodPost, func(r *resty.Request) {
|
||||
r.SetBody(data).SetContext(ctx)
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// reuse=true 或未返回上传 Key 均视为秒传成功
|
||||
if !resp.Data.Reuse && resp.Data.Key != "" {
|
||||
return nil, errs.ErrHashMismatch
|
||||
}
|
||||
|
||||
return &model.ObjThumb{
|
||||
Object: model.Object{
|
||||
ID: strconv.FormatInt(resp.Data.FileId, 10),
|
||||
Name: req.Name,
|
||||
Size: req.Size,
|
||||
Modified: time.Now(),
|
||||
IsFolder: false,
|
||||
HashInfo: utils.NewHashInfo(utils.MD5, strings.ToLower(etag)),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package _123_open
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
// RapidHashAlgos 返回 123 开放平台支持的秒传哈希算法(SHA1)
|
||||
func (d *Open123) RapidHashAlgos() []utils.HashType {
|
||||
return []utils.HashType{*utils.SHA1}
|
||||
}
|
||||
|
||||
// RapidHashNeedsPieces 123 开放平台不需要分片哈希
|
||||
func (d *Open123) RapidHashNeedsPieces() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// RapidUploadByHashes 使用种子中的 SHA1 哈希尝试秒传
|
||||
func (d *Open123) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
|
||||
sha1Hash := req.Whole.GetHash(utils.SHA1)
|
||||
if len(sha1Hash) < utils.SHA1.Width {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
|
||||
parentID, err := strconv.ParseInt(dstDir.GetID(), 10, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
resp, err := d.sha1Reuse(parentID, req.Name, sha1Hash, req.Size, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !resp.Data.Reuse {
|
||||
return nil, errs.ErrHashMismatch
|
||||
}
|
||||
|
||||
return &model.ObjThumb{
|
||||
Object: model.Object{
|
||||
ID: strconv.FormatInt(resp.Data.FileID, 10),
|
||||
Name: req.Name,
|
||||
Size: req.Size,
|
||||
IsFolder: false,
|
||||
Path: path.Join(dstDir.GetPath(), req.Name),
|
||||
Modified: time.Now(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -6,7 +6,6 @@ import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/torrent"
|
||||
@@ -15,12 +14,8 @@ import (
|
||||
|
||||
// GenerateTorrent 根据上传过程中收集的哈希信息生成包含 CAS 扩展的 torrent 文件
|
||||
func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) {
|
||||
// 计算 sliceMD5
|
||||
sliceMD5 := fileMD5
|
||||
if len(sliceMD5s) > 1 {
|
||||
joined := strings.Join(sliceMD5s, "\n")
|
||||
sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined))
|
||||
}
|
||||
// 计算 sliceMD5(统一走规范实现)
|
||||
sliceMD5 := torrent.SliceMD5FromPieces(sliceMD5s, fileMD5)
|
||||
|
||||
t := torrent.NewTorrent(fileName, fileSize, fileMD5)
|
||||
t.Info.PieceLength = sliceSize
|
||||
@@ -30,7 +25,7 @@ func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s
|
||||
SliceMD5: sliceMD5,
|
||||
SliceMD5s: sliceMD5s,
|
||||
SliceSize: sliceSize,
|
||||
Cloud: "189",
|
||||
Cloud: torrent.Cloud189,
|
||||
})
|
||||
|
||||
return t.Encode()
|
||||
@@ -95,7 +90,7 @@ func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64,
|
||||
sliceSize = torrent.DefaultPieceSize
|
||||
}
|
||||
|
||||
hw := torrent.NewHashWriter(sliceSize, sliceSize)
|
||||
hw := torrent.NewHashWriter(sliceSize, sliceSize, fileSize)
|
||||
|
||||
buf := make([]byte, 32*1024)
|
||||
for {
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
package _189_tv
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
// RapidHashAlgos 返回 189 电视支持的秒传哈希算法(MD5)
|
||||
func (d *Cloud189TV) RapidHashAlgos() []utils.HashType {
|
||||
return []utils.HashType{*utils.MD5}
|
||||
}
|
||||
|
||||
// RapidHashNeedsPieces 189 电视不需要分片哈希
|
||||
func (d *Cloud189TV) RapidHashNeedsPieces() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// RapidUploadByHashes 使用种子中的 MD5 哈希尝试秒传
|
||||
func (d *Cloud189TV) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
|
||||
md5Hash := req.Whole.GetHash(utils.MD5)
|
||||
if len(md5Hash) < utils.MD5.Width {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
|
||||
stream := driver.NewSeedHashStream(req)
|
||||
obj, err := d.RapidUpload(ctx, dstDir, stream, d.isFamily(), overwrite)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package _189pc
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
// RapidHashAlgos 返回 189pc 支持的秒传哈希算法(MD5)
|
||||
func (d *Cloud189PC) RapidHashAlgos() []utils.HashType {
|
||||
return []utils.HashType{*utils.MD5}
|
||||
}
|
||||
|
||||
// RapidHashNeedsPieces 189pc 的 CAS 秒传依赖分片 MD5
|
||||
func (d *Cloud189PC) RapidHashNeedsPieces() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// RapidUploadByHashes 使用种子中的 MD5 哈希尝试秒传
|
||||
func (d *Cloud189PC) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
|
||||
md5Hash := req.Whole.GetHash(utils.MD5)
|
||||
if len(md5Hash) < utils.MD5.Width {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
|
||||
stream := driver.NewSeedHashStream(req)
|
||||
obj, err := d.RapidUpload(ctx, dstDir, stream, d.isFamily(), overwrite)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
+17
-109
@@ -7,7 +7,6 @@ import (
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -28,12 +27,8 @@ import (
|
||||
// fileName: 文件名
|
||||
// fileSize: 文件大小
|
||||
func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, pieceHashes []byte) ([]byte, error) {
|
||||
// 计算 sliceMD5
|
||||
sliceMD5 := fileMD5
|
||||
if len(sliceMD5s) > 1 {
|
||||
joined := strings.Join(sliceMD5s, "\n")
|
||||
sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined))
|
||||
}
|
||||
// 计算 sliceMD5(统一走规范实现)
|
||||
sliceMD5 := torrent.SliceMD5FromPieces(sliceMD5s, fileMD5)
|
||||
|
||||
t := torrent.NewTorrent(fileName, fileSize, fileMD5)
|
||||
t.Info.PieceLength = sliceSize
|
||||
@@ -43,7 +38,7 @@ func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s
|
||||
SliceMD5: sliceMD5,
|
||||
SliceMD5s: sliceMD5s,
|
||||
SliceSize: sliceSize,
|
||||
Cloud: "189",
|
||||
Cloud: torrent.Cloud189,
|
||||
})
|
||||
|
||||
return t.Encode()
|
||||
@@ -69,101 +64,18 @@ func (y *Cloud189PC) RapidUploadFromTorrent(ctx context.Context, dstDir model.Ob
|
||||
fileName := t.Info.Name
|
||||
fileSize := t.GetTotalSize()
|
||||
|
||||
// 统一 MD5 为大写(与正常上传保持一致,天翼云盘要求大写)
|
||||
fileMD5Upper := strings.ToUpper(cas.FileMD5)
|
||||
|
||||
// 优先使用 torrent 中嵌入的分片大小,与生成时保持一致
|
||||
sliceSize := cas.SliceSize
|
||||
if sliceSize <= 0 {
|
||||
sliceSize = partSize(fileSize)
|
||||
// 优先使用 torrent 中嵌入的分片 MD5 与大写整文件 MD5
|
||||
fileMD5 := strings.ToUpper(cas.FileMD5)
|
||||
sliceMD5s := make([]string, len(cas.SliceMD5s))
|
||||
for i, s := range cas.SliceMD5s {
|
||||
sliceMD5s[i] = strings.ToUpper(s)
|
||||
}
|
||||
|
||||
// 计算 sliceMd5(与上传时一致的算法)
|
||||
// 优先使用 torrent 中已有的 SliceMD5;仅当有多分片列表时才重新计算
|
||||
sliceMd5Hex := strings.ToUpper(cas.SliceMD5)
|
||||
if sliceMd5Hex == "" {
|
||||
sliceMd5Hex = fileMD5Upper
|
||||
}
|
||||
if len(cas.SliceMD5s) > 1 {
|
||||
// 分片 MD5 也需要统一大写后再拼接计算
|
||||
upperSliceMD5s := make([]string, len(cas.SliceMD5s))
|
||||
for i, s := range cas.SliceMD5s {
|
||||
upperSliceMD5s[i] = strings.ToUpper(s)
|
||||
}
|
||||
sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(upperSliceMD5s, "\n")))
|
||||
}
|
||||
|
||||
// 使用与 Web 端一致的三步秒传流程
|
||||
fullUrl := "https://upload.cloud.189.cn"
|
||||
if isFamily {
|
||||
fullUrl += "/family"
|
||||
} else {
|
||||
fullUrl += "/person"
|
||||
}
|
||||
|
||||
// Step 1: initMultiUpload(不传 fileMd5/sliceMd5,只传 lazyCheck)
|
||||
initParams := Params{
|
||||
"parentFolderId": dstDir.GetID(),
|
||||
"fileName": url.QueryEscape(fileName),
|
||||
"fileSize": fmt.Sprint(fileSize),
|
||||
"sliceSize": fmt.Sprint(sliceSize),
|
||||
"lazyCheck": "1",
|
||||
}
|
||||
if isFamily {
|
||||
initParams.Set("familyId", y.FamilyID)
|
||||
}
|
||||
|
||||
var uploadInfo InitMultiUploadResp
|
||||
_, err = y.request(fullUrl+"/initMultiUpload", "GET", func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
}, initParams, &uploadInfo, isFamily)
|
||||
// 复用统一的 CAS 秒传核心实现
|
||||
respObj, err := y.rapidUploadByCAS(ctx, dstDir, fileName, fileSize, fileMD5, sliceMD5s, cas.SliceSize, overwrite)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("initMultiUpload 失败: %w", err)
|
||||
}
|
||||
|
||||
uploadFileId := uploadInfo.Data.UploadFileID
|
||||
|
||||
// Step 2: checkTransSecond(用 fileMd5 + sliceMd5 + uploadFileId 检查秒传)
|
||||
checkParams := Params{
|
||||
"fileMd5": fileMD5Upper,
|
||||
"sliceMd5": sliceMd5Hex,
|
||||
"uploadFileId": uploadFileId,
|
||||
}
|
||||
|
||||
var checkResp struct {
|
||||
Data struct {
|
||||
FileDataExists int `json:"fileDataExists"`
|
||||
} `json:"data"`
|
||||
}
|
||||
_, err = y.request(fullUrl+"/checkTransSecond", "GET", func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
}, checkParams, &checkResp, isFamily)
|
||||
if err != nil {
|
||||
utils.Log.Errorf("[RapidUpload] checkTransSecond 失败: uploadFileId=%s, err=%v", uploadFileId, err)
|
||||
return nil, fmt.Errorf("秒传检查失败: %w", err)
|
||||
}
|
||||
|
||||
if checkResp.Data.FileDataExists != 1 {
|
||||
return nil, fmt.Errorf("秒传失败:云端不存在该文件(fileMD5=%s, sliceMD5=%s, size=%d)", fileMD5Upper, sliceMd5Hex, fileSize)
|
||||
}
|
||||
|
||||
// Step 3: commitMultiUploadFile(传 fileMd5 + sliceMd5)
|
||||
|
||||
var resp CommitMultiUploadFileResp
|
||||
commitParams := Params{
|
||||
"uploadFileId": uploadFileId,
|
||||
"fileMd5": fileMD5Upper,
|
||||
"sliceMd5": sliceMd5Hex,
|
||||
"lazyCheck": "1",
|
||||
"opertype": IF(overwrite, "3", "1"),
|
||||
}
|
||||
|
||||
_, err = y.request(fullUrl+"/commitMultiUploadFile", "GET", func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
}, commitParams, &resp, isFamily)
|
||||
if err != nil {
|
||||
utils.Log.Errorf("[RapidUpload] commitMultiUploadFile 失败: uploadFileId=%s, err=%v", uploadFileId, err)
|
||||
return nil, fmt.Errorf("提交上传失败: %w", err)
|
||||
utils.Log.Errorf("[RapidUpload] 秒传失败: fileMD5=%s, err=%v", fileMD5, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 秒传成功后,将 torrent 文件上传到目标目录(异步,不影响秒传结果)
|
||||
@@ -196,7 +108,7 @@ func (y *Cloud189PC) RapidUploadFromTorrent(ctx context.Context, dstDir model.Ob
|
||||
}()
|
||||
}
|
||||
|
||||
return resp.toFile(), nil
|
||||
return respObj, nil
|
||||
}
|
||||
|
||||
// ComputeTorrentFromReader 从 io.Reader 计算并生成 torrent 文件
|
||||
@@ -206,7 +118,7 @@ func ComputeTorrentFromReader(reader io.Reader, fileName string, fileSize int64,
|
||||
sliceSize = torrent.DefaultPieceSize
|
||||
}
|
||||
|
||||
hw := torrent.NewHashWriter(sliceSize, sliceSize)
|
||||
hw := torrent.NewHashWriter(sliceSize, sliceSize, fileSize)
|
||||
|
||||
buf := make([]byte, 32*1024)
|
||||
for {
|
||||
@@ -259,12 +171,8 @@ func InjectCASIntoTorrent(torrentData []byte, fileMD5 string, sliceMD5s []string
|
||||
return nil, fmt.Errorf("解析 torrent 失败: %w", err)
|
||||
}
|
||||
|
||||
// 计算 sliceMD5
|
||||
sliceMD5 := fileMD5
|
||||
if len(sliceMD5s) > 1 {
|
||||
joined := strings.Join(sliceMD5s, "\n")
|
||||
sliceMD5 = strings.ToUpper(torrent.GetMD5Str(joined))
|
||||
}
|
||||
// 计算 sliceMD5(统一走规范实现)
|
||||
sliceMD5 := torrent.SliceMD5FromPieces(sliceMD5s, fileMD5)
|
||||
|
||||
// 注入 CAS 信息
|
||||
t.SetCASInfo(&torrent.CASInfo{
|
||||
@@ -272,7 +180,7 @@ func InjectCASIntoTorrent(torrentData []byte, fileMD5 string, sliceMD5s []string
|
||||
SliceMD5: sliceMD5,
|
||||
SliceMD5s: sliceMD5s,
|
||||
SliceSize: sliceSize,
|
||||
Cloud: "189",
|
||||
Cloud: torrent.Cloud189,
|
||||
})
|
||||
|
||||
// 同时更新 info 中的 md5sum 字段
|
||||
|
||||
@@ -1329,6 +1329,102 @@ func (y *Cloud189PC) OldUploadCommit(ctx context.Context, fileCommitUrl string,
|
||||
return resp.toFile(), nil
|
||||
}
|
||||
|
||||
// rapidUploadByCAS 使用 MD5 + 分片 MD5(CAS)执行天翼云盘秒传。
|
||||
//
|
||||
// 流程与 Web 端一致:
|
||||
// 1. initMultiUpload(仅传 lazyCheck=1)
|
||||
// 2. checkTransSecond(用 fileMd5 + sliceMd5 检查云端是否已存在文件数据)
|
||||
// 3. commitMultiUploadFile(提交并返回文件对象)
|
||||
func (y *Cloud189PC) rapidUploadByCAS(ctx context.Context, dstDir model.Obj, fileName string, fileSize int64, fileMD5 string, sliceMD5s []string, sliceSize int64, overwrite bool) (model.Obj, error) {
|
||||
isFamily := y.isFamily()
|
||||
|
||||
// 统一 MD5 为大写(天翼云盘要求大写)
|
||||
fileMD5Upper := strings.ToUpper(fileMD5)
|
||||
|
||||
// 优先使用传入的分片大小,否则按文件大小推导
|
||||
if sliceSize <= 0 {
|
||||
sliceSize = partSize(fileSize)
|
||||
}
|
||||
|
||||
// 计算 sliceMd5(与上传时一致的算法)
|
||||
sliceMd5Hex := fileMD5Upper
|
||||
if len(sliceMD5s) > 1 {
|
||||
upperSliceMD5s := make([]string, len(sliceMD5s))
|
||||
for i, s := range sliceMD5s {
|
||||
upperSliceMD5s[i] = strings.ToUpper(s)
|
||||
}
|
||||
sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(upperSliceMD5s, "\n")))
|
||||
} else if len(sliceMD5s) == 1 {
|
||||
sliceMd5Hex = strings.ToUpper(sliceMD5s[0])
|
||||
}
|
||||
|
||||
fullUrl := "https://upload.cloud.189.cn"
|
||||
if isFamily {
|
||||
fullUrl += "/family"
|
||||
} else {
|
||||
fullUrl += "/person"
|
||||
}
|
||||
|
||||
// Step 1: initMultiUpload(不传 fileMd5/sliceMd5,只传 lazyCheck)
|
||||
initParams := Params{
|
||||
"parentFolderId": dstDir.GetID(),
|
||||
"fileName": url.QueryEscape(fileName),
|
||||
"fileSize": fmt.Sprint(fileSize),
|
||||
"sliceSize": fmt.Sprint(sliceSize),
|
||||
"lazyCheck": "1",
|
||||
}
|
||||
if isFamily {
|
||||
initParams.Set("familyId", y.FamilyID)
|
||||
}
|
||||
|
||||
var uploadInfo InitMultiUploadResp
|
||||
if _, err := y.request(fullUrl+"/initMultiUpload", "GET", func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
}, initParams, &uploadInfo, isFamily); err != nil {
|
||||
return nil, fmt.Errorf("initMultiUpload 失败: %w", err)
|
||||
}
|
||||
uploadFileId := uploadInfo.Data.UploadFileID
|
||||
|
||||
// Step 2: checkTransSecond(用 fileMd5 + sliceMd5 + uploadFileId 检查秒传)
|
||||
checkParams := Params{
|
||||
"fileMd5": fileMD5Upper,
|
||||
"sliceMd5": sliceMd5Hex,
|
||||
"uploadFileId": uploadFileId,
|
||||
}
|
||||
|
||||
var checkResp struct {
|
||||
Data struct {
|
||||
FileDataExists int `json:"fileDataExists"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if _, err := y.request(fullUrl+"/checkTransSecond", "GET", func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
}, checkParams, &checkResp, isFamily); err != nil {
|
||||
return nil, fmt.Errorf("秒传检查失败: %w", err)
|
||||
}
|
||||
if checkResp.Data.FileDataExists != 1 {
|
||||
return nil, fmt.Errorf("秒传失败:云端不存在该文件(fileMD5=%s, sliceMD5=%s, size=%d)", fileMD5Upper, sliceMd5Hex, fileSize)
|
||||
}
|
||||
|
||||
// Step 3: commitMultiUploadFile(传 fileMd5 + sliceMd5)
|
||||
commitParams := Params{
|
||||
"uploadFileId": uploadFileId,
|
||||
"fileMd5": fileMD5Upper,
|
||||
"sliceMd5": sliceMd5Hex,
|
||||
"lazyCheck": "1",
|
||||
"opertype": IF(overwrite, "3", "1"),
|
||||
}
|
||||
|
||||
var resp CommitMultiUploadFileResp
|
||||
if _, err := y.request(fullUrl+"/commitMultiUploadFile", "GET", func(req *resty.Request) {
|
||||
req.SetContext(ctx)
|
||||
}, commitParams, &resp, isFamily); err != nil {
|
||||
return nil, fmt.Errorf("提交上传失败: %w", err)
|
||||
}
|
||||
|
||||
return resp.toFile(), nil
|
||||
}
|
||||
|
||||
func (y *Cloud189PC) isFamily() bool {
|
||||
return y.Type == "family"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package aliyundrive_open
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/base"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/go-resty/resty/v2"
|
||||
)
|
||||
|
||||
// RapidHashAlgos 返回阿里云盘支持的秒传哈希算法(SHA1)
|
||||
func (d *AliyundriveOpen) RapidHashAlgos() []utils.HashType {
|
||||
return []utils.HashType{*utils.SHA1}
|
||||
}
|
||||
|
||||
// RapidHashNeedsPieces 阿里云盘不需要分片哈希
|
||||
func (d *AliyundriveOpen) RapidHashNeedsPieces() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// RapidUploadByHashes 使用种子中的 SHA1 哈希尝试秒传。
|
||||
//
|
||||
// 阿里云盘的秒传还需要 proof_code(按 proof range 读取的一段内容),
|
||||
// 因此当内容源不可用时无法完成秒传。
|
||||
func (d *AliyundriveOpen) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
|
||||
sha1Hash := req.Whole.GetHash(utils.SHA1)
|
||||
if len(sha1Hash) < utils.SHA1.Width {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
if req.Open == nil {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
|
||||
stream, err := req.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer stream.Close()
|
||||
|
||||
proofCode, err := d.calProofCode(stream)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var resp CreateResp
|
||||
_, err = d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(r *resty.Request) {
|
||||
r.SetBody(base.Json{
|
||||
"drive_id": d.DriveId,
|
||||
"parent_file_id": dstDir.GetID(),
|
||||
"name": req.Name,
|
||||
"type": "file",
|
||||
"check_name_mode": "auto_rename",
|
||||
"size": req.Size,
|
||||
"content_hash": sha1Hash,
|
||||
"content_hash_name": "sha1",
|
||||
"proof_version": "v1",
|
||||
"proof_code": proofCode,
|
||||
}).SetResult(&resp)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !resp.RapidUpload {
|
||||
return nil, errs.ErrHashMismatch
|
||||
}
|
||||
|
||||
if resp.FileId != "" {
|
||||
obj, err := d.completeUpload(ctx, resp.FileId, resp.UploadId)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
return &model.ObjThumb{
|
||||
Object: model.Object{
|
||||
Name: req.Name,
|
||||
Size: req.Size,
|
||||
Modified: time.Now(),
|
||||
IsFolder: false,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package baidu_netdisk
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
// RapidHashAlgos 返回百度网盘支持的秒传哈希算法(MD5)
|
||||
func (d *BaiduNetdisk) RapidHashAlgos() []utils.HashType {
|
||||
return []utils.HashType{*utils.MD5}
|
||||
}
|
||||
|
||||
// RapidHashNeedsPieces 百度网盘不需要分片哈希
|
||||
func (d *BaiduNetdisk) RapidHashNeedsPieces() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// RapidUploadByHashes 使用种子中的 MD5 哈希尝试秒传
|
||||
func (d *BaiduNetdisk) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
|
||||
md5Hash := req.Whole.GetHash(utils.MD5)
|
||||
if len(md5Hash) < utils.MD5.Width {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
|
||||
stream := driver.NewSeedHashStream(req)
|
||||
obj, err := d.PutRapid(ctx, dstDir, stream)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return obj, nil
|
||||
}
|
||||
@@ -245,6 +245,7 @@ func mFile2Object(file MFile) *model.ObjThumbURL {
|
||||
Ctime: ctime,
|
||||
IsFolder: false,
|
||||
Size: file.Size,
|
||||
HashInfo: utils.NewHashInfo(utils.MD5, file.MD5),
|
||||
},
|
||||
Thumbnail: model.Thumbnail{
|
||||
Thumbnail: file.ThumbnailURL,
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
package pikpak
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/base"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
|
||||
"github.com/go-resty/resty/v2"
|
||||
)
|
||||
|
||||
// RapidHashAlgos 返回 PikPak 支持的秒传哈希算法(GCID)
|
||||
func (d *PikPak) RapidHashAlgos() []utils.HashType {
|
||||
return []utils.HashType{*hash_extend.GCID}
|
||||
}
|
||||
|
||||
// RapidHashNeedsPieces PikPak 不需要分片哈希
|
||||
func (d *PikPak) RapidHashNeedsPieces() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// RapidUploadByHashes 使用种子中的 GCID 哈希尝试秒传
|
||||
func (d *PikPak) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
|
||||
gcid := req.Whole.GetHash(hash_extend.GCID)
|
||||
if len(gcid) < hash_extend.GCID.Width {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
|
||||
var resp UploadTaskData
|
||||
_, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(r *resty.Request) {
|
||||
r.SetContext(ctx).SetBody(base.Json{
|
||||
"kind": "drive#file",
|
||||
"name": req.Name,
|
||||
"size": req.Size,
|
||||
"hash": strings.ToUpper(gcid),
|
||||
"upload_type": "UPLOAD_TYPE_RESUMABLE",
|
||||
"objProvider": base.Json{"provider": "UPLOAD_TYPE_UNKNOWN"},
|
||||
"parent_id": dstDir.GetID(),
|
||||
"folder_type": "NORMAL",
|
||||
})
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 秒传成功时不会返回 Resumable
|
||||
if resp.Resumable == nil {
|
||||
file := fileToObj(resp.File)
|
||||
return file, nil
|
||||
}
|
||||
|
||||
return nil, errs.ErrHashMismatch
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package quark_open
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
// RapidHashAlgos 返回夸克网盘支持的秒传哈希算法(MD5 + SHA1)
|
||||
func (d *QuarkOpen) RapidHashAlgos() []utils.HashType {
|
||||
return []utils.HashType{*utils.MD5, *utils.SHA1}
|
||||
}
|
||||
|
||||
// RapidHashNeedsPieces 夸克网盘不需要分片哈希
|
||||
func (d *QuarkOpen) RapidHashNeedsPieces() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// RapidUploadByHashes 使用种子中的 MD5/SHA1 哈希尝试秒传。
|
||||
//
|
||||
// 夸克网盘的预上传需要 proof_code(按 proof range 读取的一段内容),
|
||||
// 因此当内容源不可用时无法完成秒传。
|
||||
func (d *QuarkOpen) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
|
||||
md5Hash := req.Whole.GetHash(utils.MD5)
|
||||
sha1Hash := req.Whole.GetHash(utils.SHA1)
|
||||
if len(md5Hash) < utils.MD5.Width || len(sha1Hash) < utils.SHA1.Width {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
if req.Open == nil {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
|
||||
stream, err := req.Open()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer stream.Close()
|
||||
|
||||
pre, err := d.upPre(ctx, stream, dstDir.GetID(), md5Hash, sha1Hash)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !pre.Data.Finish {
|
||||
return nil, errs.ErrHashMismatch
|
||||
}
|
||||
|
||||
return &model.ObjThumb{
|
||||
Object: model.Object{
|
||||
ID: pre.Data.Fid,
|
||||
Name: req.Name,
|
||||
Size: req.Size,
|
||||
IsFolder: false,
|
||||
Modified: time.Now(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package quark_open
|
||||
|
||||
import (
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -57,6 +58,7 @@ func fileToObj(f File) *model.ObjThumb {
|
||||
Modified: time.UnixMilli(f.UpdatedAt),
|
||||
IsFolder: f.FileType == "0",
|
||||
Ctime: time.UnixMilli(f.CreatedAt),
|
||||
HashInfo: utils.NewHashInfo(utils.SHA1, f.ContentHash),
|
||||
},
|
||||
Thumbnail: model.Thumbnail{Thumbnail: f.ThumbnailURL},
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
package thunder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/base"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
|
||||
"github.com/go-resty/resty/v2"
|
||||
)
|
||||
|
||||
// RapidHashAlgos 返回迅雷支持的秒传哈希算法(GCID)
|
||||
func (xc *XunLeiCommon) RapidHashAlgos() []utils.HashType {
|
||||
return []utils.HashType{*hash_extend.GCID}
|
||||
}
|
||||
|
||||
// RapidHashNeedsPieces 迅雷不需要分片哈希
|
||||
func (xc *XunLeiCommon) RapidHashNeedsPieces() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// RapidUploadByHashes 使用种子中的 GCID 哈希尝试秒传
|
||||
func (xc *XunLeiCommon) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
|
||||
gcid := req.Whole.GetHash(hash_extend.GCID)
|
||||
if len(gcid) < hash_extend.GCID.Width {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
|
||||
var resp UploadTaskResponse
|
||||
_, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) {
|
||||
r.SetContext(ctx)
|
||||
r.SetBody(&base.Json{
|
||||
"kind": FILE,
|
||||
"parent_id": dstDir.GetID(),
|
||||
"name": req.Name,
|
||||
"size": req.Size,
|
||||
"hash": gcid,
|
||||
"upload_type": UPLOAD_TYPE_RESUMABLE,
|
||||
"space": xc.Space,
|
||||
})
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 秒传成功(UploadType != UPLOAD_TYPE_RESUMABLE)
|
||||
if resp.UploadType != UPLOAD_TYPE_RESUMABLE {
|
||||
return &resp.File, nil
|
||||
}
|
||||
|
||||
return nil, errs.ErrHashMismatch
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package thunder_browser
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/base"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
|
||||
"github.com/go-resty/resty/v2"
|
||||
)
|
||||
|
||||
// RapidHashAlgos 返回迅雷浏览器支持的秒传哈希算法(GCID)
|
||||
func (xc *XunLeiBrowserCommon) RapidHashAlgos() []utils.HashType {
|
||||
return []utils.HashType{*hash_extend.GCID}
|
||||
}
|
||||
|
||||
// RapidHashNeedsPieces 迅雷浏览器不需要分片哈希
|
||||
func (xc *XunLeiBrowserCommon) RapidHashNeedsPieces() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// RapidUploadByHashes 使用种子中的 GCID 哈希尝试秒传
|
||||
func (xc *XunLeiBrowserCommon) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
|
||||
gcid := req.Whole.GetHash(hash_extend.GCID)
|
||||
if len(gcid) < hash_extend.GCID.Width {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
|
||||
var resp UploadTaskResponse
|
||||
_, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) {
|
||||
r.SetContext(ctx)
|
||||
r.SetBody(&base.Json{
|
||||
"kind": FILE,
|
||||
"parent_id": dstDir.GetID(),
|
||||
"name": req.Name,
|
||||
"size": req.Size,
|
||||
"hash": gcid,
|
||||
"upload_type": UPLOAD_TYPE_RESUMABLE,
|
||||
})
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 秒传成功(UploadType != UPLOAD_TYPE_RESUMABLE)
|
||||
if resp.UploadType != UPLOAD_TYPE_RESUMABLE {
|
||||
return &resp.File, nil
|
||||
}
|
||||
|
||||
return nil, errs.ErrHashMismatch
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
package thunderx
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/base"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
|
||||
"github.com/go-resty/resty/v2"
|
||||
)
|
||||
|
||||
// RapidHashAlgos 返回迅雷X支持的秒传哈希算法(GCID)
|
||||
func (xc *XunLeiXCommon) RapidHashAlgos() []utils.HashType {
|
||||
return []utils.HashType{*hash_extend.GCID}
|
||||
}
|
||||
|
||||
// RapidHashNeedsPieces 迅雷X不需要分片哈希
|
||||
func (xc *XunLeiXCommon) RapidHashNeedsPieces() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// RapidUploadByHashes 使用种子中的 GCID 哈希尝试秒传
|
||||
func (xc *XunLeiXCommon) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
|
||||
gcid := req.Whole.GetHash(hash_extend.GCID)
|
||||
if len(gcid) < hash_extend.GCID.Width {
|
||||
return nil, errs.ErrUnavailableHash
|
||||
}
|
||||
|
||||
var resp UploadTaskResponse
|
||||
_, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) {
|
||||
r.SetContext(ctx)
|
||||
r.SetBody(&base.Json{
|
||||
"kind": FILE,
|
||||
"parent_id": dstDir.GetID(),
|
||||
"name": req.Name,
|
||||
"size": req.Size,
|
||||
"hash": gcid,
|
||||
"upload_type": UPLOAD_TYPE_RESUMABLE,
|
||||
})
|
||||
}, &resp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// 秒传成功(UploadType != UPLOAD_TYPE_RESUMABLE)
|
||||
if resp.UploadType != UPLOAD_TYPE_RESUMABLE {
|
||||
return &resp.File, nil
|
||||
}
|
||||
|
||||
return nil, errs.ErrHashMismatch
|
||||
}
|
||||
@@ -187,6 +187,14 @@ func InitialSettings() []model.SettingItem {
|
||||
{Key: conf.HandleHookAfterWriting, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE},
|
||||
{Key: conf.HandleHookRateLimit, Value: "0", Type: conf.TypeNumber, Group: model.GLOBAL, Flag: model.PRIVATE},
|
||||
{Key: conf.IgnoreSystemFiles, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `When enabled, ignores common system files during upload (.DS_Store, desktop.ini, Thumbs.db, and files starting with ._)`},
|
||||
{Key: conf.SeedSiteURL, Value: "", Type: conf.TypeString, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Public base URL embedded in generated transfer seed sources when configured`},
|
||||
{Key: conf.SeedDefaultMatrix, Value: `{"md5":{"whole":true,"pieces":false},"sha1":{"whole":true,"pieces":false},"sha256":{"whole":true,"pieces":false}}`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Default right-click hash matrix for transfer seed generation`},
|
||||
{Key: conf.SeedFormatPolicies, Value: `{"oss":"off","torrent":"off","cas":"off"}`, Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE},
|
||||
{Key: conf.SeedDefaultFormat, Value: "oss", Type: conf.TypeSelect, Options: "oss,torrent,cas", Group: model.GLOBAL, Flag: model.PRIVATE},
|
||||
{Key: conf.SeedSingleDirectPreview, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC},
|
||||
{Key: conf.SeedCASDirectAccess, Value: "false", Type: conf.TypeBool, Group: model.GLOBAL, Flag: model.PUBLIC, Help: `When opening a single-file CAS seed, immediately rapid-upload it into the same folder and preview the restored file`},
|
||||
{Key: conf.SeedAutoGeneratePolicy, Value: "off", Type: conf.TypeSelect, Options: "off,on", Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Global upload sidecar policy; storage-specific inheritance can be layered without changing the safe default`},
|
||||
{Key: conf.SeedDefaultTrackers, Value: "", Type: conf.TypeText, Group: model.GLOBAL, Flag: model.PRIVATE, Help: `Default tracker list offered when generating torrent seeds (one tracker per line)`},
|
||||
|
||||
// single settings
|
||||
{Key: conf.Token, Value: token, Type: conf.TypeString, Group: model.SINGLE, Flag: model.PRIVATE},
|
||||
|
||||
@@ -49,4 +49,5 @@ func InitTaskManager() {
|
||||
op.RegisterSettingChangingCallback(func() {
|
||||
fs.ArchiveContentUploadTaskManager.SetWorkersNumActive(taskFilterNegative(setting.GetInt(conf.TaskDecompressUploadThreadsNum, conf.Conf.Tasks.DecompressUpload.Workers)))
|
||||
})
|
||||
fs.SeedGenerateTaskManager = tache.NewManager[*fs.SeedGenerateTask](tache.WithWorks(setting.GetInt(conf.TaskUploadThreadsNum, conf.Conf.Tasks.Upload.Workers)), tache.WithMaxRetry(conf.Conf.Tasks.Upload.MaxRetry)) //seed generation will not support persist
|
||||
}
|
||||
|
||||
@@ -60,6 +60,16 @@ const (
|
||||
HandleHookRateLimit = "handle_hook_rate_limit"
|
||||
IgnoreSystemFiles = "ignore_system_files"
|
||||
|
||||
// transfer seeds
|
||||
SeedSiteURL = "seed_site_url"
|
||||
SeedDefaultMatrix = "seed_default_matrix"
|
||||
SeedFormatPolicies = "seed_format_policies"
|
||||
SeedDefaultFormat = "seed_default_format"
|
||||
SeedSingleDirectPreview = "seed_single_direct_preview"
|
||||
SeedCASDirectAccess = "seed_cas_direct_access"
|
||||
SeedAutoGeneratePolicy = "seed_auto_generate_policy"
|
||||
SeedDefaultTrackers = "seed_default_trackers"
|
||||
|
||||
// index
|
||||
SearchIndex = "search_index"
|
||||
AutoUpdateIndex = "auto_update_index"
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
type Driver interface {
|
||||
@@ -218,3 +219,54 @@ type DirectUploader interface {
|
||||
// return errs.NotImplement if the driver does not support the given direct upload tool
|
||||
GetDirectUploadInfo(ctx context.Context, tool string, dstDir model.Obj, fileName string, fileSize int64) (any, error)
|
||||
}
|
||||
|
||||
// SeedRapidUploadRequest carries everything a driver needs to attempt a
|
||||
// hash-driven rapid upload (秒传/CAS) without transferring the full content.
|
||||
//
|
||||
// It is populated from a parsed transfer seed, so that any cloud drive
|
||||
// supporting hash-based rapid upload can be used as a seed save target.
|
||||
type SeedRapidUploadRequest struct {
|
||||
// Name is the target file name.
|
||||
Name string
|
||||
// Size is the total file size in bytes.
|
||||
Size int64
|
||||
// Whole holds whole-file hashes indexed by algorithm (e.g. utils.MD5,
|
||||
// utils.SHA1). It must never be nil; individual entries may be empty.
|
||||
Whole *utils.HashInfo
|
||||
// SliceSize is the per-slice/piece size in bytes (0 when unknown).
|
||||
SliceSize int64
|
||||
// SliceMD5s is the ordered per-slice MD5 list (used by 189pc-style CAS).
|
||||
SliceMD5s []string
|
||||
// SliceSHA1s is the ordered per-slice SHA1 list (used by SHA1-piece drives).
|
||||
SliceSHA1s []string
|
||||
// Open lazily yields the file content as a model.FileStreamer. Drivers whose
|
||||
// rapid-upload protocol needs partial or full content (e.g. a leading
|
||||
// pre-hash or a proof-code) may call it; hash-only drivers may ignore it.
|
||||
// Open may be nil when no content source is available, in which case
|
||||
// drivers that strictly require content must fail gracefully.
|
||||
Open func() (model.FileStreamer, error)
|
||||
}
|
||||
|
||||
// SeedRapidUploader is an optional capability interface implemented by drivers
|
||||
// that can perform a "rapid upload" (秒传/CAS) driven by precomputed hashes
|
||||
// instead of a full content transfer.
|
||||
//
|
||||
// It generalizes the previous hard-coded 189pc-specific CAS path so that any
|
||||
// cloud drive supporting hash-based rapid upload (e.g. 189pc via MD5+slice MD5,
|
||||
// 115/aliyundrive_open via SHA1) can be used as a transfer-seed save target.
|
||||
type SeedRapidUploader interface {
|
||||
// RapidUploadByHashes attempts a rapid upload of a file into dstDir.
|
||||
//
|
||||
// Implementations should return errs.NotImplement (or a descriptive error)
|
||||
// when the required hash is missing, the file does not exist remotely, or
|
||||
// the rapid upload cannot be confirmed.
|
||||
RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *SeedRapidUploadRequest, overwrite bool) (model.Obj, error)
|
||||
|
||||
// RapidHashAlgos reports the whole-file hash algorithms accepted by
|
||||
// RapidUploadByHashes (e.g. utils.MD5, utils.SHA1).
|
||||
RapidHashAlgos() []utils.HashType
|
||||
|
||||
// RapidHashNeedsPieces reports whether RapidUploadByHashes relies on
|
||||
// per-slice hashes (CAS slice MD5s / SHA1 pieces) for this driver.
|
||||
RapidHashNeedsPieces() bool
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
package driver
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
// SeedHashStream is a model.FileStreamer that only carries file metadata and
|
||||
// precomputed hashes; it never yields real content.
|
||||
//
|
||||
// It exists so that hash-driven rapid upload (秒传/CAS) implementations can reuse
|
||||
// the drivers' existing Put/RapidUpload code paths, which expect a
|
||||
// model.FileStreamer but only read the name/size/hash. Drivers that additionally
|
||||
// need the real content (e.g. to compute a leading proof hash) should supply
|
||||
// Source, which is exposed via GetReadCloser-like lazy opening.
|
||||
type SeedHashStream struct {
|
||||
name string
|
||||
size int64
|
||||
hashInfo utils.HashInfo
|
||||
// Source lazily opens the underlying content streamer. May be nil.
|
||||
Source func() (model.FileStreamer, error)
|
||||
}
|
||||
|
||||
var (
|
||||
_ model.FileStreamer = (*SeedHashStream)(nil)
|
||||
_ utils.ClosersIF = (*SeedHashStream)(nil)
|
||||
)
|
||||
|
||||
// NewSeedHashStream builds a hash-only streamer from a rapid-upload request.
|
||||
func NewSeedHashStream(req *SeedRapidUploadRequest) *SeedHashStream {
|
||||
s := &SeedHashStream{name: req.Name, size: req.Size, Source: req.Open}
|
||||
if req.Whole != nil {
|
||||
s.hashInfo = *req.Whole
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *SeedHashStream) GetName() string { return s.name }
|
||||
func (s *SeedHashStream) GetSize() int64 { return s.size }
|
||||
func (s *SeedHashStream) GetHash() utils.HashInfo { return s.hashInfo }
|
||||
func (s *SeedHashStream) GetMimetype() string { return "" }
|
||||
func (s *SeedHashStream) ModTime() time.Time { return time.Now() }
|
||||
func (s *SeedHashStream) CreateTime() time.Time { return time.Now() }
|
||||
func (s *SeedHashStream) IsDir() bool { return false }
|
||||
func (s *SeedHashStream) GetID() string { return "" }
|
||||
func (s *SeedHashStream) GetPath() string { return "" }
|
||||
|
||||
func (s *SeedHashStream) NeedStore() bool { return false }
|
||||
func (s *SeedHashStream) IsForceStreamUpload() bool { return true }
|
||||
func (s *SeedHashStream) GetExist() model.Obj { return nil }
|
||||
func (s *SeedHashStream) SetExist(model.Obj) {}
|
||||
func (s *SeedHashStream) GetFile() model.File { return nil }
|
||||
func (s *SeedHashStream) Add(io.Closer) {}
|
||||
func (s *SeedHashStream) AddIfCloser(any) {}
|
||||
func (s *SeedHashStream) Close() error { return nil }
|
||||
|
||||
// Read returns EOF: the stream carries hashes only, no content.
|
||||
func (s *SeedHashStream) Read([]byte) (int, error) { return 0, io.EOF }
|
||||
|
||||
// RangeRead returns an empty reader, since no content is available.
|
||||
func (s *SeedHashStream) RangeRead(http_range.Range) (io.Reader, error) {
|
||||
return nil, io.EOF
|
||||
}
|
||||
|
||||
// CacheFullAndWriter reports that the content cannot be materialized.
|
||||
func (s *SeedHashStream) CacheFullAndWriter(*model.UpdateProgress, io.Writer) (model.File, error) {
|
||||
return nil, io.EOF
|
||||
}
|
||||
@@ -4,4 +4,16 @@ import "errors"
|
||||
|
||||
var (
|
||||
EmptyToken = errors.New("empty token")
|
||||
|
||||
// ErrUnavailableHash indicates the seed does not carry the hash algorithm
|
||||
// required by the destination driver, so rapid upload cannot be attempted.
|
||||
ErrUnavailableHash = errors.New("required hash is unavailable")
|
||||
// ErrEmptyHash indicates a required hash exists but is empty/too short.
|
||||
ErrEmptyHash = errors.New("empty hash")
|
||||
// ErrHashMismatch indicates the remote side rejected the provided hash, so
|
||||
// a full content transfer is required instead of a rapid upload.
|
||||
ErrHashMismatch = errors.New("hash mismatch")
|
||||
// ErrRapidUploadFailed indicates the driver attempted a rapid upload but
|
||||
// could not confirm success.
|
||||
ErrRapidUploadFailed = errors.New("rapid upload failed")
|
||||
)
|
||||
|
||||
@@ -0,0 +1,603 @@
|
||||
package fs
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
stdpath "path"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/task"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/torrent"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/OpenListTeam/OpenList/v4/server/common"
|
||||
"github.com/OpenListTeam/tache"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// MaxSeedGenerateSyncSize bounds synchronous seed generation (1GB). Larger
|
||||
// requests are turned into an asynchronous task.
|
||||
const MaxSeedGenerateSyncSize = 1 * 1024 * 1024 * 1024
|
||||
|
||||
// SeedGenerateNeedsAsync reports whether the given files must be generated
|
||||
// asynchronously because they exceed the synchronous size limit. It resolves
|
||||
// each path and sums the file sizes, returning the first error encountered.
|
||||
func SeedGenerateNeedsAsync(ctx context.Context, user *model.User, paths []string) (bool, error) {
|
||||
var total int64
|
||||
for _, requestedPath := range paths {
|
||||
fullPath, err := user.JoinPath(requestedPath)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
storage, actualPath, err := op.GetStorageAndActualPath(fullPath)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
obj, err := op.Get(ctx, storage, actualPath)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("seed path must be a readable file: %s", requestedPath)
|
||||
}
|
||||
if obj.IsDir() {
|
||||
return false, fmt.Errorf("seed path must be a readable file: %s", requestedPath)
|
||||
}
|
||||
total += obj.GetSize()
|
||||
if total > MaxSeedGenerateSyncSize {
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// SeedHashSelection controls whole-file and piece hash inclusion.
|
||||
type SeedHashSelection struct {
|
||||
Whole bool `json:"whole"`
|
||||
Pieces bool `json:"pieces"`
|
||||
}
|
||||
|
||||
// SeedHashMatrix controls the optional hash metadata stored in a seed.
|
||||
type SeedHashMatrix struct {
|
||||
MD5 SeedHashSelection `json:"md5"`
|
||||
SHA1 SeedHashSelection `json:"sha1"`
|
||||
SHA256 SeedHashSelection `json:"sha256"`
|
||||
}
|
||||
|
||||
// SeedGenerateParams carries a fully-resolved seed generation request, free of
|
||||
// any HTTP transport concerns so it can run synchronously or as a task.
|
||||
type SeedGenerateParams struct {
|
||||
Paths []string
|
||||
Formats []string
|
||||
Name string
|
||||
Comment string
|
||||
FileComments map[string]string
|
||||
HashMatrix SeedHashMatrix
|
||||
PieceSize int64
|
||||
Trackers []string
|
||||
Channels []torrent.SeedChannel
|
||||
OutputPath string
|
||||
IncludeShare bool
|
||||
IncludeDirectSource bool
|
||||
ShareFiles []string
|
||||
DirectFiles []string
|
||||
}
|
||||
|
||||
// SeedArtifact is one generated seed container.
|
||||
type SeedArtifact struct {
|
||||
Format string `json:"format"`
|
||||
Name string `json:"name"`
|
||||
FileName string `json:"file_name"`
|
||||
SeedData string `json:"seed_data"`
|
||||
Size int `json:"size"`
|
||||
Path string `json:"path,omitempty"`
|
||||
}
|
||||
|
||||
// DeriveSeedName derives a sensible default seed name from the source paths:
|
||||
// single selection uses the file name, multi selection uses the common base
|
||||
// name (ignoring extensions) when all files share one, otherwise the folder name.
|
||||
func DeriveSeedName(paths []string) string {
|
||||
if len(paths) == 0 {
|
||||
return "OpenList Seed"
|
||||
}
|
||||
if len(paths) == 1 {
|
||||
return seedBaseName(paths[0])
|
||||
}
|
||||
// Common base name ignoring extensions (e.g. a.docx + a.exe -> "a").
|
||||
commonBase := seedBaseName(paths[0])
|
||||
for _, p := range paths[1:] {
|
||||
if base := seedBaseName(p); base != commonBase {
|
||||
commonBase = ""
|
||||
break
|
||||
}
|
||||
}
|
||||
if commonBase != "" {
|
||||
return commonBase
|
||||
}
|
||||
// Fall back to the common parent directory name.
|
||||
dir := commonParentDir(paths)
|
||||
if base := stdpath.Base(dir); base != "" && base != "/" && base != "." {
|
||||
return base
|
||||
}
|
||||
return "OpenList Seed"
|
||||
}
|
||||
|
||||
// seedBaseName returns the file name without its extension.
|
||||
func seedBaseName(p string) string {
|
||||
base := stdpath.Base(p)
|
||||
return strings.TrimSuffix(base, stdpath.Ext(base))
|
||||
}
|
||||
|
||||
// commonParentDir returns the longest common parent directory of the given paths.
|
||||
func commonParentDir(paths []string) string {
|
||||
if len(paths) == 0 {
|
||||
return "/"
|
||||
}
|
||||
parts := strings.Split(strings.Trim(stdpath.Dir(paths[0]), "/"), "/")
|
||||
for _, p := range paths[1:] {
|
||||
cur := strings.Split(strings.Trim(stdpath.Dir(p), "/"), "/")
|
||||
n := 0
|
||||
for n < len(parts) && n < len(cur) && parts[n] == cur[n] {
|
||||
n++
|
||||
}
|
||||
parts = parts[:n]
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "/"
|
||||
}
|
||||
return "/" + strings.Join(parts, "/")
|
||||
}
|
||||
|
||||
// NormalizeSeedFormats validates and deduplicates a list of seed format names.
|
||||
func NormalizeSeedFormats(rawFormats []string) ([]string, error) {
|
||||
formats := append([]string(nil), rawFormats...)
|
||||
if len(formats) == 1 && strings.TrimSpace(formats[0]) == "" {
|
||||
formats[0] = setting.GetStr(conf.SeedDefaultFormat, "oss")
|
||||
}
|
||||
if len(formats) > 3 {
|
||||
return nil, fmt.Errorf("at most three seed formats may be generated")
|
||||
}
|
||||
result := make([]string, 0, len(formats))
|
||||
seen := make(map[string]struct{}, len(formats))
|
||||
for _, rawFormat := range formats {
|
||||
format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(rawFormat), "."))
|
||||
if format == "bt" {
|
||||
format = "torrent"
|
||||
}
|
||||
if format != "oss" && format != "torrent" && format != "cas" {
|
||||
return nil, fmt.Errorf("unsupported seed format %q", rawFormat)
|
||||
}
|
||||
if _, exists := seen[format]; exists {
|
||||
continue
|
||||
}
|
||||
seen[format] = struct{}{}
|
||||
result = append(result, format)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func seedFormats(params SeedGenerateParams) ([]string, error) {
|
||||
formats, err := NormalizeSeedFormats(params.Formats)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(formats) == 0 {
|
||||
formats = []string{setting.GetStr(conf.SeedDefaultFormat, "oss")}
|
||||
}
|
||||
return formats, nil
|
||||
}
|
||||
|
||||
func seedMatrixEmpty(matrix SeedHashMatrix) bool {
|
||||
return !matrix.MD5.Whole && !matrix.MD5.Pieces && !matrix.SHA1.Whole && !matrix.SHA1.Pieces && !matrix.SHA256.Whole && !matrix.SHA256.Pieces
|
||||
}
|
||||
|
||||
func loadSeedDefaultMatrix() SeedHashMatrix {
|
||||
var matrix SeedHashMatrix
|
||||
if raw := strings.TrimSpace(setting.GetStr(conf.SeedDefaultMatrix)); raw != "" {
|
||||
_ = json.Unmarshal([]byte(raw), &matrix)
|
||||
}
|
||||
return matrix
|
||||
}
|
||||
|
||||
func normalizedSeedMatrix(matrix SeedHashMatrix, formats []string) SeedHashMatrix {
|
||||
if seedMatrixEmpty(matrix) {
|
||||
matrix = loadSeedDefaultMatrix()
|
||||
}
|
||||
if seedMatrixEmpty(matrix) {
|
||||
matrix = SeedHashMatrix{
|
||||
MD5: SeedHashSelection{Whole: true, Pieces: true}, SHA1: SeedHashSelection{Whole: true, Pieces: true},
|
||||
SHA256: SeedHashSelection{Whole: true, Pieces: true},
|
||||
}
|
||||
}
|
||||
for _, format := range formats {
|
||||
switch format {
|
||||
case "torrent":
|
||||
matrix.SHA1 = SeedHashSelection{Whole: true, Pieces: true}
|
||||
case "cas":
|
||||
matrix.MD5 = SeedHashSelection{Whole: true, Pieces: true}
|
||||
}
|
||||
}
|
||||
return matrix
|
||||
}
|
||||
|
||||
func canReuseListedHashes(hashInfo utils.HashInfo, matrix SeedHashMatrix) bool {
|
||||
if matrix.MD5.Pieces || matrix.SHA1.Pieces || matrix.SHA256.Pieces {
|
||||
return false
|
||||
}
|
||||
if matrix.MD5.Whole && hashInfo.GetHash(utils.MD5) == "" {
|
||||
return false
|
||||
}
|
||||
if matrix.SHA1.Whole && hashInfo.GetHash(utils.SHA1) == "" {
|
||||
return false
|
||||
}
|
||||
if matrix.SHA256.Whole && hashInfo.GetHash(utils.SHA256) == "" {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func applySeedMatrix(file *torrent.SeedFile, matrix SeedHashMatrix) {
|
||||
if !matrix.MD5.Whole {
|
||||
file.Hashes.MD5 = ""
|
||||
}
|
||||
if !matrix.SHA1.Whole {
|
||||
file.Hashes.SHA1 = ""
|
||||
}
|
||||
if !matrix.SHA256.Whole {
|
||||
file.Hashes.SHA256 = ""
|
||||
}
|
||||
if file.Hashes.Pieces == nil {
|
||||
return
|
||||
}
|
||||
if !matrix.MD5.Pieces {
|
||||
file.Hashes.Pieces.MD5 = nil
|
||||
}
|
||||
if !matrix.SHA1.Pieces {
|
||||
file.Hashes.Pieces.SHA1 = nil
|
||||
}
|
||||
if !matrix.SHA256.Pieces {
|
||||
file.Hashes.Pieces.SHA256 = nil
|
||||
}
|
||||
if len(file.Hashes.Pieces.MD5) == 0 && len(file.Hashes.Pieces.SHA1) == 0 && len(file.Hashes.Pieces.SHA256) == 0 {
|
||||
file.Hashes.Pieces = nil
|
||||
}
|
||||
}
|
||||
|
||||
// EncodeGeneratedSeed serializes a seed in the requested container format.
|
||||
func EncodeGeneratedSeed(seed *torrent.Seed, format string, standardPieces []byte) ([]byte, error) {
|
||||
if format != "torrent" {
|
||||
return torrent.EncodeSeed(seed, format)
|
||||
}
|
||||
t := &torrent.Torrent{
|
||||
Info: torrent.TorrentInfo{Name: seed.Name, PieceLength: seed.PieceSize, Pieces: standardPieces},
|
||||
Comment: seed.Comment,
|
||||
CreatedBy: seed.CreatedBy,
|
||||
CreationDate: time.Now().Unix(),
|
||||
OpenList: seed,
|
||||
}
|
||||
if len(seed.Trackers) > 0 {
|
||||
t.Announce = seed.Trackers[0]
|
||||
for _, tracker := range seed.Trackers {
|
||||
t.AnnounceList = append(t.AnnounceList, []string{tracker})
|
||||
}
|
||||
}
|
||||
if len(seed.Files) == 1 {
|
||||
file := seed.Files[0]
|
||||
t.Info.Name = stdpath.Base(file.Path)
|
||||
t.Info.Length = file.Size
|
||||
t.Info.MD5Sum = file.Hashes.MD5
|
||||
casCloud := file.CASCloud
|
||||
if casCloud == "" {
|
||||
casCloud = torrent.Cloud189
|
||||
}
|
||||
if file.CASSliceMD5 != "" {
|
||||
t.SetCASInfo(&torrent.CASInfo{
|
||||
FileMD5: strings.ToUpper(file.Hashes.MD5), SliceMD5: strings.ToUpper(file.CASSliceMD5),
|
||||
SliceSize: torrent.DefaultPieceSize, Cloud: casCloud,
|
||||
})
|
||||
} else if seed.PieceSize == torrent.DefaultPieceSize && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 {
|
||||
t.SetCASInfo(torrent.BuildCASInfoFromMD5sWithCloud(file.Hashes.MD5, file.Hashes.Pieces.MD5, torrent.DefaultPieceSize, casCloud))
|
||||
}
|
||||
} else {
|
||||
for _, file := range seed.Files {
|
||||
t.Info.Files = append(t.Info.Files, torrent.TorrentFile{Length: file.Size, Path: strings.Split(file.Path, "/"), MD5Sum: file.Hashes.MD5})
|
||||
}
|
||||
}
|
||||
return t.Encode()
|
||||
}
|
||||
|
||||
// GenerateSeedArtifacts reads each file once while computing the requested
|
||||
// hashes, then emits one or more seed containers. It has no HTTP dependency and
|
||||
// can be driven synchronously or from a background task.
|
||||
func GenerateSeedArtifacts(ctx context.Context, user *model.User, params SeedGenerateParams) ([]SeedArtifact, *torrent.Seed, error) {
|
||||
if len(params.Paths) == 0 || len(params.Paths) > torrent.DefaultMaxSeedFiles {
|
||||
return nil, nil, fmt.Errorf("invalid seed file count")
|
||||
}
|
||||
formats, err := seedFormats(params)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
matrix := normalizedSeedMatrix(params.HashMatrix, formats)
|
||||
pieceSize := params.PieceSize
|
||||
if pieceSize <= 0 {
|
||||
pieceSize = torrent.DefaultPieceSize
|
||||
}
|
||||
if slices.Contains(formats, "cas") {
|
||||
pieceSize = torrent.DefaultPieceSize
|
||||
}
|
||||
seedName := strings.TrimSpace(params.Name)
|
||||
if seedName == "" {
|
||||
seedName = DeriveSeedName(params.Paths)
|
||||
}
|
||||
seed := torrent.NewSeed(seedName, "OpenList", pieceSize)
|
||||
seed.Comment = params.Comment
|
||||
seed.Trackers = params.Trackers
|
||||
seed.Channels = params.Channels
|
||||
shareSet := make(map[string]bool, len(params.ShareFiles))
|
||||
for _, p := range params.ShareFiles {
|
||||
if strings.TrimSpace(p) != "" {
|
||||
shareSet[p] = true
|
||||
}
|
||||
}
|
||||
directSet := make(map[string]bool, len(params.DirectFiles))
|
||||
for _, p := range params.DirectFiles {
|
||||
if strings.TrimSpace(p) != "" {
|
||||
directSet[p] = true
|
||||
}
|
||||
}
|
||||
useGlobalShare := len(shareSet) == 0 && params.IncludeShare
|
||||
useGlobalDirect := len(directSet) == 0 && params.IncludeDirectSource
|
||||
hasShare := useGlobalShare || len(shareSet) > 0
|
||||
hasDirect := useGlobalDirect || len(directSet) > 0
|
||||
if hasShare && !user.CanShare() {
|
||||
return nil, nil, errs.PermissionDenied
|
||||
}
|
||||
if hasDirect && setting.GetBool(conf.SignAll) && !hasShare {
|
||||
return nil, nil, fmt.Errorf("direct sources require an automatic share when global signing is enabled")
|
||||
}
|
||||
if (hasShare || hasDirect) && strings.TrimSpace(setting.GetStr(conf.SeedSiteURL)) == "" {
|
||||
return nil, nil, fmt.Errorf("seed_site_url must be configured before embedding download sources")
|
||||
}
|
||||
globalHasher := torrent.NewHashWriter(pieceSize, pieceSize, 0)
|
||||
fullPaths := make([]string, 0, len(params.Paths))
|
||||
var total int64
|
||||
for _, requestedPath := range params.Paths {
|
||||
fullPath, err := user.JoinPath(requestedPath)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
meta, err := op.GetNearestMeta(fullPath)
|
||||
if err != nil && !errors.Is(errors.Cause(err), errs.MetaNotFound) {
|
||||
return nil, nil, err
|
||||
}
|
||||
if !common.CanRead(user, meta, fullPath) {
|
||||
return nil, nil, errs.PermissionDenied
|
||||
}
|
||||
storage, actualPath, err := op.GetStorageAndActualPath(fullPath)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
obj, err := op.Get(ctx, storage, actualPath)
|
||||
if err != nil || obj.IsDir() {
|
||||
return nil, nil, fmt.Errorf("seed path must be a readable file: %s", requestedPath)
|
||||
}
|
||||
total += obj.GetSize()
|
||||
modified := ""
|
||||
if !obj.ModTime().IsZero() {
|
||||
modified = obj.ModTime().UTC().Format(time.RFC3339)
|
||||
}
|
||||
seedPath := stdpath.Base(requestedPath)
|
||||
if len(params.Paths) > 1 {
|
||||
seedPath = strings.TrimPrefix(stdpath.Clean(requestedPath), "/")
|
||||
}
|
||||
|
||||
hashInfo := obj.GetHash()
|
||||
if canReuseListedHashes(hashInfo, matrix) {
|
||||
seedFile := torrent.SeedFile{
|
||||
Path: seedPath,
|
||||
Size: obj.GetSize(),
|
||||
Modified: modified,
|
||||
Hashes: torrent.SeedHashes{
|
||||
MD5: strings.ToLower(hashInfo.GetHash(utils.MD5)),
|
||||
SHA1: strings.ToLower(hashInfo.GetHash(utils.SHA1)),
|
||||
SHA256: strings.ToLower(hashInfo.GetHash(utils.SHA256)),
|
||||
},
|
||||
}
|
||||
applySeedMatrix(&seedFile, matrix)
|
||||
if comment := strings.TrimSpace(params.FileComments[requestedPath]); comment != "" {
|
||||
seedFile.Comment = comment
|
||||
} else if comment := strings.TrimSpace(params.FileComments[obj.GetName()]); comment != "" {
|
||||
seedFile.Comment = comment
|
||||
}
|
||||
if useGlobalDirect || directSet[requestedPath] {
|
||||
baseURL := strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/")
|
||||
seedFile.Sources = []torrent.SeedSource{{Type: "openlist-direct", URL: baseURL + utils.EncodePath("/d"+fullPath)}}
|
||||
}
|
||||
seed.Files = append(seed.Files, seedFile)
|
||||
fullPaths = append(fullPaths, fullPath)
|
||||
continue
|
||||
}
|
||||
|
||||
link, _, err := op.Link(ctx, storage, actualPath, model.LinkArgs{})
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("storage cannot stream %s: %v", requestedPath, err)
|
||||
}
|
||||
rangeReader, err := stream.GetRangeReaderFromLink(obj.GetSize(), link)
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("storage cannot stream %s", requestedPath)
|
||||
}
|
||||
rc, err := rangeReader.RangeRead(ctx, http_range.Range{Length: obj.GetSize()})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
fileHasher := torrent.NewHashWriter(pieceSize, pieceSize, 0)
|
||||
n, copyErr := io.Copy(io.MultiWriter(globalHasher, fileHasher), rc)
|
||||
_ = rc.Close()
|
||||
if copyErr != nil {
|
||||
return nil, nil, fmt.Errorf("read %s: %w", requestedPath, copyErr)
|
||||
}
|
||||
if n != obj.GetSize() {
|
||||
return nil, nil, fmt.Errorf("read %s: got %d of %d bytes", requestedPath, n, obj.GetSize())
|
||||
}
|
||||
fileHasher.Finish()
|
||||
seedFile := fileHasher.BuildSeedFile(seedPath, modified)
|
||||
applySeedMatrix(&seedFile, matrix)
|
||||
if comment := strings.TrimSpace(params.FileComments[requestedPath]); comment != "" {
|
||||
seedFile.Comment = comment
|
||||
} else if comment := strings.TrimSpace(params.FileComments[obj.GetName()]); comment != "" {
|
||||
seedFile.Comment = comment
|
||||
}
|
||||
if useGlobalDirect || directSet[requestedPath] {
|
||||
baseURL := strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/")
|
||||
seedFile.Sources = []torrent.SeedSource{{Type: "openlist-direct", URL: baseURL + utils.EncodePath("/d"+fullPath)}}
|
||||
}
|
||||
seed.Files = append(seed.Files, seedFile)
|
||||
fullPaths = append(fullPaths, fullPath)
|
||||
}
|
||||
globalHasher.Finish()
|
||||
createdShares := make([]string, 0, len(seed.Files))
|
||||
keepCreatedShares := false
|
||||
defer func() {
|
||||
if !keepCreatedShares {
|
||||
for _, createdID := range createdShares {
|
||||
_ = op.DeleteSharing(createdID)
|
||||
}
|
||||
}
|
||||
}()
|
||||
if hasShare {
|
||||
for index, fullPath := range fullPaths {
|
||||
if !useGlobalShare && !shareSet[params.Paths[index]] {
|
||||
continue
|
||||
}
|
||||
sharing := &model.Sharing{
|
||||
SharingDB: &model.SharingDB{Remark: "Transfer seed source"},
|
||||
Files: []string{fullPath}, Creator: user,
|
||||
}
|
||||
shareID, createErr := op.CreateSharing(sharing)
|
||||
if createErr != nil {
|
||||
return nil, nil, fmt.Errorf("create seed share: %w", createErr)
|
||||
}
|
||||
createdShares = append(createdShares, shareID)
|
||||
seed.Files[index].Sources = []torrent.SeedSource{{
|
||||
Type: "openlist-share", URL: strings.TrimRight(setting.GetStr(conf.SeedSiteURL), "/") + "/sd/" + shareID,
|
||||
ShareID: shareID,
|
||||
}}
|
||||
}
|
||||
}
|
||||
if err := torrent.ValidateSeed(seed, torrent.DefaultParseLimits()); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
outputPath := strings.TrimSpace(params.OutputPath)
|
||||
artifacts := make([]SeedArtifact, 0, len(formats))
|
||||
seenFormats := make(map[string]struct{}, len(formats))
|
||||
|
||||
// writeArtifact persists the encoded container into the destination folder
|
||||
// when an output path is configured, returning the fully-populated artifact.
|
||||
writeArtifact := func(format, fileName string, data []byte) (SeedArtifact, error) {
|
||||
artifact := SeedArtifact{
|
||||
Format: format,
|
||||
Name: fileName,
|
||||
FileName: fileName,
|
||||
SeedData: base64.StdEncoding.EncodeToString(data),
|
||||
Size: len(data),
|
||||
}
|
||||
if outputPath == "" {
|
||||
return artifact, nil
|
||||
}
|
||||
dstDir, err := user.JoinPath(outputPath)
|
||||
if err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
meta, metaErr := op.GetNearestMeta(dstDir)
|
||||
if metaErr != nil && !errors.Is(errors.Cause(metaErr), errs.MetaNotFound) {
|
||||
return artifact, metaErr
|
||||
}
|
||||
if (!user.CanWriteContent() && !common.CanWriteContentBypassUserPerms(meta, dstDir)) || !common.CanWrite(user, meta, dstDir) {
|
||||
return artifact, errs.PermissionDenied
|
||||
}
|
||||
fileStream := &stream.FileStream{
|
||||
Ctx: ctx,
|
||||
Obj: &model.Object{Name: fileName, Size: int64(len(data)), Modified: time.Now()},
|
||||
Reader: bytes.NewReader(data), Mimetype: "application/octet-stream",
|
||||
}
|
||||
if err = PutDirectly(ctx, dstDir, fileStream); err != nil {
|
||||
return artifact, err
|
||||
}
|
||||
artifact.Path = stdpath.Join(outputPath, fileName)
|
||||
return artifact, nil
|
||||
}
|
||||
|
||||
for _, requestedFormat := range formats {
|
||||
format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(requestedFormat), "."))
|
||||
if format == "bt" {
|
||||
format = "torrent"
|
||||
}
|
||||
if _, exists := seenFormats[format]; exists {
|
||||
continue
|
||||
}
|
||||
seenFormats[format] = struct{}{}
|
||||
|
||||
data, err := EncodeGeneratedSeed(seed, format, globalHasher.GetPieceHashes())
|
||||
if err != nil {
|
||||
return nil, nil, fmt.Errorf("generate %s seed: %w", format, err)
|
||||
}
|
||||
fileName := stdpath.Base(seed.Name) + "." + format
|
||||
artifact, err := writeArtifact(format, fileName, data)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
artifacts = append(artifacts, artifact)
|
||||
}
|
||||
keepCreatedShares = true
|
||||
return artifacts, seed, nil
|
||||
}
|
||||
|
||||
// SeedGenerateTask generates seed containers asynchronously, writing them into
|
||||
// params.OutputPath so they appear in the target folder once done.
|
||||
type SeedGenerateTask struct {
|
||||
task.TaskExtension
|
||||
params SeedGenerateParams
|
||||
}
|
||||
|
||||
func (t *SeedGenerateTask) GetName() string {
|
||||
if len(t.params.Paths) == 1 {
|
||||
return fmt.Sprintf("generate seed for %s", stdpath.Base(t.params.Paths[0]))
|
||||
}
|
||||
return fmt.Sprintf("generate seed for %d files", len(t.params.Paths))
|
||||
}
|
||||
|
||||
func (t *SeedGenerateTask) GetStatus() string {
|
||||
return "generating seed"
|
||||
}
|
||||
|
||||
func (t *SeedGenerateTask) Run() error {
|
||||
t.ClearEndTime()
|
||||
t.SetStartTime(time.Now())
|
||||
defer func() { t.SetEndTime(time.Now()) }()
|
||||
_, _, err := GenerateSeedArtifacts(t.Ctx(), t.Creator, t.params)
|
||||
return err
|
||||
}
|
||||
|
||||
var SeedGenerateTaskManager *tache.Manager[*SeedGenerateTask]
|
||||
|
||||
// AddSeedGenerateTask schedules an asynchronous seed generation task.
|
||||
func AddSeedGenerateTask(ctx context.Context, user *model.User, params SeedGenerateParams) (task.TaskExtensionInfo, error) {
|
||||
t := &SeedGenerateTask{
|
||||
TaskExtension: task.TaskExtension{
|
||||
Creator: user,
|
||||
ApiUrl: common.GetApiUrl(ctx),
|
||||
},
|
||||
params: params,
|
||||
}
|
||||
SeedGenerateTaskManager.Add(t)
|
||||
return t, nil
|
||||
}
|
||||
@@ -19,6 +19,7 @@ type Storage struct {
|
||||
Disabled bool `json:"disabled"` // if disabled
|
||||
DisableIndex bool `json:"disable_index"`
|
||||
EnableSign bool `json:"enable_sign"`
|
||||
SeedPolicy string `json:"seed_policy" gorm:"default:inherit"`
|
||||
Sort
|
||||
Proxy
|
||||
}
|
||||
|
||||
@@ -173,6 +173,14 @@ func getMainItems(config driver.Config) []driver.Item {
|
||||
Default: "false",
|
||||
Required: true,
|
||||
})
|
||||
items = append(items, driver.Item{
|
||||
Name: "seed_policy",
|
||||
Type: conf.TypeSelect,
|
||||
Options: "inherit,on,off",
|
||||
Default: "inherit",
|
||||
Required: true,
|
||||
Help: "Override automatic transfer-seed generation for this storage",
|
||||
})
|
||||
return items
|
||||
}
|
||||
func getAdditionalItems(t reflect.Type, defaultRoot string) []driver.Item {
|
||||
|
||||
+32
-13
@@ -145,15 +145,24 @@ func bencodeEncodeOrderedDict(w io.Writer, d OrderedDict) error {
|
||||
|
||||
// BencodeDecode 从字节数组解码 bencode 数据
|
||||
func BencodeDecode(data []byte) (interface{}, error) {
|
||||
if int64(len(data)) > DefaultMaxSeedSize {
|
||||
return nil, fmt.Errorf("bencode: input exceeds %d bytes", DefaultMaxSeedSize)
|
||||
}
|
||||
reader := bytes.NewReader(data)
|
||||
val, err := bencodeDecodeValue(reader)
|
||||
val, err := bencodeDecodeValue(reader, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if reader.Len() != 0 {
|
||||
return nil, fmt.Errorf("bencode: trailing data")
|
||||
}
|
||||
return val, nil
|
||||
}
|
||||
|
||||
func bencodeDecodeValue(r *bytes.Reader) (interface{}, error) {
|
||||
func bencodeDecodeValue(r *bytes.Reader, depth int) (interface{}, error) {
|
||||
if depth > DefaultParseLimits().MaxDepth {
|
||||
return nil, fmt.Errorf("bencode: nesting depth exceeds limit")
|
||||
}
|
||||
b, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -163,9 +172,9 @@ func bencodeDecodeValue(r *bytes.Reader) (interface{}, error) {
|
||||
case b == 'i':
|
||||
return bencodeDecodeInt(r)
|
||||
case b == 'l':
|
||||
return bencodeDecodeList(r)
|
||||
return bencodeDecodeList(r, depth+1)
|
||||
case b == 'd':
|
||||
return bencodeDecodeDict(r)
|
||||
return bencodeDecodeDict(r, depth+1)
|
||||
case b >= '0' && b <= '9':
|
||||
r.UnreadByte()
|
||||
return bencodeDecodeString(r)
|
||||
@@ -206,10 +215,14 @@ func bencodeDecodeString(r *bytes.Reader) ([]byte, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bencode: invalid string length: %v", err)
|
||||
}
|
||||
if length < 0 || length > 100*1024*1024 {
|
||||
return nil, fmt.Errorf("bencode: string length out of bounds: %d", length)
|
||||
// A single string can never exceed the whole input, which BencodeDecode
|
||||
// already caps at DefaultMaxSeedSize. Deriving the bound from the same
|
||||
// constant keeps the constraint self-consistent instead of maintaining a
|
||||
// second, unreachable 100MB ceiling.
|
||||
if length < 0 || length > DefaultMaxSeedSize {
|
||||
return nil, fmt.Errorf("bencode: string length out of bounds: %d (limit %d)", length, DefaultMaxSeedSize)
|
||||
}
|
||||
// Safe to convert to int: bounds check above ensures length <= 100MB which fits in int32
|
||||
// Bounded by DefaultMaxSeedSize, so the int conversion cannot truncate.
|
||||
data := make([]byte, int(length))
|
||||
_, err = io.ReadFull(r, data)
|
||||
if err != nil {
|
||||
@@ -218,9 +231,12 @@ func bencodeDecodeString(r *bytes.Reader) ([]byte, error) {
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func bencodeDecodeList(r *bytes.Reader) ([]interface{}, error) {
|
||||
func bencodeDecodeList(r *bytes.Reader, depth int) ([]interface{}, error) {
|
||||
var list []interface{}
|
||||
for {
|
||||
if len(list) >= DefaultMaxSeedFiles*4 {
|
||||
return nil, fmt.Errorf("bencode: list item limit exceeded")
|
||||
}
|
||||
b, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -228,8 +244,8 @@ func bencodeDecodeList(r *bytes.Reader) ([]interface{}, error) {
|
||||
if b == 'e' {
|
||||
return list, nil
|
||||
}
|
||||
r.UnreadByte()
|
||||
val, err := bencodeDecodeValue(r)
|
||||
_ = r.UnreadByte()
|
||||
val, err := bencodeDecodeValue(r, depth)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -237,9 +253,12 @@ func bencodeDecodeList(r *bytes.Reader) ([]interface{}, error) {
|
||||
}
|
||||
}
|
||||
|
||||
func bencodeDecodeDict(r *bytes.Reader) (map[string]interface{}, error) {
|
||||
func bencodeDecodeDict(r *bytes.Reader, depth int) (map[string]interface{}, error) {
|
||||
dict := make(map[string]interface{})
|
||||
for {
|
||||
if len(dict) >= DefaultMaxSeedFiles*4 {
|
||||
return nil, fmt.Errorf("bencode: dictionary item limit exceeded")
|
||||
}
|
||||
b, err := r.ReadByte()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -247,12 +266,12 @@ func bencodeDecodeDict(r *bytes.Reader) (map[string]interface{}, error) {
|
||||
if b == 'e' {
|
||||
return dict, nil
|
||||
}
|
||||
r.UnreadByte()
|
||||
_ = r.UnreadByte()
|
||||
keyBytes, err := bencodeDecodeString(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
val, err := bencodeDecodeValue(r)
|
||||
val, err := bencodeDecodeValue(r, depth)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+31
-10
@@ -1,9 +1,10 @@
|
||||
package torrent
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"path"
|
||||
)
|
||||
|
||||
// GenerateFromFile 从文件路径生成通用的 torrent 文件(不含 CAS 扩展)
|
||||
@@ -30,7 +31,7 @@ func GenerateFromReader(reader io.Reader, fileName string, fileSize int64, piece
|
||||
pieceSize = DefaultPieceSize
|
||||
}
|
||||
|
||||
hw := NewHashWriter(pieceSize, pieceSize)
|
||||
hw := NewHashWriter(pieceSize, pieceSize, fileSize)
|
||||
|
||||
buf := make([]byte, 32*1024)
|
||||
for {
|
||||
@@ -64,7 +65,7 @@ func GenerateFromReaderWithCAS(reader io.Reader, fileName string, fileSize int64
|
||||
pieceSize = DefaultPieceSize
|
||||
}
|
||||
|
||||
hw := NewHashWriter(pieceSize, pieceSize)
|
||||
hw := NewHashWriter(pieceSize, pieceSize, fileSize)
|
||||
|
||||
buf := make([]byte, 32*1024)
|
||||
for {
|
||||
@@ -85,12 +86,8 @@ func GenerateFromReaderWithCAS(reader io.Reader, fileName string, fileSize int64
|
||||
sliceMD5s := hw.GetSliceMD5s()
|
||||
pieceHashes := hw.GetPieceHashes()
|
||||
|
||||
// 计算 sliceMD5
|
||||
sliceMD5 := fileMD5
|
||||
if len(sliceMD5s) > 1 {
|
||||
joined := strings.Join(sliceMD5s, "\n")
|
||||
sliceMD5 = strings.ToUpper(GetMD5Str(joined))
|
||||
}
|
||||
// 计算 sliceMD5(统一走规范实现)
|
||||
sliceMD5 := SliceMD5FromPieces(sliceMD5s, fileMD5)
|
||||
|
||||
t := NewTorrent(fileName, fileSize, fileMD5)
|
||||
t.Info.PieceLength = pieceSize
|
||||
@@ -100,7 +97,7 @@ func GenerateFromReaderWithCAS(reader io.Reader, fileName string, fileSize int64
|
||||
SliceMD5: sliceMD5,
|
||||
SliceMD5s: sliceMD5s,
|
||||
SliceSize: pieceSize,
|
||||
Cloud: "189",
|
||||
Cloud: Cloud189,
|
||||
})
|
||||
|
||||
return t.Encode()
|
||||
@@ -121,3 +118,27 @@ func GenerateFromFileWithCAS(filePath string) ([]byte, error) {
|
||||
|
||||
return GenerateFromReaderWithCAS(f, info.Name(), info.Size(), DefaultPieceSize)
|
||||
}
|
||||
|
||||
// GenerateSeedFromReader computes the complete OSS hash matrix in one stream pass.
|
||||
func GenerateSeedFromReader(reader io.Reader, filePath string, expectedSize, pieceSize int64, createdBy string) (*Seed, error) {
|
||||
if pieceSize <= 0 {
|
||||
pieceSize = DefaultPieceSize
|
||||
}
|
||||
if err := validateRelativeSeedPath(filePath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hw := NewHashWriter(pieceSize, pieceSize, expectedSize)
|
||||
if _, err := CopyAndHash(nil, reader, hw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hw.Finish()
|
||||
if expectedSize >= 0 && hw.GetTotalWritten() != expectedSize {
|
||||
return nil, fmt.Errorf("stream size mismatch: read %d bytes, expected %d", hw.GetTotalWritten(), expectedSize)
|
||||
}
|
||||
seed := NewSeed(path.Base(filePath), createdBy, pieceSize)
|
||||
seed.Files = []SeedFile{hw.BuildSeedFile(filePath, "")}
|
||||
if err := ValidateSeed(seed, DefaultParseLimits()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return seed, nil
|
||||
}
|
||||
|
||||
+129
-22
@@ -3,11 +3,14 @@ package torrent
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
|
||||
)
|
||||
|
||||
// HashWriter 同时计算文件的 MD5、分片 MD5 和 SHA-1 piece hash
|
||||
@@ -15,15 +18,24 @@ import (
|
||||
type HashWriter struct {
|
||||
// 整文件 MD5
|
||||
fileMD5 hash.Hash
|
||||
// fileSHA1 and fileSHA256 complete the portable full-file matrix.
|
||||
fileSHA1 hash.Hash
|
||||
fileSHA256 hash.Hash
|
||||
// fileGCID 用于迅雷、PikPak 等
|
||||
fileGCID hash.Hash
|
||||
// 当前分片 MD5
|
||||
sliceMD5 hash.Hash
|
||||
// 当前 piece 的 SHA-1
|
||||
pieceSHA1 hash.Hash
|
||||
// Per-piece hashers are updated in the same pass as whole-file hashes.
|
||||
pieceMD5 hash.Hash
|
||||
pieceSHA1 hash.Hash
|
||||
pieceSHA256 hash.Hash
|
||||
|
||||
// 分片大小(默认 10MB)
|
||||
sliceSize int64
|
||||
// piece 大小(与 sliceSize 相同,保持对齐)
|
||||
pieceSize int64
|
||||
// 文件总大小(用于 GCID 初始化)
|
||||
fileSize int64
|
||||
|
||||
// 当前分片已写入字节数
|
||||
sliceWritten int64
|
||||
@@ -34,14 +46,19 @@ type HashWriter struct {
|
||||
|
||||
// 每个分片的 MD5(大写十六进制)
|
||||
sliceMD5Hexs []string
|
||||
// 所有 piece 的 SHA-1 哈希拼接
|
||||
// all standard BitTorrent SHA-1 piece hashes concatenated
|
||||
pieceHashes []byte
|
||||
// portable per-file piece matrix
|
||||
pieceMD5Hexs []string
|
||||
pieceSHA1Hexs []string
|
||||
pieceSHA256Hexs []string
|
||||
}
|
||||
|
||||
// NewHashWriter 创建一个新的 HashWriter
|
||||
// sliceSize: CAS 分片大小(通常 10MB)
|
||||
// pieceSize: BT piece 大小(设为与 sliceSize 相同以保持对齐)
|
||||
func NewHashWriter(sliceSize, pieceSize int64) *HashWriter {
|
||||
// fileSize: 文件总大小(用于 GCID 初始化,0 表示未知)
|
||||
func NewHashWriter(sliceSize, pieceSize, fileSize int64) *HashWriter {
|
||||
if sliceSize <= 0 {
|
||||
sliceSize = DefaultPieceSize
|
||||
}
|
||||
@@ -49,17 +66,23 @@ func NewHashWriter(sliceSize, pieceSize int64) *HashWriter {
|
||||
pieceSize = DefaultPieceSize
|
||||
}
|
||||
return &HashWriter{
|
||||
fileMD5: md5.New(),
|
||||
sliceMD5: md5.New(),
|
||||
pieceSHA1: sha1.New(),
|
||||
sliceSize: sliceSize,
|
||||
pieceSize: pieceSize,
|
||||
fileMD5: md5.New(),
|
||||
fileSHA1: sha1.New(),
|
||||
fileSHA256: sha256.New(),
|
||||
fileGCID: hash_extend.GCID.NewFunc(fileSize),
|
||||
sliceMD5: md5.New(),
|
||||
pieceMD5: md5.New(),
|
||||
pieceSHA1: sha1.New(),
|
||||
pieceSHA256: sha256.New(),
|
||||
sliceSize: sliceSize,
|
||||
pieceSize: pieceSize,
|
||||
fileSize: fileSize,
|
||||
}
|
||||
}
|
||||
|
||||
// NewDefaultHashWriter 创建默认的 HashWriter(10MB 分片)
|
||||
func NewDefaultHashWriter() *HashWriter {
|
||||
return NewHashWriter(DefaultPieceSize, DefaultPieceSize)
|
||||
return NewHashWriter(DefaultPieceSize, DefaultPieceSize, 0)
|
||||
}
|
||||
|
||||
// Write 实现 io.Writer 接口
|
||||
@@ -76,12 +99,15 @@ func (hw *HashWriter) Write(p []byte) (n int, err error) {
|
||||
|
||||
chunk := p[offset : offset+int(canWrite)]
|
||||
|
||||
// 写入整文件 MD5
|
||||
hw.fileMD5.Write(chunk)
|
||||
// 写入当前分片 MD5
|
||||
hw.sliceMD5.Write(chunk)
|
||||
// 写入当前 piece SHA-1
|
||||
hw.pieceSHA1.Write(chunk)
|
||||
// Write all whole-file and boundary-specific hashes in one pass.
|
||||
_, _ = hw.fileMD5.Write(chunk)
|
||||
_, _ = hw.fileSHA1.Write(chunk)
|
||||
_, _ = hw.fileSHA256.Write(chunk)
|
||||
_, _ = hw.fileGCID.Write(chunk)
|
||||
_, _ = hw.sliceMD5.Write(chunk)
|
||||
_, _ = hw.pieceMD5.Write(chunk)
|
||||
_, _ = hw.pieceSHA1.Write(chunk)
|
||||
_, _ = hw.pieceSHA256.Write(chunk)
|
||||
|
||||
hw.sliceWritten += canWrite
|
||||
hw.pieceWritten += canWrite
|
||||
@@ -112,8 +138,16 @@ func (hw *HashWriter) finishSlice() {
|
||||
|
||||
// finishPiece 完成当前 piece 的 SHA-1 计算
|
||||
func (hw *HashWriter) finishPiece() {
|
||||
hw.pieceHashes = append(hw.pieceHashes, hw.pieceSHA1.Sum(nil)...)
|
||||
md5Sum := hw.pieceMD5.Sum(nil)
|
||||
sha1Sum := hw.pieceSHA1.Sum(nil)
|
||||
sha256Sum := hw.pieceSHA256.Sum(nil)
|
||||
hw.pieceMD5Hexs = append(hw.pieceMD5Hexs, hex.EncodeToString(md5Sum))
|
||||
hw.pieceSHA1Hexs = append(hw.pieceSHA1Hexs, hex.EncodeToString(sha1Sum))
|
||||
hw.pieceSHA256Hexs = append(hw.pieceSHA256Hexs, hex.EncodeToString(sha256Sum))
|
||||
hw.pieceHashes = append(hw.pieceHashes, sha1Sum...)
|
||||
hw.pieceMD5.Reset()
|
||||
hw.pieceSHA1.Reset()
|
||||
hw.pieceSHA256.Reset()
|
||||
hw.pieceWritten = 0
|
||||
}
|
||||
|
||||
@@ -134,6 +168,56 @@ func (hw *HashWriter) GetFileMD5() string {
|
||||
return strings.ToUpper(hex.EncodeToString(hw.fileMD5.Sum(nil)))
|
||||
}
|
||||
|
||||
// GetFileSHA1 returns the lowercase whole-file SHA-1 digest.
|
||||
func (hw *HashWriter) GetFileSHA1() string {
|
||||
return hex.EncodeToString(hw.fileSHA1.Sum(nil))
|
||||
}
|
||||
|
||||
// GetFileSHA256 returns the lowercase whole-file SHA-256 digest.
|
||||
func (hw *HashWriter) GetFileSHA256() string {
|
||||
return hex.EncodeToString(hw.fileSHA256.Sum(nil))
|
||||
}
|
||||
|
||||
// GetFileGCID returns the uppercase GCID digest for Thunder/PikPak.
|
||||
func (hw *HashWriter) GetFileGCID() string {
|
||||
return strings.ToUpper(hex.EncodeToString(hw.fileGCID.Sum(nil)))
|
||||
}
|
||||
|
||||
// GetPieceMD5s returns independent per-file MD5 piece hashes.
|
||||
func (hw *HashWriter) GetPieceMD5s() []string {
|
||||
return append([]string(nil), hw.pieceMD5Hexs...)
|
||||
}
|
||||
|
||||
// GetPieceSHA1s returns independent per-file SHA-1 piece hashes.
|
||||
func (hw *HashWriter) GetPieceSHA1s() []string {
|
||||
return append([]string(nil), hw.pieceSHA1Hexs...)
|
||||
}
|
||||
|
||||
// GetPieceSHA256s returns independent per-file SHA-256 piece hashes.
|
||||
func (hw *HashWriter) GetPieceSHA256s() []string {
|
||||
return append([]string(nil), hw.pieceSHA256Hexs...)
|
||||
}
|
||||
|
||||
// BuildSeedFile exports all hashes accumulated during this single stream pass.
|
||||
func (hw *HashWriter) BuildSeedFile(filePath string, modified string) SeedFile {
|
||||
return SeedFile{
|
||||
Path: filePath,
|
||||
Size: hw.totalWritten,
|
||||
Modified: modified,
|
||||
Hashes: SeedHashes{
|
||||
MD5: strings.ToLower(hw.GetFileMD5()),
|
||||
SHA1: hw.GetFileSHA1(),
|
||||
SHA256: hw.GetFileSHA256(),
|
||||
GCID: strings.ToLower(hw.GetFileGCID()),
|
||||
Pieces: &SeedPieceHashes{
|
||||
MD5: hw.GetPieceMD5s(),
|
||||
SHA1: hw.GetPieceSHA1s(),
|
||||
SHA256: hw.GetPieceSHA256s(),
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// GetSliceMD5s 获取所有分片的 MD5 列表
|
||||
func (hw *HashWriter) GetSliceMD5s() []string {
|
||||
return hw.sliceMD5Hexs
|
||||
@@ -141,11 +225,34 @@ func (hw *HashWriter) GetSliceMD5s() []string {
|
||||
|
||||
// GetSliceMD5 获取最终的 sliceMD5(用于秒传)
|
||||
func (hw *HashWriter) GetSliceMD5(fileMD5 string) string {
|
||||
if len(hw.sliceMD5Hexs) <= 1 {
|
||||
return fileMD5
|
||||
return SliceMD5FromPieces(hw.sliceMD5Hexs, fileMD5)
|
||||
}
|
||||
|
||||
// SliceMD5FromPieces is the single canonical implementation of the sliceMd5
|
||||
// rule shared by every CAS producer and consumer:
|
||||
//
|
||||
// - no piece, or a single piece -> the whole-file MD5
|
||||
// - two or more pieces -> MD5 of the piece MD5s joined by "\n"
|
||||
//
|
||||
// Keeping one implementation matters because this value is what the remote
|
||||
// provider compares against: a divergence between the hash-generation side and
|
||||
// the torrent/CAS encoding side silently turns rapid uploads into mismatches.
|
||||
// All comparisons and the returned value are upper-case.
|
||||
func SliceMD5FromPieces(sliceMD5s []string, fileMD5 string) string {
|
||||
switch len(sliceMD5s) {
|
||||
case 0, 1:
|
||||
// A single piece covers the whole file, so the two hashes coincide.
|
||||
if len(sliceMD5s) == 1 && sliceMD5s[0] != "" {
|
||||
return strings.ToUpper(sliceMD5s[0])
|
||||
}
|
||||
return strings.ToUpper(fileMD5)
|
||||
default:
|
||||
upper := make([]string, len(sliceMD5s))
|
||||
for i, piece := range sliceMD5s {
|
||||
upper[i] = strings.ToUpper(piece)
|
||||
}
|
||||
return strings.ToUpper(GetMD5Str(strings.Join(upper, "\n")))
|
||||
}
|
||||
joined := strings.Join(hw.sliceMD5Hexs, "\n")
|
||||
return strings.ToUpper(GetMD5Str(joined))
|
||||
}
|
||||
|
||||
// GetPieceHashes 获取所有 piece 的 SHA-1 哈希拼接
|
||||
@@ -170,7 +277,7 @@ func (hw *HashWriter) BuildTorrent(fileName string, fileSize int64) *Torrent {
|
||||
SliceMD5: sliceMD5,
|
||||
SliceMD5s: hw.GetSliceMD5s(),
|
||||
SliceSize: hw.sliceSize,
|
||||
Cloud: "189",
|
||||
Cloud: Cloud189,
|
||||
})
|
||||
|
||||
return t
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
package torrent
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// --- path traversal / malformed path handling ---------------------------------
|
||||
|
||||
func TestValidateSeedRejectsUnsafePaths(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
path string
|
||||
}{
|
||||
{"parent traversal", "../secret"},
|
||||
{"nested traversal", "a/../../secret"},
|
||||
{"absolute unix", "/etc/passwd"},
|
||||
{"empty", ""},
|
||||
{"current dir", "."},
|
||||
{"nul byte", "a\x00b"},
|
||||
{"backslash traversal", `..\secret`},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
seed := testSeed()
|
||||
seed.Files[0].Path = tc.path
|
||||
if err := ValidateSeed(seed, DefaultParseLimits()); err == nil {
|
||||
t.Fatalf("ValidateSeed() accepted unsafe path %q", tc.path)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSeedRejectsTooManyFiles(t *testing.T) {
|
||||
seed := testSeed()
|
||||
seed.Files = make([]SeedFile, DefaultMaxSeedFiles+1)
|
||||
for i := range seed.Files {
|
||||
seed.Files[i] = SeedFile{
|
||||
Path: "f" + strings.Repeat("0", i%3) + ".bin",
|
||||
Size: 1,
|
||||
Hashes: SeedHashes{MD5: strings.Repeat("1", 32)},
|
||||
}
|
||||
}
|
||||
if err := ValidateSeed(seed, DefaultParseLimits()); err == nil {
|
||||
t.Fatal("ValidateSeed() accepted more files than DefaultMaxSeedFiles")
|
||||
}
|
||||
}
|
||||
|
||||
// --- sliceMd5 canonical rule ---------------------------------------------------
|
||||
|
||||
func TestSliceMD5FromPiecesMatchesSpec(t *testing.T) {
|
||||
fileMD5 := strings.Repeat("a", 32)
|
||||
|
||||
// Zero pieces: fall back to the whole-file MD5.
|
||||
if got := SliceMD5FromPieces(nil, fileMD5); got != strings.ToUpper(fileMD5) {
|
||||
t.Fatalf("SliceMD5FromPieces(nil) = %q, want %q", got, strings.ToUpper(fileMD5))
|
||||
}
|
||||
// A single piece covers the whole file, so it equals the file MD5.
|
||||
single := strings.ToUpper(fileMD5)
|
||||
if got := SliceMD5FromPieces([]string{single}, fileMD5); got != single {
|
||||
t.Fatalf("SliceMD5FromPieces(single) = %q, want %q", got, single)
|
||||
}
|
||||
// Two or more pieces: MD5 of the newline-joined, upper-cased piece list.
|
||||
pieces := []string{strings.Repeat("b", 32), strings.Repeat("c", 32)}
|
||||
want := strings.ToUpper(GetMD5Str(strings.Join([]string{strings.Repeat("B", 32), strings.Repeat("C", 32)}, "\n")))
|
||||
if got := SliceMD5FromPieces(pieces, fileMD5); got != want {
|
||||
t.Fatalf("SliceMD5FromPieces(multi) = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSliceMD5AgreesWithBuildCASInfo locks the rule shared by the hash-generation
|
||||
// side and the CAS-encoding side. A divergence here silently turns rapid uploads
|
||||
// into hash mismatches, so the two entry points must never drift apart.
|
||||
func TestSliceMD5AgreesWithBuildCASInfo(t *testing.T) {
|
||||
fileMD5 := strings.Repeat("a", 32)
|
||||
sets := [][]string{
|
||||
nil,
|
||||
{strings.Repeat("b", 32)},
|
||||
{strings.Repeat("b", 32), strings.Repeat("c", 32)},
|
||||
{strings.Repeat("b", 32), strings.Repeat("c", 32), strings.Repeat("d", 32)},
|
||||
}
|
||||
for i, pieces := range sets {
|
||||
hw := &HashWriter{sliceMD5Hexs: pieces}
|
||||
fromWriter := hw.GetSliceMD5(fileMD5)
|
||||
fromCAS := BuildCASInfoFromMD5s(fileMD5, pieces, DefaultPieceSize).SliceMD5
|
||||
if fromWriter != fromCAS {
|
||||
t.Fatalf("case %d: GetSliceMD5() = %q, BuildCASInfoFromMD5s() = %q", i, fromWriter, fromCAS)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- bencode robustness --------------------------------------------------------
|
||||
|
||||
func TestBencodeDecodeRejectsOversizedStringLength(t *testing.T) {
|
||||
// Declares a 4GiB string while the buffer is empty; parsing must fail on the
|
||||
// declared length instead of attempting a huge allocation.
|
||||
payload := []byte("9999999999:")
|
||||
if _, err := BencodeDecode(payload); err == nil {
|
||||
t.Fatal("BencodeDecode() accepted an out-of-bounds string length")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBencodeDecodeRejectsDeepNesting(t *testing.T) {
|
||||
depth := DefaultParseLimits().MaxDepth + 2
|
||||
payload := strings.Repeat("l", depth) + strings.Repeat("e", depth)
|
||||
if _, err := BencodeDecode([]byte(payload)); err == nil {
|
||||
t.Fatal("BencodeDecode() accepted nesting beyond MaxDepth")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBencodeDecodeRejectsTrailingData(t *testing.T) {
|
||||
if _, err := BencodeDecode([]byte("i1eextra")); err == nil {
|
||||
t.Fatal("BencodeDecode() accepted trailing data")
|
||||
}
|
||||
}
|
||||
|
||||
// --- cross-format conversion consistency --------------------------------------
|
||||
|
||||
// TestConvertConsistencyAcrossFormats ensures a seed survives OSS -> torrent ->
|
||||
// OSS and OSS -> CAS -> OSS without losing whole-file hashes.
|
||||
func TestConvertConsistencyAcrossFormats(t *testing.T) {
|
||||
original := testSeed()
|
||||
|
||||
torrentData, err := EncodeSeed(original, "torrent")
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeSeed(torrent) error = %v", err)
|
||||
}
|
||||
fromTorrent, err := DecodeSeed(torrentData, "torrent", DefaultParseLimits())
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeSeed(torrent) error = %v", err)
|
||||
}
|
||||
casData, err := EncodeCAS(fromTorrent)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeCAS() error = %v", err)
|
||||
}
|
||||
fromCAS, err := DecodeCAS(casData, DefaultParseLimits())
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeCAS() error = %v", err)
|
||||
}
|
||||
if got := fromCAS.Files[0].Hashes.MD5; got != strings.ToUpper(original.Files[0].Hashes.MD5) {
|
||||
t.Fatalf("MD5 changed across formats: %q", got)
|
||||
}
|
||||
if len(fromCAS.Files) != len(original.Files) {
|
||||
t.Fatalf("file count changed across formats: %d", len(fromCAS.Files))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeSeedRejectsUnknownFormat(t *testing.T) {
|
||||
data, err := EncodeOSS(testSeed())
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeOSS() error = %v", err)
|
||||
}
|
||||
if _, err := DecodeSeed(data, "does-not-exist", DefaultParseLimits()); err == nil {
|
||||
t.Fatal("DecodeSeed() accepted an unknown format")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package torrent
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testSeed() *Seed {
|
||||
return &Seed{
|
||||
Format: OSSFormat,
|
||||
Version: OSSVersion,
|
||||
Name: "example.bin",
|
||||
CreatedAt: time.Unix(1, 0).UTC().Format(time.RFC3339),
|
||||
CreatedBy: "OpenList",
|
||||
PieceSize: DefaultPieceSize,
|
||||
Files: []SeedFile{{
|
||||
Path: "example.bin",
|
||||
Size: DefaultPieceSize + 1,
|
||||
Hashes: SeedHashes{
|
||||
MD5: strings.Repeat("1", 32),
|
||||
SHA1: strings.Repeat("2", 40),
|
||||
SHA256: strings.Repeat("3", 64),
|
||||
Pieces: &SeedPieceHashes{
|
||||
MD5: []string{strings.Repeat("4", 32), strings.Repeat("5", 32)},
|
||||
SHA1: []string{strings.Repeat("6", 40), strings.Repeat("7", 40)},
|
||||
SHA256: []string{strings.Repeat("8", 64), strings.Repeat("9", 64)},
|
||||
},
|
||||
},
|
||||
}},
|
||||
}
|
||||
}
|
||||
|
||||
func TestOSSRoundTrip(t *testing.T) {
|
||||
encoded, err := EncodeOSS(testSeed())
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeOSS() error = %v", err)
|
||||
}
|
||||
decoded, err := DecodeOSS(encoded, DefaultParseLimits())
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeOSS() error = %v", err)
|
||||
}
|
||||
if decoded.Name != "example.bin" || len(decoded.Files) != 1 || decoded.Files[0].Hashes.SHA256 == "" {
|
||||
t.Fatalf("DecodeOSS() returned incomplete seed: %#v", decoded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCASWireFormatIsLegacyCompatible(t *testing.T) {
|
||||
encoded, err := EncodeCAS(testSeed())
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeCAS() error = %v", err)
|
||||
}
|
||||
decodedJSON, err := base64.StdEncoding.DecodeString(string(encoded))
|
||||
if err != nil {
|
||||
t.Fatalf("base64.DecodeString() error = %v", err)
|
||||
}
|
||||
var payload map[string]any
|
||||
if err = json.Unmarshal(decodedJSON, &payload); err != nil {
|
||||
t.Fatalf("json.Unmarshal() error = %v", err)
|
||||
}
|
||||
// The five legacy fields must always be present so the reference client can
|
||||
// parse the payload. slice_md5s / slice_size are optional extensions.
|
||||
for _, key := range []string{"name", "size", "md5", "sliceMd5", "create_time"} {
|
||||
if _, ok := payload[key]; !ok {
|
||||
t.Fatalf("CAS payload missing %q: %#v", key, payload)
|
||||
}
|
||||
}
|
||||
decoded, err := DecodeCAS(encoded, DefaultParseLimits())
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeCAS() error = %v", err)
|
||||
}
|
||||
if decoded.Files[0].Hashes.MD5 != strings.Repeat("1", 32) {
|
||||
t.Fatalf("DecodeCAS() MD5 = %q", decoded.Files[0].Hashes.MD5)
|
||||
}
|
||||
// Per-piece MD5 list must round-trip through the slice_md5s extension.
|
||||
if decoded.Files[0].Hashes.Pieces == nil || len(decoded.Files[0].Hashes.Pieces.MD5) != 2 {
|
||||
t.Fatalf("DecodeCAS() piece MD5 list = %#v", decoded.Files[0].Hashes.Pieces)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTorrentRoundTripPreservesOpenListExtension(t *testing.T) {
|
||||
encoded, err := EncodeSeed(testSeed(), "torrent")
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeSeed(torrent) error = %v", err)
|
||||
}
|
||||
decoded, err := DecodeSeed(encoded, "torrent", DefaultParseLimits())
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeSeed(torrent) error = %v", err)
|
||||
}
|
||||
if decoded.Files[0].Hashes.SHA256 != strings.Repeat("3", 64) {
|
||||
t.Fatalf("torrent extension lost SHA-256: %#v", decoded.Files[0].Hashes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCASCloudGeneralizationRoundTrip(t *testing.T) {
|
||||
seed := testSeed()
|
||||
seed.Files[0].CASSliceMD5 = strings.Repeat("a", 32)
|
||||
seed.Files[0].CASCloud = CloudAliyundriveOpen
|
||||
|
||||
// CAS (base64 JSON) round-trip must preserve the cloud identifier.
|
||||
encoded, err := EncodeCAS(seed)
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeCAS() error = %v", err)
|
||||
}
|
||||
decoded, err := DecodeCAS(encoded, DefaultParseLimits())
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeCAS() error = %v", err)
|
||||
}
|
||||
if got := decoded.Files[0].CASCloud; got != CloudAliyundriveOpen {
|
||||
t.Fatalf("DecodeCAS() cloud = %q, want %q", got, CloudAliyundriveOpen)
|
||||
}
|
||||
|
||||
// Torrent bencode round-trip must preserve the cloud identifier too.
|
||||
torrentData, err := EncodeSeed(seed, "torrent")
|
||||
if err != nil {
|
||||
t.Fatalf("EncodeSeed(torrent) error = %v", err)
|
||||
}
|
||||
decodedTorrent, err := DecodeSeed(torrentData, "torrent", DefaultParseLimits())
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeSeed(torrent) error = %v", err)
|
||||
}
|
||||
if got := decodedTorrent.Files[0].CASCloud; got != CloudAliyundriveOpen {
|
||||
t.Fatalf("DecodeSeed(torrent) cloud = %q, want %q", got, CloudAliyundriveOpen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildCASInfoFromMD5sDefaultsToCloud189(t *testing.T) {
|
||||
info := BuildCASInfoFromMD5s(strings.Repeat("1", 32), []string{strings.Repeat("4", 32)}, DefaultPieceSize)
|
||||
if info.Cloud != Cloud189 {
|
||||
t.Fatalf("BuildCASInfoFromMD5s() cloud = %q, want %q", info.Cloud, Cloud189)
|
||||
}
|
||||
other := BuildCASInfoFromMD5sWithCloud(strings.Repeat("1", 32), []string{strings.Repeat("4", 32)}, DefaultPieceSize, Cloud115)
|
||||
if other.Cloud != Cloud115 {
|
||||
t.Fatalf("BuildCASInfoFromMD5sWithCloud() cloud = %q, want %q", other.Cloud, Cloud115)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSeedRejectsTraversalAndInvalidHash(t *testing.T) {
|
||||
seed := testSeed()
|
||||
seed.Files[0].Path = "../secret"
|
||||
if err := ValidateSeed(seed, DefaultParseLimits()); err == nil {
|
||||
t.Fatal("ValidateSeed() accepted path traversal")
|
||||
}
|
||||
seed = testSeed()
|
||||
seed.Files[0].Hashes.MD5 = "not-a-hash"
|
||||
if err := ValidateSeed(seed, DefaultParseLimits()); err == nil {
|
||||
t.Fatal("ValidateSeed() accepted an invalid hash")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeSeedHonorsSizeLimit(t *testing.T) {
|
||||
data := []byte(`{"format":"openlist-sharing-seed"}`)
|
||||
limits := DefaultParseLimits()
|
||||
limits.MaxBytes = int64(len(data) - 1)
|
||||
if _, err := DecodeSeed(data, "oss", limits); err == nil {
|
||||
t.Fatal("DecodeSeed() accepted input above MaxBytes")
|
||||
}
|
||||
}
|
||||
+1036
-9
File diff suppressed because it is too large
Load Diff
+102
-5
@@ -63,6 +63,7 @@ type MoveCopyReq struct {
|
||||
Overwrite bool `json:"overwrite"`
|
||||
SkipExisting bool `json:"skip_existing"`
|
||||
Merge bool `json:"merge"`
|
||||
FollowSeed bool `json:"follow_seed"`
|
||||
}
|
||||
|
||||
// FsMove performs batch move (individual item permission checks skipped for performance).
|
||||
@@ -152,6 +153,14 @@ func FsMove(c *gin.Context) {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
if req.FollowSeed {
|
||||
seedTasks, followErr := followSeedTransfer(c, "move", p, dstDir)
|
||||
if followErr != nil {
|
||||
common.ErrorResp(c, followErr, 500)
|
||||
return
|
||||
}
|
||||
addedTasks = append(addedTasks, seedTasks...)
|
||||
}
|
||||
}
|
||||
|
||||
// Return immediately with task information
|
||||
@@ -261,6 +270,14 @@ func FsCopy(c *gin.Context) {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
if req.FollowSeed {
|
||||
seedTasks, followErr := followSeedTransfer(c, "copy", p, dstDir)
|
||||
if followErr != nil {
|
||||
common.ErrorResp(c, followErr, 500)
|
||||
return
|
||||
}
|
||||
addedTasks = append(addedTasks, seedTasks...)
|
||||
}
|
||||
}
|
||||
|
||||
// Return immediately with task information
|
||||
@@ -277,9 +294,10 @@ func FsCopy(c *gin.Context) {
|
||||
}
|
||||
|
||||
type RenameReq struct {
|
||||
Path string `json:"path"`
|
||||
Name string `json:"name"`
|
||||
Overwrite bool `json:"overwrite"`
|
||||
Path string `json:"path"`
|
||||
Name string `json:"name"`
|
||||
Overwrite bool `json:"overwrite"`
|
||||
FollowSeed bool `json:"follow_seed"`
|
||||
}
|
||||
|
||||
func FsRename(c *gin.Context) {
|
||||
@@ -324,6 +342,12 @@ func FsRename(c *gin.Context) {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
if req.FollowSeed {
|
||||
if err := followSeedRename(c, reqPath, req.Name); err != nil {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
}
|
||||
common.SuccessResp(c)
|
||||
}
|
||||
|
||||
@@ -335,8 +359,9 @@ func checkRelativePath(path string) error {
|
||||
}
|
||||
|
||||
type RemoveReq struct {
|
||||
Dir string `json:"dir"`
|
||||
Names []string `json:"names"`
|
||||
Dir string `json:"dir"`
|
||||
Names []string `json:"names"`
|
||||
FollowSeed bool `json:"follow_seed"`
|
||||
}
|
||||
|
||||
// FsRemove performs batch remove (individual item permission checks skipped for performance).
|
||||
@@ -384,16 +409,88 @@ func FsRemove(c *gin.Context) {
|
||||
if path == "" {
|
||||
continue
|
||||
}
|
||||
source, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true})
|
||||
err := fs.Remove(c.Request.Context(), path)
|
||||
if err != nil {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
if req.FollowSeed && source != nil && !source.IsDir() {
|
||||
if err = followSeedRemove(c, path); err != nil {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
//fs.ClearCache(req.Dir)
|
||||
common.SuccessResp(c)
|
||||
}
|
||||
|
||||
func seedSidecarPaths(filePath string) []string {
|
||||
return []string{filePath + ".oss", filePath + ".torrent", filePath + ".cas", filePath + ".cas.torrent"}
|
||||
}
|
||||
|
||||
func followSeedTransfer(c *gin.Context, operation, srcPath, dstDir string) ([]task.TaskExtensionInfo, error) {
|
||||
source, err := fs.Get(c.Request.Context(), srcPath, &fs.GetArgs{NoLog: true})
|
||||
if err != nil || source == nil || source.IsDir() {
|
||||
return nil, nil
|
||||
}
|
||||
var tasks []task.TaskExtensionInfo
|
||||
for _, sidecarPath := range seedSidecarPaths(srcPath) {
|
||||
obj, err := fs.Get(c.Request.Context(), sidecarPath, &fs.GetArgs{NoLog: true})
|
||||
if err != nil || obj == nil || obj.IsDir() {
|
||||
continue
|
||||
}
|
||||
var current task.TaskExtensionInfo
|
||||
switch operation {
|
||||
case "copy":
|
||||
current, err = fs.Copy(c.Request.Context(), sidecarPath, dstDir, true)
|
||||
case "move":
|
||||
current, err = fs.Move(c.Request.Context(), sidecarPath, dstDir, true)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported seed sidecar operation %q", operation)
|
||||
}
|
||||
if err != nil {
|
||||
return tasks, fmt.Errorf("%s seed sidecar %s: %w", operation, sidecarPath, err)
|
||||
}
|
||||
if current != nil {
|
||||
tasks = append(tasks, current)
|
||||
}
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
func followSeedRename(c *gin.Context, srcPath, newName string) error {
|
||||
source, err := fs.Get(c.Request.Context(), srcPath, &fs.GetArgs{NoLog: true})
|
||||
if err != nil || source == nil || source.IsDir() {
|
||||
return nil
|
||||
}
|
||||
for _, sidecarPath := range seedSidecarPaths(srcPath) {
|
||||
obj, err := fs.Get(c.Request.Context(), sidecarPath, &fs.GetArgs{NoLog: true})
|
||||
if err != nil || obj == nil || obj.IsDir() {
|
||||
continue
|
||||
}
|
||||
suffix := strings.TrimPrefix(sidecarPath, srcPath)
|
||||
if err = fs.Rename(c.Request.Context(), sidecarPath, newName+suffix, true); err != nil {
|
||||
return fmt.Errorf("rename seed sidecar %s: %w", sidecarPath, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func followSeedRemove(c *gin.Context, srcPath string) error {
|
||||
for _, sidecarPath := range seedSidecarPaths(srcPath) {
|
||||
obj, err := fs.Get(c.Request.Context(), sidecarPath, &fs.GetArgs{NoLog: true})
|
||||
if err != nil || obj == nil || obj.IsDir() {
|
||||
continue
|
||||
}
|
||||
if err = fs.Remove(c.Request.Context(), sidecarPath); err != nil {
|
||||
return fmt.Errorf("remove seed sidecar %s: %w", sidecarPath, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type RemoveEmptyDirectoryReq struct {
|
||||
SrcDir string `json:"src_dir"`
|
||||
}
|
||||
|
||||
+146
-2
@@ -1,19 +1,25 @@
|
||||
package handles
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
stdpath "path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/fs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/task"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/torrent"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/OpenListTeam/OpenList/v4/server/common"
|
||||
"github.com/gin-gonic/gin"
|
||||
@@ -97,6 +103,17 @@ func FsStream(c *gin.Context) {
|
||||
if len(mimetype) == 0 {
|
||||
mimetype = utils.GetMimeType(name)
|
||||
}
|
||||
generateSeed := shouldGenerateUploadSeed(c, dir)
|
||||
if generateSeed && asTask {
|
||||
common.ErrorStrResp(c, "seed sidecar generation requires synchronous upload", 400)
|
||||
return
|
||||
}
|
||||
var seedHasher *torrent.HashWriter
|
||||
var uploadReader io.Reader = c.Request.Body
|
||||
if generateSeed {
|
||||
seedHasher = torrent.NewHashWriter(seedPieceSize(c), seedPieceSize(c), 0)
|
||||
uploadReader = io.TeeReader(c.Request.Body, seedHasher)
|
||||
}
|
||||
s := &stream.FileStream{
|
||||
Obj: &model.Object{
|
||||
Name: name,
|
||||
@@ -104,7 +121,7 @@ func FsStream(c *gin.Context) {
|
||||
Modified: getLastModified(c),
|
||||
HashInfo: utils.NewHashInfoByMap(h),
|
||||
},
|
||||
Reader: c.Request.Body,
|
||||
Reader: uploadReader,
|
||||
Mimetype: mimetype,
|
||||
WebPutAsTask: asTask,
|
||||
}
|
||||
@@ -118,6 +135,12 @@ func FsStream(c *gin.Context) {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
if generateSeed {
|
||||
if err = writeUploadSeedSidecar(c, dir, name, size, seedHasher); err != nil {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
}
|
||||
if t == nil {
|
||||
common.SuccessResp(c)
|
||||
return
|
||||
@@ -194,6 +217,17 @@ func FsForm(c *gin.Context) {
|
||||
if len(mimetype) == 0 {
|
||||
mimetype = utils.GetMimeType(name)
|
||||
}
|
||||
generateSeed := shouldGenerateUploadSeed(c, dir)
|
||||
if generateSeed && asTask {
|
||||
common.ErrorStrResp(c, "seed sidecar generation requires synchronous upload", 400)
|
||||
return
|
||||
}
|
||||
var seedHasher *torrent.HashWriter
|
||||
var uploadReader io.Reader = f
|
||||
if generateSeed {
|
||||
seedHasher = torrent.NewHashWriter(seedPieceSize(c), seedPieceSize(c), 0)
|
||||
uploadReader = io.TeeReader(f, seedHasher)
|
||||
}
|
||||
s := &stream.FileStream{
|
||||
Obj: &model.Object{
|
||||
Name: name,
|
||||
@@ -201,7 +235,7 @@ func FsForm(c *gin.Context) {
|
||||
Modified: getLastModified(c),
|
||||
HashInfo: utils.NewHashInfoByMap(h),
|
||||
},
|
||||
Reader: f,
|
||||
Reader: uploadReader,
|
||||
Mimetype: mimetype,
|
||||
WebPutAsTask: asTask,
|
||||
}
|
||||
@@ -218,6 +252,12 @@ func FsForm(c *gin.Context) {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
if generateSeed {
|
||||
if err = writeUploadSeedSidecar(c, dir, name, file.Size, seedHasher); err != nil {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
}
|
||||
if t == nil {
|
||||
common.SuccessResp(c)
|
||||
return
|
||||
@@ -226,3 +266,107 @@ func FsForm(c *gin.Context) {
|
||||
"task": getTaskInfo(t),
|
||||
})
|
||||
}
|
||||
|
||||
func shouldGenerateUploadSeed(c *gin.Context, path string) bool {
|
||||
if strings.TrimSpace(c.GetHeader("X-Seed-Sidecars")) != "" {
|
||||
return true
|
||||
}
|
||||
policy := strings.ToLower(strings.TrimSpace(c.GetHeader("X-Generate-Seed")))
|
||||
if policy != "" && policy != "inherit" {
|
||||
return policy == "on" || policy == "true" || policy == "1"
|
||||
}
|
||||
if storage := op.GetBalancedStorage(path); storage != nil {
|
||||
policy = strings.ToLower(strings.TrimSpace(storage.GetStorage().SeedPolicy))
|
||||
}
|
||||
if policy == "" || policy == "inherit" {
|
||||
policy = strings.ToLower(setting.GetStr(conf.SeedAutoGeneratePolicy, "off"))
|
||||
}
|
||||
return (policy == "on" || policy == "true" || policy == "1") && configuredSeedFormats() != ""
|
||||
}
|
||||
|
||||
func seedPieceSize(c *gin.Context) int64 {
|
||||
for _, format := range strings.Split(strings.ToLower(c.GetHeader("X-Seed-Sidecars")), ",") {
|
||||
if strings.TrimSpace(format) == "cas" {
|
||||
return torrent.DefaultPieceSize
|
||||
}
|
||||
}
|
||||
value := c.GetHeader("X-Seed-Piece-Size")
|
||||
if value == "" {
|
||||
return torrent.DefaultPieceSize
|
||||
}
|
||||
size, err := strconv.ParseInt(value, 10, 64)
|
||||
if err != nil || size <= 0 || size > 1<<30 {
|
||||
return torrent.DefaultPieceSize
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
func configuredSeedFormats() string {
|
||||
policies := make(map[string]string)
|
||||
if err := json.Unmarshal([]byte(setting.GetStr(conf.SeedFormatPolicies)), &policies); err != nil {
|
||||
return ""
|
||||
}
|
||||
formats := make([]string, 0, 3)
|
||||
for _, format := range []string{"oss", "torrent", "cas"} {
|
||||
if strings.EqualFold(strings.TrimSpace(policies[format]), "on") {
|
||||
formats = append(formats, format)
|
||||
}
|
||||
}
|
||||
return strings.Join(formats, ",")
|
||||
}
|
||||
|
||||
func writeUploadSeedSidecar(c *gin.Context, dir, name string, expectedSize int64, hasher *torrent.HashWriter) error {
|
||||
if hasher == nil {
|
||||
return nil
|
||||
}
|
||||
hasher.Finish()
|
||||
if expectedSize >= 0 && hasher.GetTotalWritten() != expectedSize {
|
||||
return fmt.Errorf("seed sidecar requires a complete stream: read %d of %d bytes", hasher.GetTotalWritten(), expectedSize)
|
||||
}
|
||||
seed := torrent.NewSeed(name, "OpenList", seedPieceSize(c))
|
||||
formatsHeader := strings.TrimSpace(c.GetHeader("X-Seed-Sidecars"))
|
||||
if formatsHeader == "" {
|
||||
formatsHeader = strings.TrimSpace(c.GetHeader("X-Seed-Format"))
|
||||
}
|
||||
if formatsHeader == "" {
|
||||
formatsHeader = configuredSeedFormats()
|
||||
}
|
||||
formats, err := fs.NormalizeSeedFormats(strings.Split(formatsHeader, ","))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
matrix := SeedHashMatrix{}
|
||||
if rawMatrix := strings.TrimSpace(c.GetHeader("X-Seed-Hash-Matrix")); rawMatrix != "" {
|
||||
if err = json.Unmarshal([]byte(rawMatrix), &matrix); err != nil {
|
||||
return fmt.Errorf("invalid seed hash matrix: %w", err)
|
||||
}
|
||||
}
|
||||
matrix = normalizedSeedMatrix(matrix, formats)
|
||||
seedFile := hasher.BuildSeedFile(name, getLastModified(c).UTC().Format(time.RFC3339))
|
||||
applySeedMatrix(&seedFile, matrix)
|
||||
seed.Files = []torrent.SeedFile{seedFile}
|
||||
seen := make(map[string]struct{}, 3)
|
||||
for _, rawFormat := range formats {
|
||||
format := strings.ToLower(strings.TrimPrefix(strings.TrimSpace(rawFormat), "."))
|
||||
if format == "bt" {
|
||||
format = "torrent"
|
||||
}
|
||||
if _, exists := seen[format]; exists {
|
||||
continue
|
||||
}
|
||||
seen[format] = struct{}{}
|
||||
data, err := fs.EncodeGeneratedSeed(seed, format, hasher.GetPieceHashes())
|
||||
if err != nil {
|
||||
return fmt.Errorf("generate %s seed sidecar: %w", format, err)
|
||||
}
|
||||
sidecar := &stream.FileStream{
|
||||
Ctx: c.Request.Context(),
|
||||
Obj: &model.Object{Name: name + "." + format, Size: int64(len(data)), Modified: time.Now()},
|
||||
Reader: bytes.NewReader(data), Mimetype: "application/octet-stream",
|
||||
}
|
||||
if err = fs.PutDirectly(c.Request.Context(), dir, sidecar, true); err != nil {
|
||||
return fmt.Errorf("upload %s seed sidecar: %w", format, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
+1512
-12
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,259 @@
|
||||
package handles
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/torrent"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
// withSeedSite pins the configured seed site for the duration of a test.
|
||||
func withSeedSite(t *testing.T, site string) {
|
||||
t.Helper()
|
||||
previous := seedSiteURLProvider
|
||||
seedSiteURLProvider = func() string { return site }
|
||||
t.Cleanup(func() { seedSiteURLProvider = previous })
|
||||
}
|
||||
|
||||
func mustParse(t *testing.T, raw string) *url.URL {
|
||||
t.Helper()
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
t.Fatalf("url.Parse(%q) error = %v", raw, err)
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// TestSameSeedHostNormalizesDefaultPort guards against the regression where a
|
||||
// configured "https://pan.example.com" and an embedded
|
||||
// "https://pan.example.com:443/..." were treated as different hosts, silently
|
||||
// discarding otherwise valid sources.
|
||||
func TestSameSeedHostNormalizesDefaultPort(t *testing.T) {
|
||||
cases := []struct {
|
||||
a, b string
|
||||
want bool
|
||||
}{
|
||||
{"https://pan.example.com", "https://pan.example.com:443/x", true},
|
||||
{"http://pan.example.com", "http://pan.example.com:80/x", true},
|
||||
{"https://pan.example.com:8443", "https://pan.example.com:8443/x", true},
|
||||
{"https://pan.example.com:8443", "https://pan.example.com", false},
|
||||
{"https://pan.example.com", "http://pan.example.com", false},
|
||||
{"https://pan.example.com", "https://evil.example.com", false},
|
||||
{"https://pan.example.com", "https://pan.example.com.evil.com", false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := sameSeedHost(mustParse(t, tc.a), mustParse(t, tc.b)); got != tc.want {
|
||||
t.Errorf("sameSeedHost(%q, %q) = %v, want %v", tc.a, tc.b, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateSeedHostRejectsForeignHosts is the core SSRF guarantee: a seed
|
||||
// source may only ever point at the operator-configured site.
|
||||
func TestValidateSeedHostRejectsForeignHosts(t *testing.T) {
|
||||
withSeedSite(t, "https://pan.example.com")
|
||||
|
||||
rejected := []string{
|
||||
"http://169.254.169.254/latest/meta-data/", // cloud metadata
|
||||
"http://127.0.0.1:5244/api/fs/list", // local admin API
|
||||
"http://localhost:5244/d/secret",
|
||||
"https://evil.example.com/d/secret",
|
||||
"https://pan.example.com.evil.com/d/x",
|
||||
"file:///etc/passwd",
|
||||
"ftp://pan.example.com/x",
|
||||
"https://user:pass@pan.example.com/d/x", // embedded credentials
|
||||
}
|
||||
for _, raw := range rejected {
|
||||
if err := validateSeedHost(mustParse(t, raw)); err == nil {
|
||||
t.Errorf("validateSeedHost(%q) accepted a disallowed URL", raw)
|
||||
}
|
||||
}
|
||||
|
||||
allowed := []string{
|
||||
"https://pan.example.com/d/some/file",
|
||||
"https://pan.example.com:443/sd/abc123",
|
||||
}
|
||||
for _, raw := range allowed {
|
||||
if err := validateSeedHost(mustParse(t, raw)); err != nil {
|
||||
t.Errorf("validateSeedHost(%q) rejected a valid URL: %v", raw, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateSeedHostWithoutConfiguredSite(t *testing.T) {
|
||||
withSeedSite(t, "")
|
||||
if err := validateSeedHost(mustParse(t, "https://pan.example.com/d/x")); err == nil {
|
||||
t.Fatal("validateSeedHost() accepted a source while seed_site_url is unset")
|
||||
}
|
||||
}
|
||||
|
||||
// TestRedirectGuardBlocksSSRF is the regression test for the bypass: the first
|
||||
// hop passes the host allow-list, but a redirect must not be allowed to escape
|
||||
// to an internal address.
|
||||
func TestRedirectGuardBlocksSSRF(t *testing.T) {
|
||||
withSeedSite(t, "https://pan.example.com")
|
||||
|
||||
origin := mustParse(t, "https://pan.example.com/d/file")
|
||||
check := seedSourceHTTPClient.CheckRedirect
|
||||
|
||||
// A redirect staying on the configured site is fine.
|
||||
sameHost := &http.Request{URL: mustParse(t, "https://pan.example.com/d/file-2")}
|
||||
if err := check(sameHost, []*http.Request{{URL: origin}}); err != nil {
|
||||
t.Fatalf("CheckRedirect() rejected a same-host redirect: %v", err)
|
||||
}
|
||||
|
||||
// A redirect to the cloud metadata endpoint must be refused.
|
||||
metadata := &http.Request{URL: mustParse(t, "http://169.254.169.254/latest/meta-data/")}
|
||||
if err := check(metadata, []*http.Request{{URL: origin}}); err == nil {
|
||||
t.Fatal("CheckRedirect() allowed a redirect to the cloud metadata endpoint")
|
||||
}
|
||||
|
||||
// A redirect to localhost must be refused.
|
||||
local := &http.Request{URL: mustParse(t, "http://127.0.0.1:5244/api/fs/list")}
|
||||
if err := check(local, []*http.Request{{URL: origin}}); err == nil {
|
||||
t.Fatal("CheckRedirect() allowed a redirect to localhost")
|
||||
}
|
||||
|
||||
// A same-host redirect that downgrades https -> http must be refused.
|
||||
downgrade := &http.Request{URL: mustParse(t, "http://pan.example.com/d/file")}
|
||||
if err := check(downgrade, []*http.Request{{URL: origin}}); err == nil {
|
||||
t.Fatal("CheckRedirect() allowed a scheme downgrade")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedirectGuardLimitsHopCount(t *testing.T) {
|
||||
withSeedSite(t, "https://pan.example.com")
|
||||
|
||||
origin := mustParse(t, "https://pan.example.com/d/file")
|
||||
via := make([]*http.Request, maxSeedSourceRedirects)
|
||||
for i := range via {
|
||||
via[i] = &http.Request{URL: origin}
|
||||
}
|
||||
next := &http.Request{URL: mustParse(t, "https://pan.example.com/d/file-2")}
|
||||
if err := seedSourceHTTPClient.CheckRedirect(next, via); err == nil {
|
||||
t.Fatal("CheckRedirect() accepted more redirects than maxSeedSourceRedirects")
|
||||
}
|
||||
}
|
||||
|
||||
// TestValidateSeedSourceEnforcesPathPrefix documents the per-type path contract.
|
||||
func TestValidateSeedSourceEnforcesPathPrefix(t *testing.T) {
|
||||
withSeedSite(t, "https://pan.example.com")
|
||||
|
||||
if err := validateSeedSource(torrent.SeedSource{
|
||||
Type: "openlist-direct",
|
||||
URL: "https://pan.example.com/sd/abc",
|
||||
}); err == nil {
|
||||
t.Fatal("validateSeedSource() accepted a share path for a direct source")
|
||||
}
|
||||
if err := validateSeedSource(torrent.SeedSource{
|
||||
Type: "openlist-share",
|
||||
URL: "https://pan.example.com/d/file",
|
||||
}); err == nil {
|
||||
t.Fatal("validateSeedSource() accepted a direct path for a share source")
|
||||
}
|
||||
if err := validateSeedSource(torrent.SeedSource{
|
||||
Type: "openlist-direct",
|
||||
URL: "https://evil.example.com/d/file",
|
||||
}); err == nil {
|
||||
t.Fatal("validateSeedSource() accepted a foreign host")
|
||||
}
|
||||
if err := validateSeedSource(torrent.SeedSource{
|
||||
Type: "openlist-direct",
|
||||
URL: "https://pan.example.com/d/file",
|
||||
}); err != nil {
|
||||
t.Fatalf("validateSeedSource() rejected a valid direct source: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFirstUsableSeedSourceSkipsExpiredAndForeign verifies the selection logic
|
||||
// only returns sources that are both on-site and not expired.
|
||||
func TestFirstUsableSeedSourceSkipsExpiredAndForeign(t *testing.T) {
|
||||
withSeedSite(t, "https://pan.example.com")
|
||||
|
||||
file := torrent.SeedFile{
|
||||
Sources: []torrent.SeedSource{
|
||||
{Type: "openlist-direct", URL: "https://evil.example.com/d/a"},
|
||||
{Type: "openlist-direct", URL: "https://pan.example.com/d/b", ExpiresAt: "2000-01-01T00:00:00Z"},
|
||||
{Type: "openlist-share", URL: "https://pan.example.com/sd/good"},
|
||||
},
|
||||
}
|
||||
if got := firstUsableSeedSource(file); got != "https://pan.example.com/sd/good" {
|
||||
t.Fatalf("firstUsableSeedSource() = %q, want the valid share source", got)
|
||||
}
|
||||
|
||||
// No usable source at all.
|
||||
none := torrent.SeedFile{
|
||||
Sources: []torrent.SeedSource{
|
||||
{Type: "openlist-direct", URL: "https://evil.example.com/d/a"},
|
||||
},
|
||||
}
|
||||
if got := firstUsableSeedSource(none); got != "" {
|
||||
t.Fatalf("firstUsableSeedSource() = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildSeedRapidUploadRequestRejectsMultiFile documents that a multi-file
|
||||
// torrent cannot be described by a single rapid-upload request. Returning nil
|
||||
// (instead of silently using Files[0] with the aggregate size) prevents sending
|
||||
// the destination a size/hash combination that contradicts itself.
|
||||
func TestBuildSeedRapidUploadRequestRejectsMultiFile(t *testing.T) {
|
||||
const md5Hex = "0123456789abcdef0123456789abcdef"
|
||||
|
||||
multi := &torrent.Torrent{
|
||||
OpenList: &torrent.Seed{
|
||||
PieceSize: torrent.DefaultPieceSize,
|
||||
Files: []torrent.SeedFile{
|
||||
{Path: "a.bin", Size: 10, Hashes: torrent.SeedHashes{MD5: md5Hex}},
|
||||
{Path: "b.bin", Size: 20, Hashes: torrent.SeedHashes{MD5: md5Hex}},
|
||||
},
|
||||
},
|
||||
}
|
||||
if req := buildSeedRapidUploadRequest(multi, nil); req != nil {
|
||||
t.Fatalf("buildSeedRapidUploadRequest() accepted a multi-file torrent: %#v", req)
|
||||
}
|
||||
|
||||
// A single file must still work.
|
||||
single := &torrent.Torrent{
|
||||
Info: torrent.TorrentInfo{Name: "a.bin", Length: 10},
|
||||
OpenList: &torrent.Seed{
|
||||
PieceSize: torrent.DefaultPieceSize,
|
||||
Files: []torrent.SeedFile{
|
||||
{Path: "a.bin", Size: 10, Hashes: torrent.SeedHashes{MD5: md5Hex}},
|
||||
},
|
||||
},
|
||||
}
|
||||
req := buildSeedRapidUploadRequest(single, nil)
|
||||
if req == nil {
|
||||
t.Fatal("buildSeedRapidUploadRequest() rejected a valid single-file torrent")
|
||||
}
|
||||
if req.Size != 10 {
|
||||
t.Fatalf("buildSeedRapidUploadRequest() size = %d, want 10", req.Size)
|
||||
}
|
||||
if got := req.Whole.GetHash(utils.MD5); !strings.EqualFold(got, md5Hex) {
|
||||
t.Fatalf("buildSeedRapidUploadRequest() MD5 = %q, want %q", got, md5Hex)
|
||||
}
|
||||
|
||||
// A single file whose metadata size disagrees with the torrent length is
|
||||
// internally inconsistent and must also be refused.
|
||||
mismatched := &torrent.Torrent{
|
||||
Info: torrent.TorrentInfo{Name: "a.bin", Length: 99},
|
||||
OpenList: &torrent.Seed{
|
||||
PieceSize: torrent.DefaultPieceSize,
|
||||
Files: []torrent.SeedFile{
|
||||
{Path: "a.bin", Size: 10, Hashes: torrent.SeedHashes{MD5: md5Hex}},
|
||||
},
|
||||
},
|
||||
}
|
||||
if req := buildSeedRapidUploadRequest(mismatched, nil); req != nil {
|
||||
t.Fatalf("buildSeedRapidUploadRequest() accepted a size/hash mismatch: %#v", req)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildSeedRapidUploadRequestHandlesNil(t *testing.T) {
|
||||
if req := buildSeedRapidUploadRequest(nil, nil); req != nil {
|
||||
t.Fatalf("buildSeedRapidUploadRequest(nil) = %#v, want nil", req)
|
||||
}
|
||||
}
|
||||
@@ -234,6 +234,19 @@ func _fs(g *gin.RouterGroup) {
|
||||
g.POST("/torrent/upload_parse", handles.UploadTorrentAndParse)
|
||||
g.POST("/torrent/rapid_upload", handles.TorrentRapidUpload)
|
||||
g.POST("/torrent/generate", handles.GenerateTorrentForPath)
|
||||
// Unified transfer seed APIs. Legacy torrent routes above remain supported.
|
||||
seed := g.Group("/seed")
|
||||
seed.POST("/parse", handles.ParseSeed)
|
||||
seed.POST("/upload_parse", handles.UploadSeedAndParse)
|
||||
seed.POST("/generate", handles.GenerateSeedForPaths)
|
||||
seed.POST("/convert", handles.ConvertSeed)
|
||||
seed.POST("/diagnose", handles.DiagnoseSeed)
|
||||
seed.POST("/capabilities", handles.SeedCapabilities)
|
||||
seed.POST("/rapid_upload", handles.QuickSaveSeed)
|
||||
seed.POST("/offline_download", handles.QuickSaveSeed)
|
||||
seed.POST("/update", handles.UpdateSeed)
|
||||
seed.POST("/quick_save", handles.QuickSaveSeed)
|
||||
seed.POST("/update_channels", handles.UpdateSeedChannels)
|
||||
// Direct upload (client-side upload to storage)
|
||||
g.POST("/get_direct_upload_info", handles.FsGetDirectUploadInfo)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user