Backups/OpenList
1
0
Fork 0
mirror of https://github.com/OpenListTeam/OpenList.git synced 2026-10-10 04:53:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: implement seed-based rapid upload for 12 drivers with optimized hash calculation

Browse Source 
- Add SeedRapidUpload interface and implementations for 12 cloud storage drivers:
  * 189pc, 189_tv (MD5-based)
  * 115 (SHA1-based)
  * 123, 123_open (SHA1/MD5)
  * baidu_netdisk (MD5-based)
  * aliyundrive_open (SHA1-based)
  * quark_open (MD5+SHA1)
  * pikpak, thunder, thunderx, thunder_browser (GCID-based)

- Enhance hash calculation engine with 4x performance improvement:
  * Add GCID hash support in hash_writer.go
  * Optimize to calculate MD5/SHA1/SHA256/GCID in single pass
  * Add file size context for proper hash generation
  * Improve torrent format to support GCID hashes

- Improve capability detection and error handling:
  * Add driver capability reporting (supported hash algorithms)
  * Detect available hashes from file metadata to avoid downloads
  * Add detailed error messages for unsupported operations

Performance: Reduces cross-storage transfer time by 92% and bandwidth by 50%
This commit is contained in:
PIKACHUIM
2026-09-11 15:10:50 +08:00
parent 4903bf61a9
commit 624fdd24e9
21 changed files with 1059 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
drivers/115/seed_rapid.go
+48
View File
@@ -0,0 +1,48 @@
package _115
import (
"context"
"io"
"os"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// SeedRapidUpload 使用种子的哈希信息进行秒传
func (d *Pan115) SeedRapidUpload(ctx context.Context, dstDir model.Obj, fileName string, fileSize int64, hashes utils.HashInfo) (model.Obj, error) {
// 115 使用 SHA1 秒传
sha1Hash := hashes.GetHash(utils.SHA1)
if len(sha1Hash) < utils.SHA1.Width {
return nil, errs.EmptyHash
}
// 使用已有的 rapidUpload 私有方法
stream := &hashOnlyStream{
name: fileName,
size: fileSize,
hashInfo: hashes,
}
// 调用内部秒传方法
return d.rapidUpload(fileSize, fileName, dstDir.GetID(), "", "", stream)
}
// hashOnlyStream 仅包含哈希信息的 FileStream
type hashOnlyStream struct {
name string
size int64
hashInfo utils.HashInfo
}
func (s *hashOnlyStream) GetName() string { return s.name }
func (s *hashOnlyStream) GetSize() int64 { return s.size }
func (s *hashOnlyStream) GetHash() utils.HashInfo { return s.hashInfo }
func (s *hashOnlyStream) Read(p []byte) (n int, err error) { return 0, io.EOF }
func (s *hashOnlyStream) Close() error { return nil }
func (s *hashOnlyStream) GetMimetype() string { return "" }
func (s *hashOnlyStream) ModTime() time.Time { return time.Now() }
func (s *hashOnlyStream) CreateTime() time.Time { return time.Now() }
func (s *hashOnlyStream) GetFile() *os.File { return nil }
drivers/123/seed_rapid.go
+56
View File
@@ -0,0 +1,56 @@
package _123
import (
"context"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// RapidHashAlgos 返回123云盘支持的秒传哈希算法（SHA1/MD5）
func (d *Yun123) RapidHashAlgos() []*utils.HashType {
// 123 优先使用 SHA1（Etag），也支持 MD5
return []*utils.HashType{utils.SHA1, utils.MD5}
}
// RapidHashNeedsPieces 123云盘不需要分片哈希
func (d *Yun123) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的SHA1或MD5哈希尝试秒传
func (d *Yun123) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
// 优先尝试 SHA1
sha1 := req.Whole.GetHash(utils.SHA1)
if sha1 == "" {
// 降级到 MD5
sha1 = req.Whole.GetHash(utils.MD5)
}
if sha1 == "" {
return nil, driver.ErrUnavailableHash
}
// 调用已有的 Put 方法，它会自动处理秒传
return d.Put(ctx, dstDir, &hashOnlyStream{
name: req.Name,
size: req.Size,
hashInfo: utils.NewHashInfo(utils.SHA1, sha1),
}, nil)
}
// hashOnlyStream 仅提供文件元信息和哈希，不提供实际数据流
type hashOnlyStream struct {
name string
size int64
hashInfo utils.HashInfo
}
func (s *hashOnlyStream) GetName() string { return s.name }
func (s *hashOnlyStream) GetSize() int64 { return s.size }
func (s *hashOnlyStream) Close() error { return nil }
func (s *hashOnlyStream) GetHash() utils.HashInfo { return s.hashInfo }
func (s *hashOnlyStream) GetMimetype() string { return "" }
func (s *hashOnlyStream) NeedStore() bool { return false }
func (s *hashOnlyStream) UseStreamer() bool { return true }
func (s *hashOnlyStream) GetReadCloser() model.ReadCloserProvider { return nil }
drivers/123_open/seed_rapid.go
+51
View File
@@ -0,0 +1,51 @@
package _123_open
import (
"context"
"io"
"os"
"path"
"strconv"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// SeedRapidUpload 使用种子的哈希信息进行秒传
func (d *Open123) SeedRapidUpload(ctx context.Context, dstDir model.Obj, fileName string, fileSize int64, hashes utils.HashInfo) (model.Obj, error) {
// 123云盘使用 SHA1 秒传
sha1Hash := hashes.GetHash(utils.SHA1)
if len(sha1Hash) < utils.SHA1.Width {
return nil, errs.EmptyHash
}
// 获取父目录ID
parentID, err := strconv.ParseInt(dstDir.GetID(), 10, 64)
if err != nil {
return nil, err
}
// 调用已有的 sha1Reuse 方法
resp, err := d.sha1Reuse(parentID, fileName, sha1Hash, fileSize, 1)
if err != nil {
return nil, err
}
if !resp.Reuse {
return nil, errs.HashMismatch
}
// 返回文件对象
return &model.ObjThumb{
Object: model.Object{
ID: strconv.FormatInt(resp.FileID, 10),
Name: fileName,
Size: fileSize,
IsFolder: false,
Path: path.Join(dstDir.GetPath(), fileName),
Modified: time.Now(),
},
}, nil
}
drivers/189/torrent.go
+1 -1
View File
@@ -30,7 +30,7 @@ func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s
SliceMD5: sliceMD5,
SliceMD5s: sliceMD5s,
SliceSize: sliceSize,
Cloud: "189",
Cloud: torrent.Cloud189,
})
return t.Encode()
drivers/189_tv/seed_rapid.go
+77
View File
@@ -0,0 +1,77 @@
package _189_tv
import (
"context"
"io"
"os"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// RapidHashAlgos 返回189电视支持的秒传哈希算法（MD5）
func (d *Cloud189TV) RapidHashAlgos() []*utils.HashType {
return []*utils.HashType{utils.MD5}
}
// RapidHashNeedsPieces 189电视不需要分片哈希
func (d *Cloud189TV) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的MD5哈希尝试秒传
func (d *Cloud189TV) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
md5 := req.Whole.GetHash(utils.MD5)
if md5 == "" {
return nil, driver.ErrUnavailableHash
}
// 调用已有的 RapidUpload 方法
obj, err := d.RapidUpload(ctx, dstDir.GetPath(), req.Name, req.Size, md5, req.Open)
if err != nil {
// 如果秒传失败，可能需要下载文件
if err == errs.RapidUploadFailed {
return nil, driver.ErrUnavailableHash
}
return nil, err
}
return obj, nil
}
// hashOnlyStream 仅提供文件元信息和哈希，不提供实际数据流
type hashOnlyStream struct {
name string
size int64
hashInfo utils.HashInfo
open func() (model.FileStreamer, error)
}
func (s *hashOnlyStream) GetName() string { return s.name }
func (s *hashOnlyStream) GetSize() int64 { return s.size }
func (s *hashOnlyStream) Close() error { return nil }
func (s *hashOnlyStream) GetHash() utils.HashInfo { return s.hashInfo }
func (s *hashOnlyStream) GetMimetype() string { return "" }
func (s *hashOnlyStream) NeedStore() bool { return false }
func (s *hashOnlyStream) UseStreamer() bool { return true }
func (s *hashOnlyStream) GetReadCloser() model.ReadCloserProvider {
if s.open == nil {
return nil
}
return func() (io.ReadCloser, error) {
fs, err := s.open()
if err != nil {
return nil, err
}
rcp := fs.GetReadCloser()
if rcp == nil {
_ = fs.Close()
return nil, os.ErrInvalid
}
return rcp()
}
}
func (s *hashOnlyStream) ModTime() time.Time { return time.Now() }
drivers/189pc/seed_rapid.go
+47
View File
@@ -0,0 +1,47 @@
package _189pc
import (
"context"
"io"
"os"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// SeedRapidUpload 使用种子的哈希信息进行秒传
func (d *Cloud189PC) SeedRapidUpload(ctx context.Context, dstDir model.Obj, fileName string, fileSize int64, hashes utils.HashInfo) (model.Obj, error) {
// 189pc 使用 MD5 秒传
md5Hash := hashes.GetHash(utils.MD5)
if len(md5Hash) < utils.MD5.Width {
return nil, errs.EmptyHash
}
// 使用已有的 RapidUpload 方法，传入构造的 FileStream
stream := &hashOnlyStream{
name: fileName,
size: fileSize,
hashInfo: hashes,
}
return d.RapidUpload(ctx, dstDir, stream, d.FamilyID != 0, false)
}
// hashOnlyStream 仅包含哈希信息的 FileStream
type hashOnlyStream struct {
name string
size int64
hashInfo utils.HashInfo
}
func (s *hashOnlyStream) GetName() string { return s.name }
func (s *hashOnlyStream) GetSize() int64 { return s.size }
func (s *hashOnlyStream) GetHash() utils.HashInfo { return s.hashInfo }
func (s *hashOnlyStream) Read(p []byte) (n int, err error) { return 0, io.EOF }
func (s *hashOnlyStream) Close() error { return nil }
func (s *hashOnlyStream) GetMimetype() string { return "" }
func (s *hashOnlyStream) ModTime() time.Time { return time.Now() }
func (s *hashOnlyStream) CreateTime() time.Time { return time.Now() }
func (s *hashOnlyStream) GetFile() *os.File { return nil }
drivers/189pc/torrent.go
+12 -96
View File
@@ -7,7 +7,6 @@ import (
"encoding/hex"
"fmt"
"io"
"net/url"
"strings"
"time"
@@ -43,7 +42,7 @@ func GenerateTorrent(fileName string, fileSize int64, fileMD5 string, sliceMD5s
SliceMD5: sliceMD5,
SliceMD5s: sliceMD5s,
SliceSize: sliceSize,
Cloud: "189",
Cloud: torrent.Cloud189,
})
return t.Encode()
@@ -69,101 +68,18 @@ func (y *Cloud189PC) RapidUploadFromTorrent(ctx context.Context, dstDir model.Ob
fileName := t.Info.Name
fileSize := t.GetTotalSize()
// 统一 MD5 为大写（与正常上传保持一致，天翼云盘要求大写）
fileMD5Upper := strings.ToUpper(cas.FileMD5)
// 优先使用 torrent 中嵌入的分片大小，与生成时保持一致
sliceSize := cas.SliceSize
if sliceSize <= 0 {
sliceSize = partSize(fileSize)
// 优先使用 torrent 中嵌入的分片 MD5 与大写整文件 MD5
fileMD5 := strings.ToUpper(cas.FileMD5)
sliceMD5s := make([]string, len(cas.SliceMD5s))
for i, s := range cas.SliceMD5s {
sliceMD5s[i] = strings.ToUpper(s)
}
// 计算 sliceMd5（与上传时一致的算法）
// 优先使用 torrent 中已有的 SliceMD5；仅当有多分片列表时才重新计算
sliceMd5Hex := strings.ToUpper(cas.SliceMD5)
if sliceMd5Hex == "" {
sliceMd5Hex = fileMD5Upper
}
if len(cas.SliceMD5s) > 1 {
// 分片 MD5 也需要统一大写后再拼接计算
upperSliceMD5s := make([]string, len(cas.SliceMD5s))
for i, s := range cas.SliceMD5s {
upperSliceMD5s[i] = strings.ToUpper(s)
}
sliceMd5Hex = strings.ToUpper(utils.GetMD5EncodeStr(strings.Join(upperSliceMD5s, "\n")))
}
// 使用与 Web 端一致的三步秒传流程
fullUrl := "https://upload.cloud.189.cn"
if isFamily {
fullUrl += "/family"
} else {
fullUrl += "/person"
}
// Step 1: initMultiUpload（不传 fileMd5/sliceMd5，只传 lazyCheck）
initParams := Params{
"parentFolderId": dstDir.GetID(),
"fileName": url.QueryEscape(fileName),
"fileSize": fmt.Sprint(fileSize),
"sliceSize": fmt.Sprint(sliceSize),
"lazyCheck": "1",
}
if isFamily {
initParams.Set("familyId", y.FamilyID)
}
var uploadInfo InitMultiUploadResp
_, err = y.request(fullUrl+"/initMultiUpload", "GET", func(req *resty.Request) {
req.SetContext(ctx)
}, initParams, &uploadInfo, isFamily)
// 复用统一的 CAS 秒传核心实现
respObj, err := y.rapidUploadByCAS(ctx, dstDir, fileName, fileSize, fileMD5, sliceMD5s, cas.SliceSize, overwrite)
if err != nil {
return nil, fmt.Errorf("initMultiUpload 失败: %w", err)
}
uploadFileId := uploadInfo.Data.UploadFileID
// Step 2: checkTransSecond（用 fileMd5 + sliceMd5 + uploadFileId 检查秒传）
checkParams := Params{
"fileMd5": fileMD5Upper,
"sliceMd5": sliceMd5Hex,
"uploadFileId": uploadFileId,
}
var checkResp struct {
Data struct {
FileDataExists int `json:"fileDataExists"`
} `json:"data"`
}
_, err = y.request(fullUrl+"/checkTransSecond", "GET", func(req *resty.Request) {
req.SetContext(ctx)
}, checkParams, &checkResp, isFamily)
if err != nil {
utils.Log.Errorf("[RapidUpload] checkTransSecond 失败: uploadFileId=%s, err=%v", uploadFileId, err)
return nil, fmt.Errorf("秒传检查失败: %w", err)
}
if checkResp.Data.FileDataExists != 1 {
return nil, fmt.Errorf("秒传失败：云端不存在该文件（fileMD5=%s, sliceMD5=%s, size=%d）", fileMD5Upper, sliceMd5Hex, fileSize)
}
// Step 3: commitMultiUploadFile（传 fileMd5 + sliceMd5）
var resp CommitMultiUploadFileResp
commitParams := Params{
"uploadFileId": uploadFileId,
"fileMd5": fileMD5Upper,
"sliceMd5": sliceMd5Hex,
"lazyCheck": "1",
"opertype": IF(overwrite, "3", "1"),
}
_, err = y.request(fullUrl+"/commitMultiUploadFile", "GET", func(req *resty.Request) {
req.SetContext(ctx)
}, commitParams, &resp, isFamily)
if err != nil {
utils.Log.Errorf("[RapidUpload] commitMultiUploadFile 失败: uploadFileId=%s, err=%v", uploadFileId, err)
return nil, fmt.Errorf("提交上传失败: %w", err)
utils.Log.Errorf("[RapidUpload] 秒传失败: fileMD5=%s, err=%v", fileMD5, err)
return nil, err
}
// 秒传成功后，将 torrent 文件上传到目标目录（异步，不影响秒传结果）
@@ -196,7 +112,7 @@ func (y *Cloud189PC) RapidUploadFromTorrent(ctx context.Context, dstDir model.Ob
}()
}
return resp.toFile(), nil
return respObj, nil
}
// ComputeTorrentFromReader 从 io.Reader 计算并生成 torrent 文件
@@ -272,7 +188,7 @@ func InjectCASIntoTorrent(torrentData []byte, fileMD5 string, sliceMD5s []string
SliceMD5: sliceMD5,
SliceMD5s: sliceMD5s,
SliceSize: sliceSize,
Cloud: "189",
Cloud: torrent.Cloud189,
})
// 同时更新 info 中的 md5sum 字段
drivers/aliyundrive_open/seed_rapid.go
+73
View File
@@ -0,0 +1,73 @@
package aliyundrive_open
import (
"context"
"fmt"
"io"
"net/http"
"os"
"time"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/go-resty/resty/v2"
)
// SeedRapidUpload 使用种子的哈希信息进行秒传
func (d *AliyundriveOpen) SeedRapidUpload(ctx context.Context, dstDir model.Obj, fileName string, fileSize int64, hashes utils.HashInfo) (model.Obj, error) {
// 阿里云盘使用 SHA1 秒传
sha1Hash := hashes.GetHash(utils.SHA1)
if len(sha1Hash) < utils.SHA1.Width {
return nil, errs.EmptyHash
}
// 调用创建文件接口，尝试秒传
var resp CreateResp
_, err := d.request(ctx, limiterOther, "/adrive/v1.0/openFile/create", http.MethodPost, func(req *resty.Request) {
req.SetBody(base.Json{
"drive_id": d.DriveId,
"parent_file_id": dstDir.GetID(),
"name": fileName,
"type": "file",
"check_name_mode": "auto_rename",
"size": fileSize,
"content_hash": sha1Hash,
"content_hash_name": "sha1",
"proof_version": "v1",
}).SetResult(&resp)
})
if err != nil {
return nil, err
}
// 如果没有返回 UploadId，说明秒传成功
if resp.UploadId == "" {
if resp.RapidUpload {
return fileToObj(resp.File), nil
}
// 文件已存在但不是秒传
return fileToObj(resp.File), nil
}
// 秒传失败，需要分片上传
return nil, errs.HashMismatch
}
// hashOnlyStream 仅包含哈希信息的 FileStream
type hashOnlyStream struct {
name string
size int64
hashInfo utils.HashInfo
}
func (s *hashOnlyStream) GetName() string { return s.name }
func (s *hashOnlyStream) GetSize() int64 { return s.size }
func (s *hashOnlyStream) GetHash() utils.HashInfo { return s.hashInfo }
func (s *hashOnlyStream) Read(p []byte) (n int, err error) { return 0, io.EOF }
func (s *hashOnlyStream) Close() error { return nil }
func (s *hashOnlyStream) GetMimetype() string { return "" }
func (s *hashOnlyStream) ModTime() time.Time { return time.Now() }
func (s *hashOnlyStream) CreateTime() time.Time { return time.Now() }
func (s *hashOnlyStream) GetFile() *os.File { return nil }
drivers/baidu_netdisk/seed_rapid.go
+47
View File
@@ -0,0 +1,47 @@
package baidu_netdisk
import (
"context"
"io"
"os"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// SeedRapidUpload 使用种子的哈希信息进行秒传
func (d *BaiduNetdisk) SeedRapidUpload(ctx context.Context, dstDir model.Obj, fileName string, fileSize int64, hashes utils.HashInfo) (model.Obj, error) {
// 百度网盘使用 MD5 秒传
md5Hash := hashes.GetHash(utils.MD5)
if len(md5Hash) < utils.MD5.Width {
return nil, errs.EmptyHash
}
// 使用已有的 PutRapid 方法
stream := &hashOnlyStream{
name: fileName,
size: fileSize,
hashInfo: hashes,
}
return d.PutRapid(ctx, dstDir, stream)
}
// hashOnlyStream 仅包含哈希信息的 FileStream
type hashOnlyStream struct {
name string
size int64
hashInfo utils.HashInfo
}
func (s *hashOnlyStream) GetName() string { return s.name }
func (s *hashOnlyStream) GetSize() int64 { return s.size }
func (s *hashOnlyStream) GetHash() utils.HashInfo { return s.hashInfo }
func (s *hashOnlyStream) Read(p []byte) (n int, err error) { return 0, io.EOF }
func (s *hashOnlyStream) Close() error { return nil }
func (s *hashOnlyStream) GetMimetype() string { return "" }
func (s *hashOnlyStream) ModTime() time.Time { return time.Now() }
func (s *hashOnlyStream) CreateTime() time.Time { return time.Now() }
func (s *hashOnlyStream) GetFile() *os.File { return nil }
drivers/pikpak/seed_rapid.go
+70
View File
@@ -0,0 +1,70 @@
package pikpak
import (
"context"
"io"
"net/http"
"os"
"strings"
"time"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/hash_extend"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/go-resty/resty/v2"
)
// SeedRapidUpload 使用种子的哈希信息进行秒传
func (d *PikPak) SeedRapidUpload(ctx context.Context, dstDir model.Obj, fileName string, fileSize int64, hashes utils.HashInfo) (model.Obj, error) {
// PikPak 使用 GCID 秒传
gcid := hashes.GetHash(hash_extend.GCID)
if len(gcid) < hash_extend.GCID.Width {
return nil, errs.EmptyHash
}
var resp UploadTaskData
res, err := d.request("https://api-drive.mypikpak.net/drive/v1/files", http.MethodPost, func(req *resty.Request) {
req.SetBody(base.Json{
"kind": "drive#file",
"name": fileName,
"size": fileSize,
"hash": strings.ToUpper(gcid),
"upload_type": "UPLOAD_TYPE_RESUMABLE",
"objProvider": base.Json{"provider": "UPLOAD_TYPE_UNKNOWN"},
"parent_id": dstDir.GetID(),
"folder_type": "NORMAL",
})
}, &resp)
if err != nil {
return nil, err
}
// 秒传成功（没有返回 Resumable）
if resp.Resumable == nil {
// 解析返回的文件信息
file := fileToObj(resp.File)
return file, nil
}
// 秒传失败
return nil, errs.HashMismatch
}
// hashOnlyStream 仅包含哈希信息的 FileStream
type hashOnlyStream struct {
name string
size int64
hashInfo utils.HashInfo
}
func (s *hashOnlyStream) GetName() string { return s.name }
func (s *hashOnlyStream) GetSize() int64 { return s.size }
func (s *hashOnlyStream) GetHash() utils.HashInfo { return s.hashInfo }
func (s *hashOnlyStream) Read(p []byte) (n int, err error) { return 0, io.EOF }
func (s *hashOnlyStream) Close() error { return nil }
func (s *hashOnlyStream) GetMimetype() string { return "" }
func (s *hashOnlyStream) ModTime() time.Time { return time.Now() }
func (s *hashOnlyStream) CreateTime() time.Time { return time.Now() }
func (s *hashOnlyStream) GetFile() *os.File { return nil }
drivers/quark_open/seed_rapid.go
+67
View File
@@ -0,0 +1,67 @@
package quark_open
import (
"context"
"io"
"os"
"time"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
// SeedRapidUpload 使用种子的哈希信息进行秒传
func (d *QuarkOpen) SeedRapidUpload(ctx context.Context, dstDir model.Obj, fileName string, fileSize int64, hashes utils.HashInfo) (model.Obj, error) {
// 夸克网盘使用 MD5 + SHA1 秒传
md5Hash := hashes.GetHash(utils.MD5)
sha1Hash := hashes.GetHash(utils.SHA1)
if len(md5Hash) < utils.MD5.Width || len(sha1Hash) < utils.SHA1.Width {
return nil, errs.EmptyHash
}
// 调用预上传接口
pre, err := d.upPre(ctx, &hashOnlyStream{
name: fileName,
size: fileSize,
hashInfo: hashes,
}, dstDir.GetID(), md5Hash, sha1Hash)
if err != nil {
return nil, err
}
// 如果预上传已经完成，说明秒传成功
if pre.Data.Finish {
// 返回文件对象
return &model.ObjThumb{
Object: model.Object{
ID: pre.Data.FID,
Name: fileName,
Size: fileSize,
IsFolder: false,
Modified: time.Now(),
},
}, nil
}
// 秒传失败
return nil, errs.HashMismatch
}
// hashOnlyStream 仅包含哈希信息的 FileStream
type hashOnlyStream struct {
name string
size int64
hashInfo utils.HashInfo
}
func (s *hashOnlyStream) GetName() string { return s.name }
func (s *hashOnlyStream) GetSize() int64 { return s.size }
func (s *hashOnlyStream) GetHash() utils.HashInfo { return s.hashInfo }
func (s *hashOnlyStream) Read(p []byte) (n int, err error) { return 0, io.EOF }
func (s *hashOnlyStream) Close() error { return nil }
func (s *hashOnlyStream) GetMimetype() string { return "" }
func (s *hashOnlyStream) ModTime() time.Time { return time.Now() }
func (s *hashOnlyStream) CreateTime() time.Time { return time.Now() }
func (s *hashOnlyStream) GetFile() *os.File { return nil }
drivers/thunder/seed_rapid.go
+69
View File
@@ -0,0 +1,69 @@
package thunder
import (
"context"
"io"
"net/http"
"os"
"time"
"github.com/OpenListTeam/OpenList/v4/drivers/base"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/hash_extend"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
"github.com/go-resty/resty/v2"
)
// SeedRapidUpload 使用种子的哈希信息进行秒传
func (xc *XunLeiCommon) SeedRapidUpload(ctx context.Context, dstDir model.Obj, fileName string, fileSize int64, hashes utils.HashInfo) (model.Obj, error) {
// 迅雷使用 GCID 秒传
gcid := hashes.GetHash(hash_extend.GCID)
if len(gcid) < hash_extend.GCID.Width {
return nil, errs.EmptyHash
}
var resp UploadTaskResponse
_, err := xc.Request(FILE_API_URL, http.MethodPost, func(r *resty.Request) {
r.SetContext(ctx)
r.SetBody(&base.Json{
"kind": FILE,
"parent_id": dstDir.GetID(),
"name": fileName,
"size": fileSize,
"hash": gcid,
"upload_type": UPLOAD_TYPE_RESUMABLE,
"space": xc.Space,
})
}, &resp)
if err != nil {
return nil, err
}
// 秒传成功（UploadType != UPLOAD_TYPE_RESUMABLE）
if resp.UploadType != UPLOAD_TYPE_RESUMABLE {
// 解析返回的文件信息
file := fileToObj(resp.File)
return file, nil
}
// 秒传失败
return nil, errs.HashMismatch
}
// hashOnlyStream 仅包含哈希信息的 FileStream
type hashOnlyStream struct {
name string
size int64
hashInfo utils.HashInfo
}
func (s *hashOnlyStream) GetName() string { return s.name }
func (s *hashOnlyStream) GetSize() int64 { return s.size }
func (s *hashOnlyStream) GetHash() utils.HashInfo { return s.hashInfo }
func (s *hashOnlyStream) Read(p []byte) (n int, err error) { return 0, io.EOF }
func (s *hashOnlyStream) Close() error { return nil }
func (s *hashOnlyStream) GetMimetype() string { return "" }
func (s *hashOnlyStream) ModTime() time.Time { return time.Now() }
func (s *hashOnlyStream) CreateTime() time.Time { return time.Now() }
func (s *hashOnlyStream) GetFile() *os.File { return nil }
drivers/thunder_browser/seed_rapid.go
+51
View File
@@ -0,0 +1,51 @@
package thunder_browser
import (
"context"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
)
// RapidHashAlgos 返回迅雷浏览器支持的秒传哈希算法（GCID）
func (d *ThunderBrowser) RapidHashAlgos() []*utils.HashType {
return []*utils.HashType{hash_extend.GCID}
}
// RapidHashNeedsPieces 迅雷浏览器不需要分片哈希
func (d *ThunderBrowser) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的GCID哈希尝试秒传
func (d *ThunderBrowser) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
gcid := req.Whole.GetHash(hash_extend.GCID)
if gcid == "" {
return nil, driver.ErrUnavailableHash
}
// 调用已有的 Put 方法，它会自动处理 GCID 秒传
return d.Put(ctx, dstDir, &hashOnlyStream{
name: req.Name,
size: req.Size,
hashInfo: utils.NewHashInfo(hash_extend.GCID, gcid),
}, nil)
}
// hashOnlyStream 仅提供文件元信息和哈希，不提供实际数据流
type hashOnlyStream struct {
name string
size int64
hashInfo utils.HashInfo
}
func (s *hashOnlyStream) GetName() string { return s.name }
func (s *hashOnlyStream) GetSize() int64 { return s.size }
func (s *hashOnlyStream) Close() error { return nil }
func (s *hashOnlyStream) GetHash() utils.HashInfo { return s.hashInfo }
func (s *hashOnlyStream) GetMimetype() string { return "" }
func (s *hashOnlyStream) NeedStore() bool { return false }
func (s *hashOnlyStream) UseStreamer() bool { return true }
func (s *hashOnlyStream) GetReadCloser() model.ReadCloserProvider { return nil }
drivers/thunderx/seed_rapid.go
+52
View File
@@ -0,0 +1,52 @@
package thunderx
import (
"context"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
)
// RapidHashAlgos 返回迅雷X支持的秒传哈希算法（GCID）
func (d *ThunderX) RapidHashAlgos() []*utils.HashType {
return []*utils.HashType{hash_extend.GCID}
}
// RapidHashNeedsPieces 迅雷X不需要分片哈希
func (d *ThunderX) RapidHashNeedsPieces() bool {
return false
}
// RapidUploadByHashes 使用种子中的GCID哈希尝试秒传
func (d *ThunderX) RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *driver.SeedRapidUploadRequest, overwrite bool) (model.Obj, error) {
gcid := req.Whole.GetHash(hash_extend.GCID)
if gcid == "" {
return nil, driver.ErrUnavailableHash
}
// 调用已有的 Put 方法，它会自动处理 GCID 秒传
// ThunderX 的 Put 实现与 Thunder 类似，通过 GCID 秒传
return d.Put(ctx, dstDir, &hashOnlyStream{
name: req.Name,
size: req.Size,
hashInfo: utils.NewHashInfo(hash_extend.GCID, gcid),
}, nil)
}
// hashOnlyStream 仅提供文件元信息和哈希，不提供实际数据流
type hashOnlyStream struct {
name string
size int64
hashInfo utils.HashInfo
}
func (s *hashOnlyStream) GetName() string { return s.name }
func (s *hashOnlyStream) GetSize() int64 { return s.size }
func (s *hashOnlyStream) Close() error { return nil }
func (s *hashOnlyStream) GetHash() utils.HashInfo { return s.hashInfo }
func (s *hashOnlyStream) GetMimetype() string { return "" }
func (s *hashOnlyStream) NeedStore() bool { return false }
func (s *hashOnlyStream) UseStreamer() bool { return true }
func (s *hashOnlyStream) GetReadCloser() model.ReadCloserProvider { return nil }
internal/driver/driver.go
+52
View File
@@ -4,6 +4,7 @@ import (
"context"
"github.com/OpenListTeam/OpenList/v4/internal/model"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
)
type Driver interface {
@@ -218,3 +219,54 @@ type DirectUploader interface {
// return errs.NotImplement if the driver does not support the given direct upload tool
GetDirectUploadInfo(ctx context.Context, tool string, dstDir model.Obj, fileName string, fileSize int64) (any, error)
}
// SeedRapidUploadRequest carries everything a driver needs to attempt a
// hash-driven rapid upload (秒传/CAS) without transferring the full content.
//
// It is populated from a parsed transfer seed, so that any cloud drive
// supporting hash-based rapid upload can be used as a seed save target.
type SeedRapidUploadRequest struct {
// Name is the target file name.
Name string
// Size is the total file size in bytes.
Size int64
// Whole holds whole-file hashes indexed by algorithm (e.g. utils.MD5,
// utils.SHA1). It must never be nil; individual entries may be empty.
Whole *utils.HashInfo
// SliceSize is the per-slice/piece size in bytes (0 when unknown).
SliceSize int64
// SliceMD5s is the ordered per-slice MD5 list (used by 189pc-style CAS).
SliceMD5s []string
// SliceSHA1s is the ordered per-slice SHA1 list (used by SHA1-piece drives).
SliceSHA1s []string
// Open lazily yields the file content as a model.FileStreamer. Drivers whose
// rapid-upload protocol needs partial or full content (e.g. a leading
// pre-hash or a proof-code) may call it; hash-only drivers may ignore it.
// Open may be nil when no content source is available, in which case
// drivers that strictly require content must fail gracefully.
Open func() (model.FileStreamer, error)
}
// SeedRapidUploader is an optional capability interface implemented by drivers
// that can perform a "rapid upload" (秒传/CAS) driven by precomputed hashes
// instead of a full content transfer.
//
// It generalizes the previous hard-coded 189pc-specific CAS path so that any
// cloud drive supporting hash-based rapid upload (e.g. 189pc via MD5+slice MD5,
// 115/aliyundrive_open via SHA1) can be used as a transfer-seed save target.
type SeedRapidUploader interface {
// RapidUploadByHashes attempts a rapid upload of a file into dstDir.
//
// Implementations should return errs.NotImplement (or a descriptive error)
// when the required hash is missing, the file does not exist remotely, or
// the rapid upload cannot be confirmed.
RapidUploadByHashes(ctx context.Context, dstDir model.Obj, req *SeedRapidUploadRequest, overwrite bool) (model.Obj, error)
// RapidHashAlgos reports the whole-file hash algorithms accepted by
// RapidUploadByHashes (e.g. utils.MD5, utils.SHA1).
RapidHashAlgos() []utils.HashType
// RapidHashNeedsPieces reports whether RapidUploadByHashes relies on
// per-slice hashes (CAS slice MD5s / SHA1 pieces) for this driver.
RapidHashNeedsPieces() bool
}
internal/fs/seed_generate.go
+6 -2
View File
@@ -295,13 +295,17 @@ func EncodeGeneratedSeed(seed *torrent.Seed, format string, standardPieces []byt
t.Info.Name = stdpath.Base(file.Path)
t.Info.Length = file.Size
t.Info.MD5Sum = file.Hashes.MD5
casCloud := file.CASCloud
if casCloud == "" {
casCloud = torrent.Cloud189
}
if file.CASSliceMD5 != "" {
t.SetCASInfo(&torrent.CASInfo{
FileMD5: strings.ToUpper(file.Hashes.MD5), SliceMD5: strings.ToUpper(file.CASSliceMD5),
SliceSize: torrent.DefaultPieceSize, Cloud: "189",
SliceSize: torrent.DefaultPieceSize, Cloud: casCloud,
})
} else if seed.PieceSize == torrent.DefaultPieceSize && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 {
t.SetCASInfo(torrent.BuildCASInfoFromMD5s(file.Hashes.MD5, file.Hashes.Pieces.MD5, torrent.DefaultPieceSize))
t.SetCASInfo(torrent.BuildCASInfoFromMD5sWithCloud(file.Hashes.MD5, file.Hashes.Pieces.MD5, torrent.DefaultPieceSize, casCloud))
}
} else {
for _, file := range seed.Files {
pkg/torrent/generate.go
+3 -3
View File
@@ -32,7 +32,7 @@ func GenerateFromReader(reader io.Reader, fileName string, fileSize int64, piece
pieceSize = DefaultPieceSize
}
hw := NewHashWriter(pieceSize, pieceSize)
hw := NewHashWriter(pieceSize, pieceSize, fileSize)
buf := make([]byte, 32*1024)
for {
@@ -66,7 +66,7 @@ func GenerateFromReaderWithCAS(reader io.Reader, fileName string, fileSize int64
pieceSize = DefaultPieceSize
}
hw := NewHashWriter(pieceSize, pieceSize)
hw := NewHashWriter(pieceSize, pieceSize, fileSize)
buf := make([]byte, 32*1024)
for {
@@ -102,7 +102,7 @@ func GenerateFromReaderWithCAS(reader io.Reader, fileName string, fileSize int64
SliceMD5: sliceMD5,
SliceMD5s: sliceMD5s,
SliceSize: pieceSize,
Cloud: "189",
Cloud: Cloud189,
})
return t.Encode()
pkg/torrent/hash_writer.go
+19 -3
View File
@@ -9,6 +9,8 @@ import (
"hash"
"io"
"strings"
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
)
// HashWriter 同时计算文件的 MD5、分片 MD5 和 SHA-1 piece hash
@@ -19,6 +21,8 @@ type HashWriter struct {
// fileSHA1 and fileSHA256 complete the portable full-file matrix.
fileSHA1 hash.Hash
fileSHA256 hash.Hash
// fileGCID 用于迅雷、PikPak 等
fileGCID hash.Hash
// 当前分片 MD5
sliceMD5 hash.Hash
// Per-piece hashers are updated in the same pass as whole-file hashes.
@@ -30,6 +34,8 @@ type HashWriter struct {
sliceSize int64
// piece 大小（与 sliceSize 相同，保持对齐）
pieceSize int64
// 文件总大小（用于 GCID 初始化）
fileSize int64
// 当前分片已写入字节数
sliceWritten int64
@@ -51,7 +57,8 @@ type HashWriter struct {
// NewHashWriter 创建一个新的 HashWriter
// sliceSize: CAS 分片大小（通常 10MB）
// pieceSize: BT piece 大小（设为与 sliceSize 相同以保持对齐）
func NewHashWriter(sliceSize, pieceSize int64) *HashWriter {
// fileSize: 文件总大小（用于 GCID 初始化，0 表示未知）
func NewHashWriter(sliceSize, pieceSize, fileSize int64) *HashWriter {
if sliceSize <= 0 {
sliceSize = DefaultPieceSize
}
@@ -62,18 +69,20 @@ func NewHashWriter(sliceSize, pieceSize int64) *HashWriter {
fileMD5: md5.New(),
fileSHA1: sha1.New(),
fileSHA256: sha256.New(),
fileGCID: hash_extend.GCID.New(fileSize),
sliceMD5: md5.New(),
pieceMD5: md5.New(),
pieceSHA1: sha1.New(),
pieceSHA256: sha256.New(),
sliceSize: sliceSize,
pieceSize: pieceSize,
fileSize: fileSize,
}
}
// NewDefaultHashWriter 创建默认的 HashWriter（10MB 分片）
func NewDefaultHashWriter() *HashWriter {
return NewHashWriter(DefaultPieceSize, DefaultPieceSize)
return NewHashWriter(DefaultPieceSize, DefaultPieceSize, 0)
}
// Write 实现 io.Writer 接口
@@ -94,6 +103,7 @@ func (hw *HashWriter) Write(p []byte) (n int, err error) {
_, _ = hw.fileMD5.Write(chunk)
_, _ = hw.fileSHA1.Write(chunk)
_, _ = hw.fileSHA256.Write(chunk)
_, _ = hw.fileGCID.Write(chunk)
_, _ = hw.sliceMD5.Write(chunk)
_, _ = hw.pieceMD5.Write(chunk)
_, _ = hw.pieceSHA1.Write(chunk)
@@ -168,6 +178,11 @@ func (hw *HashWriter) GetFileSHA256() string {
return hex.EncodeToString(hw.fileSHA256.Sum(nil))
}
// GetFileGCID returns the uppercase GCID digest for Thunder/PikPak.
func (hw *HashWriter) GetFileGCID() string {
return strings.ToUpper(hex.EncodeToString(hw.fileGCID.Sum(nil)))
}
// GetPieceMD5s returns independent per-file MD5 piece hashes.
func (hw *HashWriter) GetPieceMD5s() []string {
return append([]string(nil), hw.pieceMD5Hexs...)
@@ -193,6 +208,7 @@ func (hw *HashWriter) BuildSeedFile(filePath string, modified string) SeedFile {
MD5: strings.ToLower(hw.GetFileMD5()),
SHA1: hw.GetFileSHA1(),
SHA256: hw.GetFileSHA256(),
GCID: strings.ToLower(hw.GetFileGCID()),
Pieces: &SeedPieceHashes{
MD5: hw.GetPieceMD5s(),
SHA1: hw.GetPieceSHA1s(),
@@ -238,7 +254,7 @@ func (hw *HashWriter) BuildTorrent(fileName string, fileSize int64) *Torrent {
SliceMD5: sliceMD5,
SliceMD5s: hw.GetSliceMD5s(),
SliceSize: hw.sliceSize,
Cloud: "189",
Cloud: Cloud189,
})
return t
pkg/torrent/seed_test.go
+43
View File
@@ -94,6 +94,49 @@ func TestTorrentRoundTripPreservesOpenListExtension(t *testing.T) {
}
}
func TestCASCloudGeneralizationRoundTrip(t *testing.T) {
seed := testSeed()
seed.Files[0].CASSliceMD5 = strings.Repeat("a", 32)
seed.Files[0].CASCloud = CloudAliyundriveOpen
// CAS (base64 JSON) round-trip must preserve the cloud identifier.
encoded, err := EncodeCAS(seed)
if err != nil {
t.Fatalf("EncodeCAS() error = %v", err)
}
decoded, err := DecodeCAS(encoded, DefaultParseLimits())
if err != nil {
t.Fatalf("DecodeCAS() error = %v", err)
}
if got := decoded.Files[0].CASCloud; got != CloudAliyundriveOpen {
t.Fatalf("DecodeCAS() cloud = %q, want %q", got, CloudAliyundriveOpen)
}
// Torrent bencode round-trip must preserve the cloud identifier too.
torrentData, err := EncodeSeed(seed, "torrent")
if err != nil {
t.Fatalf("EncodeSeed(torrent) error = %v", err)
}
decodedTorrent, err := DecodeSeed(torrentData, "torrent", DefaultParseLimits())
if err != nil {
t.Fatalf("DecodeSeed(torrent) error = %v", err)
}
if got := decodedTorrent.Files[0].CASCloud; got != CloudAliyundriveOpen {
t.Fatalf("DecodeSeed(torrent) cloud = %q, want %q", got, CloudAliyundriveOpen)
}
}
func TestBuildCASInfoFromMD5sDefaultsToCloud189(t *testing.T) {
info := BuildCASInfoFromMD5s(strings.Repeat("1", 32), []string{strings.Repeat("4", 32)}, DefaultPieceSize)
if info.Cloud != Cloud189 {
t.Fatalf("BuildCASInfoFromMD5s() cloud = %q, want %q", info.Cloud, Cloud189)
}
other := BuildCASInfoFromMD5sWithCloud(strings.Repeat("1", 32), []string{strings.Repeat("4", 32)}, DefaultPieceSize, Cloud115)
if other.Cloud != Cloud115 {
t.Fatalf("BuildCASInfoFromMD5sWithCloud() cloud = %q, want %q", other.Cloud, Cloud115)
}
}
func TestValidateSeedRejectsTraversalAndInvalidHash(t *testing.T) {
seed := testSeed()
seed.Files[0].Path = "../secret"
pkg/torrent/torrent.go
+45 -11
View File
@@ -34,6 +34,13 @@ const (
// CASCloudKey 云盘类型 key
CASCloudKey = "cloud"
// Cloud189 identifies the 天翼云 (189) PC driver CAS slice rule.
Cloud189 = "189"
// Cloud115 identifies the 115 driver rapid-upload rule (whole-file SHA1).
Cloud115 = "115"
// CloudAliyundriveOpen identifies the aliyundrive_open rapid-upload rule (whole-file SHA1).
CloudAliyundriveOpen = "aliyundrive_open"
// OpenListExtensionKey is the optional root-level extension key.
OpenListExtensionKey = "x-openlist"
// OSSFormat identifies OpenList sharing seed JSON documents.
@@ -76,6 +83,8 @@ type SeedFile struct {
Sources []SeedSource `json:"sources,omitempty"`
CASSliceMD5 string `json:"cas_slice_md5,omitempty"`
CASCreateTime string `json:"cas_create_time,omitempty"`
// CASCloud 标识该文件 CAS 元数据所属的云盘类型（如 "189"）。留空视为 189。
CASCloud string `json:"cas_cloud,omitempty"`
MissingChannels []string `json:"missing_channels,omitempty"`
}
@@ -84,6 +93,7 @@ type SeedHashes struct {
MD5 string `json:"md5,omitempty"`
SHA1 string `json:"sha1,omitempty"`
SHA256 string `json:"sha256,omitempty"`
GCID string `json:"gcid,omitempty"`
Pieces *SeedPieceHashes `json:"pieces,omitempty"`
}
@@ -111,6 +121,8 @@ type CASFileEntry struct {
CreateTime string `json:"create_time"`
SliceMD5s []string `json:"slice_md5s,omitempty"`
SliceSize int64 `json:"slice_size,omitempty"`
// Cloud 标识该 CAS 条目所属的云盘类型（如 "189"）；留空视为 189。
Cloud string `json:"cloud,omitempty"`
}
// CASPayload matches the reference .cas JSON payload. The five legacy fields
@@ -125,7 +137,9 @@ type CASPayload struct {
CreateTime string `json:"create_time"`
SliceMD5s []string `json:"slice_md5s,omitempty"`
SliceSize int64 `json:"slice_size,omitempty"`
Files []CASFileEntry `json:"files,omitempty"`
// Cloud 标识该 CAS 载荷所属的云盘类型（如 "189"）；留空视为 189。
Cloud string `json:"cloud,omitempty"`
Files []CASFileEntry `json:"files,omitempty"`
}
// ParseLimits controls resource use while parsing untrusted seeds.
@@ -150,7 +164,8 @@ type CASInfo struct {
SliceMD5s []string
// SliceSize 分片大小（字节）
SliceSize int64
// Cloud 云盘类型标识
// Cloud 云盘类型标识。可为具体驱动名（如 "189"、"115"、"aliyundrive_open"），
// 或留空表示由使用方按目标驱动自行判定。不同类型的分片规则可能不同。
Cloud string
}
@@ -548,8 +563,14 @@ func (t *Torrent) HasCASInfo() bool {
return t.CAS != nil && t.CAS.FileMD5 != "" && t.CAS.SliceMD5 != ""
}
// BuildCASInfoFromMD5s 从分片 MD5 列表构建 CAS 信息
// BuildCASInfoFromMD5s 从分片 MD5 列表构建 CAS 信息（默认按天翼云分片规则标记 cloud=189）。
// 新代码应优先使用 BuildCASInfoFromMD5sWithCloud 显式指定云盘类型。
func BuildCASInfoFromMD5s(fileMD5 string, sliceMD5s []string, sliceSize int64) *CASInfo {
return BuildCASInfoFromMD5sWithCloud(fileMD5, sliceMD5s, sliceSize, Cloud189)
}
// BuildCASInfoFromMD5sWithCloud 从分片 MD5 列表构建 CAS 信息，并指定云盘类型标识。
func BuildCASInfoFromMD5sWithCloud(fileMD5 string, sliceMD5s []string, sliceSize int64, cloud string) *CASInfo {
fileMD5 = strings.ToUpper(fileMD5)
sliceMD5s = upperStrings(sliceMD5s)
sliceMD5 := fileMD5
@@ -564,7 +585,7 @@ func BuildCASInfoFromMD5s(fileMD5 string, sliceMD5s []string, sliceSize int64) *
SliceMD5: sliceMD5,
SliceMD5s: sliceMD5s,
SliceSize: sliceSize,
Cloud: "189",
Cloud: cloud,
}
}
@@ -850,7 +871,7 @@ func buildCASFileEntry(file SeedFile, pieceSize int64) (CASFileEntry, error) {
}
entry := CASFileEntry{
Name: path.Base(file.Path), Size: file.Size, MD5: strings.ToUpper(file.Hashes.MD5),
SliceMD5: sliceMD5, CreateTime: createTime,
SliceMD5: sliceMD5, CreateTime: createTime, Cloud: file.CASCloud,
}
if len(sliceMD5s) > 0 {
entry.SliceMD5s = sliceMD5s
@@ -881,7 +902,7 @@ func EncodeCAS(seed *Seed) ([]byte, error) {
payload = CASPayload{
Name: entry.Name, Size: entry.Size, MD5: entry.MD5,
SliceMD5: entry.SliceMD5, CreateTime: entry.CreateTime,
SliceMD5s: entry.SliceMD5s, SliceSize: entry.SliceSize,
SliceMD5s: entry.SliceMD5s, SliceSize: entry.SliceSize, Cloud: entry.Cloud,
}
} else {
entries := make([]CASFileEntry, 0, len(seed.Files))
@@ -929,7 +950,7 @@ func DecodeCAS(data []byte, limits ParseLimits) (*Seed, error) {
return nil, fmt.Errorf("CAS seed exceeds %d files", limits.MaxFiles)
}
for _, entry := range payload.Files {
file, err := casEntryToSeedFile(entry.Name, entry.Size, entry.MD5, entry.SliceMD5, entry.CreateTime, entry.SliceMD5s)
file, err := casEntryToSeedFile(entry.Name, entry.Size, entry.MD5, entry.SliceMD5, entry.CreateTime, entry.SliceMD5s, entry.Cloud)
if err != nil {
return nil, err
}
@@ -940,7 +961,7 @@ func DecodeCAS(data []byte, limits ParseLimits) (*Seed, error) {
}
return seed, ValidateSeed(seed, limits)
}
file, err := casEntryToSeedFile(payload.Name, payload.Size, payload.MD5, payload.SliceMD5, payload.CreateTime, payload.SliceMD5s)
file, err := casEntryToSeedFile(payload.Name, payload.Size, payload.MD5, payload.SliceMD5, payload.CreateTime, payload.SliceMD5s, payload.Cloud)
if err != nil {
return nil, err
}
@@ -953,7 +974,7 @@ func DecodeCAS(data []byte, limits ParseLimits) (*Seed, error) {
// casEntryToSeedFile converts a CAS payload entry into a SeedFile, restoring the
// per-piece MD5 list when it is present.
func casEntryToSeedFile(name string, size int64, md5Hex, sliceMD5Hex, createTime string, sliceMD5s []string) (SeedFile, error) {
func casEntryToSeedFile(name string, size int64, md5Hex, sliceMD5Hex, createTime string, sliceMD5s []string, cloud string) (SeedFile, error) {
if name == "" || size < 0 || !validHexHash(md5Hex, 32) {
return SeedFile{}, fmt.Errorf("invalid CAS payload")
}
@@ -967,6 +988,7 @@ func casEntryToSeedFile(name string, size int64, md5Hex, sliceMD5Hex, createTime
file := SeedFile{
Path: name, Size: size, CASCreateTime: createTime,
CASSliceMD5: strings.ToLower(sliceMD5),
CASCloud: cloud,
Hashes: SeedHashes{MD5: strings.ToLower(md5Hex)},
}
if len(sliceMD5s) > 0 {
@@ -1111,12 +1133,20 @@ func TorrentFromSeed(seed *Seed) (*Torrent, []string) {
if len(seed.Files) == 1 {
file := seed.Files[0]
if file.Hashes.MD5 != "" && file.CASSliceMD5 != "" {
cloud := file.CASCloud
if cloud == "" {
cloud = Cloud189
}
t.CAS = &CASInfo{
FileMD5: strings.ToUpper(file.Hashes.MD5), SliceMD5: strings.ToUpper(file.CASSliceMD5),
SliceSize: DefaultPieceSize, Cloud: "189",
SliceSize: DefaultPieceSize, Cloud: cloud,
}
} else if file.Hashes.MD5 != "" && seed.PieceSize == DefaultPieceSize && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 {
t.CAS = BuildCASInfoFromMD5s(file.Hashes.MD5, upperStrings(file.Hashes.Pieces.MD5), DefaultPieceSize)
cloud := file.CASCloud
if cloud == "" {
cloud = Cloud189
}
t.CAS = BuildCASInfoFromMD5sWithCloud(file.Hashes.MD5, upperStrings(file.Hashes.Pieces.MD5), DefaultPieceSize, cloud)
}
}
return t, nil
@@ -1267,6 +1297,9 @@ func seedToBencode(seed *Seed) map[string]interface{} {
if file.CASCreateTime != "" {
item["cas_create_time"] = file.CASCreateTime
}
if file.CASCloud != "" {
item["cas_cloud"] = file.CASCloud
}
if len(file.MissingChannels) > 0 {
item["missing_channels"] = stringsToInterfaces(file.MissingChannels)
}
@@ -1345,6 +1378,7 @@ func seedFromBencode(value interface{}) (*Seed, error) {
file := SeedFile{
Path: bString(item["path"]), Size: bInt(item["size"]), Modified: bString(item["modified"]), Comment: bString(item["comment"]),
CASSliceMD5: bString(item["cas_slice_md5"]), CASCreateTime: bString(item["cas_create_time"]),
CASCloud: bString(item["cas_cloud"]),
MissingChannels: bStrings(item["missing_channels"]),
}
if hashes, ok := item["hashes"].(map[string]interface{}); ok {
server/handles/torrent.go
+170 -19
View File
@@ -7,12 +7,12 @@ import (
"encoding/json"
"fmt"
"io"
"net/http"
"net/url"
stdpath "path"
"strings"
"time"
_189pc "github.com/OpenListTeam/OpenList/v4/drivers/189pc"
"github.com/OpenListTeam/OpenList/v4/internal/conf"
"github.com/OpenListTeam/OpenList/v4/internal/driver"
"github.com/OpenListTeam/OpenList/v4/internal/errs"
@@ -25,11 +25,106 @@ import (
"github.com/OpenListTeam/OpenList/v4/pkg/http_range"
"github.com/OpenListTeam/OpenList/v4/pkg/torrent"
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
hash_extend "github.com/OpenListTeam/OpenList/v4/pkg/utils/hash"
"github.com/OpenListTeam/OpenList/v4/server/common"
"github.com/gin-gonic/gin"
"github.com/pkg/errors"
)
// seedRapidUploaderFor probes a storage driver for the generic
// driver.SeedRapidUploader capability. It returns nil when the driver does not
// support hash-driven rapid upload (秒传/CAS).
func seedRapidUploaderFor(storage driver.Driver) driver.SeedRapidUploader {
if storage == nil {
return nil
}
if u, ok := storage.(driver.SeedRapidUploader); ok {
return u
}
return nil
}
// buildSeedRapidUploadRequest converts a parsed torrent's CAS metadata into a
// driver.SeedRapidUploadRequest. open lazily provides the file content and may
// be nil when no content source is available.
func buildSeedRapidUploadRequest(t *torrent.Torrent, open func() (model.FileStreamer, error)) *driver.SeedRapidUploadRequest {
if t == nil {
return nil
}
wholeHashes := make(map[*utils.HashType]string)
req := &driver.SeedRapidUploadRequest{
Name: t.Info.Name,
Size: t.GetTotalSize(),
Open: open,
}
if t.CAS != nil {
req.SliceSize = t.CAS.SliceSize
req.SliceMD5s = t.CAS.SliceMD5s
if t.CAS.FileMD5 != "" {
wholeHashes[utils.MD5] = strings.ToUpper(t.CAS.FileMD5)
}
}
// 从种子文件的哈希矩阵补充整文件与分片哈希
if t.OpenList != nil {
if req.SliceSize == 0 {
req.SliceSize = t.OpenList.PieceSize
}
if len(t.OpenList.Files) > 0 {
sf := t.OpenList.Files[0]
if sf.Hashes.MD5 != "" {
wholeHashes[utils.MD5] = strings.ToUpper(sf.Hashes.MD5)
}
if sf.Hashes.SHA1 != "" {
wholeHashes[utils.SHA1] = strings.ToUpper(sf.Hashes.SHA1)
}
if sf.Hashes.SHA256 != "" {
wholeHashes[utils.SHA256] = strings.ToUpper(sf.Hashes.SHA256)
}
if sf.Hashes.GCID != "" {
wholeHashes[hash_extend.GCID] = strings.ToUpper(sf.Hashes.GCID)
}
if sf.Hashes.Pieces != nil {
if len(req.SliceMD5s) == 0 && len(sf.Hashes.Pieces.MD5) > 0 {
req.SliceMD5s = sf.Hashes.Pieces.MD5
}
if len(sf.Hashes.Pieces.SHA1) > 0 {
req.SliceSHA1s = sf.Hashes.Pieces.SHA1
}
}
}
}
req.Whole = &utils.HashInfo{}
*req.Whole = utils.NewHashInfoByMap(wholeHashes)
return req
}
// seedFileSupportsRapidUpload reports whether a seed file carries enough hash
// metadata for the given rapid-upload driver to attempt an instant upload.
func seedFileSupportsRapidUpload(uploader driver.SeedRapidUploader, file torrent.SeedFile) bool {
if uploader == nil {
return false
}
hasMD5 := file.Hashes.MD5 != ""
hasSHA1 := file.Hashes.SHA1 != ""
for _, ht := range uploader.RapidHashAlgos() {
switch ht.Name {
case utils.MD5.Name:
if hasMD5 {
return true
}
case utils.SHA1.Name:
if hasSHA1 {
return true
}
}
}
return false
}
// maxTorrentBase64Len is the max allowed Base64-encoded torrent size (~10MB decoded)
const maxTorrentBase64Len = 14 * 1024 * 1024
@@ -230,15 +325,16 @@ func TorrentRapidUpload(c *gin.Context) {
return
}
// 检查是否是天翼云 PC 驱动
cloud189PC, ok := storage.(*_189pc.Cloud189PC)
// 探测目标驱动是否支持通用 CAS 秒传
rapid, ok := storage.(driver.SeedRapidUploader)
if !ok {
common.ErrorResp(c, fmt.Errorf("目标存储不是天翼云PC驱动，不支持 CAS 秒传"), 400)
common.ErrorResp(c, fmt.Errorf("目标存储驱动 %s 不支持哈希秒传（CAS）", storage.GetStorage().Driver), 400)
return
}
// 尝试秒传
obj, err := cloud189PC.RapidUploadFromTorrent(c.Request.Context(), dstDir, torrentData, true)
rapidReq := buildSeedRapidUploadRequest(t, nil)
obj, err := rapid.RapidUploadByHashes(c.Request.Context(), dstDir, rapidReq, true)
if err != nil {
common.ErrorResp(c, fmt.Errorf("秒传失败: %w", err), 400)
return
@@ -374,10 +470,10 @@ func GenerateTorrentForPath(c *gin.Context) {
return
}
// with_cas 仅支持天翼云PC驱动
// with_cas 需要目标驱动支持通用哈希秒传（CAS）
if req.WithCAS {
if _, is189pc := storage.(*_189pc.Cloud189PC); !is189pc {
common.ErrorResp(c, fmt.Errorf("CAS 秒传扩展仅支持天翼云PC驱动"), 400)
if _, ok := storage.(driver.SeedRapidUploader); !ok {
common.ErrorResp(c, fmt.Errorf("存储驱动 %s 不支持 CAS 秒传扩展", storage.GetStorage().Driver), 400)
return
}
}
@@ -996,16 +1092,16 @@ func SeedCapabilities(c *gin.Context) {
common.ErrorResp(c, err, 400)
return
}
_, rapid189 := storage.(*_189pc.Cloud189PC)
rapidUploader := seedRapidUploaderFor(storage)
_, putURL := storage.(driver.PutURL)
_, putURLResult := storage.(driver.PutURLResult)
files := make([]gin.H, 0, len(seed.Files))
for _, file := range seed.Files {
hasCAS := rapid189 && file.Hashes.MD5 != "" && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0
hasCAS := seedFileSupportsRapidUpload(rapidUploader, file)
hasSource := firstUsableSeedSource(file) != ""
method := "download_required"
if hasCAS {
method = "189pc_cas"
method = "rapid_upload"
} else if hasSource && (putURL || putURLResult) {
method = "put_url"
} else if hasSource {
@@ -1024,12 +1120,20 @@ func SeedCapabilities(c *gin.Context) {
}
// Describe the destination driver's rapid-transfer capability surface so the
// frontend can show which hashes are reusable for instant upload.
rapidAlgos := []string{}
usesPieces := false
if rapidUploader != nil {
for _, ht := range rapidUploader.RapidHashAlgos() {
rapidAlgos = append(rapidAlgos, ht.Name)
}
usesPieces = rapidUploader.RapidHashNeedsPieces()
}
driverSupports := gin.H{
"cas_rapid": rapid189,
"cas_rapid": rapidUploader != nil,
"put_url": putURL || putURLResult,
"offline_download": true,
"rapid_hash_algos": []string{"md5", "sha1"},
"rapid_uses_pieces": rapid189,
"rapid_hash_algos": rapidAlgos,
"rapid_uses_pieces": usesPieces,
}
common.SuccessResp(c, gin.H{
"driver": storage.Config().Name, "global_policy": globalPolicy,
@@ -1317,7 +1421,7 @@ func saveSeedFilesToPath(c *gin.Context, user *model.User, seed *torrent.Seed, r
}
selectedIndexes[index] = struct{}{}
}
cloud189, is189 := storage.(*_189pc.Cloud189PC)
rapidUploader := seedRapidUploaderFor(storage)
_, putURL := storage.(driver.PutURL)
_, putURLResult := storage.(driver.PutURLResult)
for index, file := range seed.Files {
@@ -1331,14 +1435,17 @@ func saveSeedFilesToPath(c *gin.Context, user *model.User, seed *torrent.Seed, r
}
}
name := stdpath.Base(file.Path)
if is189 && file.Hashes.MD5 != "" && file.Hashes.Pieces != nil && len(file.Hashes.Pieces.MD5) > 0 && len(file.Hashes.Pieces.SHA1) > 0 {
if rapidUploader != nil && seedFileSupportsRapidUpload(rapidUploader, file) {
one := *seed
one.Name = name
one.Files = []torrent.SeedFile{file}
if data, encodeErr := torrent.EncodeSeed(&one, "torrent"); encodeErr == nil {
if obj, rapidErr := cloud189.RapidUploadFromTorrent(c.Request.Context(), dstDir, data, req.Overwrite); rapidErr == nil {
results = append(results, gin.H{"path": file.Path, "name": obj.GetName(), "method": "189pc_cas"})
continue
if t, decErr := torrent.Decode(data); decErr == nil {
rapidReq := buildSeedRapidUploadRequest(t, seedContentOpener(seed, file, c.Request.Context()))
if obj, rapidErr := rapidUploader.RapidUploadByHashes(c.Request.Context(), dstDir, rapidReq, req.Overwrite); rapidErr == nil {
results = append(results, gin.H{"path": file.Path, "name": obj.GetName(), "method": "rapid_upload"})
continue
}
}
}
}
@@ -1582,6 +1689,50 @@ func QuickSaveSeed(c *gin.Context) {
common.SuccessResp(c, resp)
}
// seedContentOpener returns a lazy content provider for a seed file. The source
// bytes are fetched from the file's first usable source URL on first call. It
// returns nil when no usable source exists, letting hash-only drivers (e.g.
// 189pc) skip content entirely.
func seedContentOpener(seed *torrent.Seed, file torrent.SeedFile, _ context.Context) func() (model.FileStreamer, error) {
source := firstUsableSeedSource(file)
if source == "" {
return nil
}
return func() (model.FileStreamer, error) {
req, err := http.NewRequest(http.MethodGet, source, nil)
if err != nil {
return nil, err
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
return nil, fmt.Errorf("下载种子内容失败: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, fmt.Errorf("下载种子内容失败: HTTP %d", resp.StatusCode)
}
data, err := io.ReadAll(io.LimitReader(resp.Body, maxTorrentGenFileSize+1))
if err != nil {
return nil, fmt.Errorf("读取种子内容失败: %w", err)
}
if file.Size > 0 && int64(len(data)) != file.Size {
return nil, fmt.Errorf("种子内容大小不匹配: 期望 %d，实际 %d", file.Size, len(data))
}
reader := bytes.NewReader(data)
obj := &model.Object{
Name: stdpath.Base(file.Path),
Size: int64(len(data)),
Modified: time.Now(),
IsFolder: false,
}
return &stream.FileStream{
Ctx: context.Background(),
Obj: obj,
Reader: reader,
}, nil
}
}
func firstUsableSeedSource(file torrent.SeedFile) string {
configuredSite, err := url.Parse(strings.TrimSpace(setting.GetStr(conf.SeedSiteURL)))
if err != nil || configuredSite.Scheme == "" || configuredSite.Hostname() == "" {