mirror of
https://github.com/OpenListTeam/OpenList.git
synced 2026-10-10 13:03:09 +08:00
Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| cc87e88f03 | |||
| 7e9082f90e | |||
| 6d4ad58678 | |||
| d8ad28f5b7 | |||
| 2e27852952 | |||
| 194f941f15 | |||
| ec1f968de1 | |||
| c97173286c | |||
| cca773f07b | |||
| a4ae2acb65 | |||
| f68f3e9f4a | |||
| 6f80df2827 | |||
| 940b69fe38 | |||
| f2f7ee4107 | |||
| cf931d4760 | |||
| be708292eb | |||
| 78f2f0b25d | |||
| 6b92cbd21f |
@@ -3,7 +3,7 @@ name: Issue or PR Auto Reply
|
||||
on:
|
||||
issues:
|
||||
types: [opened]
|
||||
pull_request:
|
||||
pull_request_target:
|
||||
types: [opened]
|
||||
|
||||
permissions:
|
||||
@@ -78,7 +78,7 @@ jobs:
|
||||
|
||||
pr-title-check:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event_name == 'pull_request'
|
||||
if: github.event_name == 'pull_request_target'
|
||||
steps:
|
||||
- name: Check PR title for required prefix and comment
|
||||
uses: actions/github-script@v9
|
||||
|
||||
+4
-2
@@ -22,6 +22,7 @@ output/
|
||||
# vendor/
|
||||
/bin/*
|
||||
*.json
|
||||
!renovate.json
|
||||
/build
|
||||
/data/
|
||||
/tmp/
|
||||
@@ -29,6 +30,7 @@ output/
|
||||
/lang/
|
||||
/daemon/
|
||||
/public/dist/*
|
||||
/!public/dist/README.md
|
||||
!/public/dist/README.md
|
||||
|
||||
.VSCodeCounter
|
||||
.VSCodeCounter
|
||||
*.syso
|
||||
+1
-1
@@ -23,7 +23,7 @@ type options struct {
|
||||
|
||||
pwd string //de/encrypt password
|
||||
salt string
|
||||
filenameEncryption string //reference drivers\crypt\meta.go Addtion
|
||||
filenameEncryption string //reference drivers\crypt\meta.go Addition
|
||||
dirnameEncryption string
|
||||
filenameEncode string
|
||||
suffix string
|
||||
|
||||
+1
-1
@@ -507,7 +507,7 @@ func (d *Yun139) shareHeaders() map[string]string {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:140.0) Gecko/20100101 Firefox/140.0",
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"Content-Type": "application/json;charset=UTF-8",
|
||||
"X-Deviceinfo": "||9|12.27.0|firefox|140.0|||linux unknow|1920X526|zh-CN|||",
|
||||
"X-Deviceinfo": "||9|12.27.0|firefox|140.0|||linux unknown|1920X526|zh-CN|||",
|
||||
"hcy-cool-flag": "1",
|
||||
"CMS-DEVICE": "default",
|
||||
"x-m4c-caller": "PC",
|
||||
|
||||
@@ -42,6 +42,7 @@ import (
|
||||
_ "github.com/OpenListTeam/OpenList/v4/drivers/github_releases"
|
||||
_ "github.com/OpenListTeam/OpenList/v4/drivers/google_drive"
|
||||
_ "github.com/OpenListTeam/OpenList/v4/drivers/google_photo"
|
||||
_ "github.com/OpenListTeam/OpenList/v4/drivers/guangyapan"
|
||||
_ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud"
|
||||
_ "github.com/OpenListTeam/OpenList/v4/drivers/halalcloud_open"
|
||||
_ "github.com/OpenListTeam/OpenList/v4/drivers/ilanzou"
|
||||
|
||||
@@ -251,7 +251,7 @@ func (d *Crypt) Link(ctx context.Context, file model.Obj, _ model.LinkArgs) (*mo
|
||||
rrf, err := stream.GetRangeReaderFromLink(remoteSize, remoteLink)
|
||||
if err != nil {
|
||||
_ = remoteLink.Close()
|
||||
return nil, fmt.Errorf("the remote storage driver need to be enhanced to support encrytion")
|
||||
return nil, fmt.Errorf("the remote storage driver need to be enhanced to support encryption")
|
||||
}
|
||||
|
||||
mu := &sync.Mutex{}
|
||||
|
||||
@@ -0,0 +1,786 @@
|
||||
package guangyapan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/base"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"github.com/go-resty/resty/v2"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
accountBaseURL = "https://account.guangyapan.com"
|
||||
apiBaseURL = "https://api.guangyapan.com"
|
||||
)
|
||||
|
||||
type GuangYaPan struct {
|
||||
model.Storage
|
||||
Addition
|
||||
|
||||
accountClient *resty.Client
|
||||
apiClient *resty.Client
|
||||
|
||||
resolvedRootFolderID string
|
||||
rootFolderResolved bool
|
||||
|
||||
// refreshMu protects concurrent access to AccessToken/RefreshToken during refresh.
|
||||
refreshMu sync.Mutex
|
||||
// statusTimer tracks delayed status updates for cancellation on Drop.
|
||||
statusTimer *time.Timer
|
||||
|
||||
// apiRateLimit throttles requests per API endpoint so that batch operations
|
||||
// (e.g. copying many files cross-storage) don't flood the upstream API.
|
||||
apiRateLimit sync.Map
|
||||
}
|
||||
|
||||
// apiRateInterval is the minimum gap between two requests to the same endpoint.
|
||||
const apiRateInterval = 500 * time.Millisecond
|
||||
|
||||
func (d *GuangYaPan) Config() driver.Config {
|
||||
return config
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) GetAddition() driver.Additional {
|
||||
return &d.Addition
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) Init(ctx context.Context) error {
|
||||
d.ClientID = strings.TrimSpace(d.ClientID)
|
||||
if d.ClientID == "" {
|
||||
return errors.New("client_id is required, please provide a valid client_id")
|
||||
}
|
||||
d.DeviceID = normalizeDeviceID(d.DeviceID)
|
||||
if d.DeviceID == "" {
|
||||
d.DeviceID = randomDeviceID()
|
||||
}
|
||||
deviceSign := strings.TrimSpace(d.DeviceSign)
|
||||
if deviceSign == "" {
|
||||
deviceSign = "wdi10." + d.DeviceID
|
||||
}
|
||||
if d.PageSize <= 0 {
|
||||
d.PageSize = 100
|
||||
}
|
||||
if d.OrderBy < 0 {
|
||||
d.OrderBy = 3
|
||||
}
|
||||
if d.SortType != 0 && d.SortType != 1 {
|
||||
d.SortType = 1
|
||||
}
|
||||
|
||||
d.RootPath = strings.TrimSpace(d.RootPath)
|
||||
d.AccessToken = strings.TrimSpace(d.AccessToken)
|
||||
d.RefreshToken = strings.TrimSpace(d.RefreshToken)
|
||||
d.PhoneNumber = strings.TrimSpace(d.PhoneNumber)
|
||||
d.VerifyCode = strings.TrimSpace(d.VerifyCode)
|
||||
d.CaptchaToken = strings.TrimSpace(d.CaptchaToken)
|
||||
d.VerificationID = strings.TrimSpace(d.VerificationID)
|
||||
d.resolvedRootFolderID = ""
|
||||
d.rootFolderResolved = false
|
||||
|
||||
d.accountClient = base.NewRestyClient().
|
||||
SetBaseURL(accountBaseURL).
|
||||
SetHeader("Accept", "application/json, text/plain, */*").
|
||||
SetHeader("Content-Type", "application/json").
|
||||
SetHeader("X-Device-Model", "chrome%2F147.0.0.0").
|
||||
SetHeader("X-Device-Name", "PC-Chrome").
|
||||
SetHeader("X-Device-Sign", deviceSign).
|
||||
SetHeader("X-Net-Work-Type", "NONE").
|
||||
SetHeader("X-OS-Version", "MacIntel").
|
||||
SetHeader("X-Platform-Version", "1").
|
||||
SetHeader("X-Protocol-Version", "301").
|
||||
SetHeader("X-Provider-Name", "NONE").
|
||||
SetHeader("X-SDK-Version", "9.0.2").
|
||||
SetHeader("X-Client-Id", d.ClientID).
|
||||
SetHeader("X-Client-Version", "0.0.1").
|
||||
SetHeader("X-Device-Id", d.DeviceID)
|
||||
|
||||
d.apiClient = base.NewRestyClient().
|
||||
SetBaseURL(apiBaseURL).
|
||||
SetHeader("Accept", "application/json, text/plain, */*").
|
||||
SetHeader("Content-Type", "application/json").
|
||||
SetHeader("Did", d.DeviceID).
|
||||
SetHeader("Dt", "4")
|
||||
|
||||
// Priority: access_token -> refresh_token -> sms login.
|
||||
if d.AccessToken != "" {
|
||||
if err := d.validateToken(ctx); err == nil {
|
||||
return d.prepareRootFolder(ctx)
|
||||
}
|
||||
d.AccessToken = ""
|
||||
}
|
||||
if d.RefreshToken != "" {
|
||||
if err := d.refreshToken(ctx); err == nil {
|
||||
if err2 := d.validateToken(ctx); err2 == nil {
|
||||
return d.prepareRootFolder(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Two-stage SMS flow:
|
||||
// 1) phone only + send_code=true: send code and cache verification_id (do not fail init).
|
||||
// 2) phone + verify_code: complete login and save tokens.
|
||||
if d.PhoneNumber != "" {
|
||||
if d.canSMSLogin() {
|
||||
if err := d.loginBySMSCode(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.validateToken(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return d.prepareRootFolder(ctx)
|
||||
}
|
||||
if d.SendCode {
|
||||
d.setTempStatus("SMS sending in progress...")
|
||||
if err := d.prepareSMSCode(ctx); err != nil {
|
||||
d.setTempStatus(fmt.Sprintf("SMS send failed: %v. Please check captcha/meta and set send_code=true to retry.", err))
|
||||
log.Warnf("guangyapan: prepare sms code failed: %v", err)
|
||||
} else {
|
||||
d.setTempStatus("SMS sent successfully. Please fill verify_code and save to complete login.")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return errors.New("login failed: provide a valid access_token, or refresh_token, or phone_number + verify_code + captcha_token")
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) Drop(ctx context.Context) error {
|
||||
if d.statusTimer != nil {
|
||||
d.statusTimer.Stop()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) GetRoot(ctx context.Context) (model.Obj, error) {
|
||||
rootID, err := d.getRootFolderID(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &model.Object{
|
||||
ID: rootID,
|
||||
Path: "/",
|
||||
Name: "root",
|
||||
Size: 0,
|
||||
Modified: d.Modified,
|
||||
IsFolder: true,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) {
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parentID := dir.GetID()
|
||||
|
||||
const maxPage = 10000
|
||||
res := make([]model.Obj, 0, d.PageSize)
|
||||
for page := 0; page < maxPage; page++ {
|
||||
var resp listResp
|
||||
body := map[string]any{
|
||||
"parentId": parentID,
|
||||
"page": page,
|
||||
"pageSize": d.PageSize,
|
||||
"orderBy": d.OrderBy,
|
||||
"sortType": d.SortType,
|
||||
}
|
||||
if err := d.postAPI(ctx, "/userres/v1/file/get_file_list", body, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, item := range resp.Data.List {
|
||||
res = append(res, &model.Object{
|
||||
ID: item.FileID,
|
||||
Path: parentID,
|
||||
Name: item.FileName,
|
||||
Size: item.FileSize,
|
||||
Modified: unixOrZero(item.UTime),
|
||||
Ctime: unixOrZero(item.CTime),
|
||||
IsFolder: item.ResType == 2,
|
||||
})
|
||||
}
|
||||
if len(resp.Data.List) < d.PageSize {
|
||||
break
|
||||
}
|
||||
if resp.Data.Total > 0 && len(res) >= resp.Data.Total {
|
||||
break
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) {
|
||||
if file.IsDir() {
|
||||
return nil, errs.NotFile
|
||||
}
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var resp downloadResp
|
||||
if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_res_download_url", map[string]any{
|
||||
"fileId": file.GetID(),
|
||||
}, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
url := strings.TrimSpace(resp.Data.SignedURL)
|
||||
if url == "" {
|
||||
url = strings.TrimSpace(resp.Data.DownloadURL)
|
||||
}
|
||||
if url == "" {
|
||||
return nil, errors.New("empty download url")
|
||||
}
|
||||
return &model.Link{URL: url}, nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error {
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
name := strings.TrimSpace(dirName)
|
||||
if name == "" {
|
||||
return errors.New("dir name is empty")
|
||||
}
|
||||
|
||||
parentID := parentDir.GetID()
|
||||
|
||||
var out createDirResp
|
||||
if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/create_dir", map[string]any{
|
||||
"parentId": parentID,
|
||||
"dirName": name,
|
||||
}, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if !isSuccessMsg(out.Msg) {
|
||||
return fmt.Errorf("make dir failed: %s", strings.TrimSpace(out.Msg))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) Rename(ctx context.Context, srcObj model.Obj, newName string) error {
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fileID := strings.TrimSpace(srcObj.GetID())
|
||||
if fileID == "" {
|
||||
return errors.New("file id is empty")
|
||||
}
|
||||
name := strings.TrimSpace(newName)
|
||||
if name == "" {
|
||||
return errors.New("new name is empty")
|
||||
}
|
||||
|
||||
var out commonResp
|
||||
if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/rename", map[string]any{
|
||||
"fileId": fileID,
|
||||
"newName": name,
|
||||
}, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if !isSuccessMsg(out.Msg) {
|
||||
return fmt.Errorf("rename failed: %s", strings.TrimSpace(out.Msg))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) Remove(ctx context.Context, obj model.Obj) error {
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fileID := strings.TrimSpace(obj.GetID())
|
||||
if fileID == "" {
|
||||
return errors.New("file id is empty")
|
||||
}
|
||||
|
||||
var del taskResp
|
||||
if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/delete_file", map[string]any{
|
||||
"fileIds": []string{fileID},
|
||||
}, &del); err != nil {
|
||||
return err
|
||||
}
|
||||
if !isSuccessMsg(del.Msg) {
|
||||
return fmt.Errorf("delete failed: %s", strings.TrimSpace(del.Msg))
|
||||
}
|
||||
|
||||
taskID := strings.TrimSpace(del.Data.TaskID)
|
||||
if taskID == "" {
|
||||
// Some backends may apply deletion synchronously.
|
||||
return nil
|
||||
}
|
||||
return d.waitTaskDone(ctx, taskID)
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) Move(ctx context.Context, srcObj, dstDir model.Obj) error {
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fileID := strings.TrimSpace(srcObj.GetID())
|
||||
if fileID == "" {
|
||||
return errors.New("file id is empty")
|
||||
}
|
||||
parentID := dstDir.GetID()
|
||||
|
||||
var out taskResp
|
||||
if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/move_file", map[string]any{
|
||||
"fileIds": []string{fileID},
|
||||
"parentId": parentID,
|
||||
}, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if !isSuccessMsg(out.Msg) {
|
||||
return fmt.Errorf("move failed: %s", strings.TrimSpace(out.Msg))
|
||||
}
|
||||
taskID := strings.TrimSpace(out.Data.TaskID)
|
||||
if taskID == "" {
|
||||
return nil
|
||||
}
|
||||
return d.waitTaskDone(ctx, taskID)
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) Copy(ctx context.Context, srcObj, dstDir model.Obj) error {
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fileID := strings.TrimSpace(srcObj.GetID())
|
||||
if fileID == "" {
|
||||
return errors.New("file id is empty")
|
||||
}
|
||||
parentID := dstDir.GetID()
|
||||
|
||||
var out taskResp
|
||||
if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/copy_file", map[string]any{
|
||||
"fileIds": []string{fileID},
|
||||
"parentId": parentID,
|
||||
}, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if !isSuccessMsg(out.Msg) {
|
||||
return fmt.Errorf("copy failed: %s", strings.TrimSpace(out.Msg))
|
||||
}
|
||||
taskID := strings.TrimSpace(out.Data.TaskID)
|
||||
if taskID == "" {
|
||||
return nil
|
||||
}
|
||||
return d.waitTaskDone(ctx, taskID)
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) {
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if file == nil {
|
||||
return nil, errors.New("file is nil")
|
||||
}
|
||||
if file.GetSize() < 0 {
|
||||
return nil, errors.New("invalid file size")
|
||||
}
|
||||
name := strings.TrimSpace(file.GetName())
|
||||
if name == "" {
|
||||
return nil, errors.New("file name is empty")
|
||||
}
|
||||
|
||||
parentID := dstDir.GetID()
|
||||
|
||||
token, code, err := d.getUploadToken(ctx, parentID, name, file.GetSize())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
taskID := strings.TrimSpace(token.TaskID)
|
||||
// code == 156 (instant upload) or AlreadyDone mean the backend has already
|
||||
// finished/imported the file; there is no OSS upload to perform.
|
||||
if code == 156 || token.AlreadyDone {
|
||||
if taskID == "" {
|
||||
return nil, errors.New("instant upload returns empty task id")
|
||||
}
|
||||
if err := d.waitUploadTaskInfo(ctx, taskID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if token.ObjectPath == "" || token.BucketName == "" || token.EndPoint == "" || token.AccessKeyID == "" || token.SecretAccessKey == "" {
|
||||
return nil, errors.New("upload token is incomplete")
|
||||
}
|
||||
|
||||
ossEndpoint := normalizeOSSEndpoint(token.EndPoint, token.BucketName)
|
||||
client, err := oss.New(ossEndpoint, token.AccessKeyID, token.SecretAccessKey, oss.SecurityToken(token.SessionToken))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create oss client failed: %w", err)
|
||||
}
|
||||
bucket, err := client.Bucket(token.BucketName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create oss bucket failed: %w", err)
|
||||
}
|
||||
|
||||
if file.GetSize() == 0 {
|
||||
if err := bucket.PutObject(token.ObjectPath, strings.NewReader("")); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if err := d.multipartUploadToOSS(ctx, bucket, token.ObjectPath, file, up); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if taskID == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if err := d.waitUploadTaskInfo(ctx, taskID); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) GetDetails(ctx context.Context) (*model.StorageDetails, error) {
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var resp assetsInfoResp
|
||||
if err := d.postAPI(ctx, "/nd.bizassets.s/v1/get_assets", nil, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.IsSuccess() && resp.Data.TotalSpaceSize > 0 {
|
||||
return &model.StorageDetails{
|
||||
DiskUsage: model.DiskUsage{
|
||||
TotalSpace: resp.Data.TotalSpaceSize,
|
||||
UsedSpace: resp.Data.UsedSpaceSize,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
return nil, errors.New("failed to get storage details")
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) getRootFolderID(ctx context.Context) (string, error) {
|
||||
if d.rootFolderResolved {
|
||||
return d.resolvedRootFolderID, nil
|
||||
}
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := d.prepareRootFolder(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return d.resolvedRootFolderID, nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) prepareRootFolder(ctx context.Context) error {
|
||||
rootID, err := d.resolveConfiguredRootFolderID(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.resolvedRootFolderID = rootID
|
||||
d.rootFolderResolved = true
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) resolveConfiguredRootFolderID(ctx context.Context) (string, error) {
|
||||
root := strings.TrimSpace(d.RootPath)
|
||||
if root == "" {
|
||||
return "", nil
|
||||
}
|
||||
return d.resolveFolderPath(ctx, root)
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) resolveFolderPath(ctx context.Context, rootPath string) (string, error) {
|
||||
cleanPath := strings.Trim(strings.ReplaceAll(strings.TrimSpace(rootPath), "\\", "/"), "/")
|
||||
if cleanPath == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
parentID := ""
|
||||
for _, name := range strings.Split(cleanPath, "/") {
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
childID, err := d.findChildFolderID(ctx, parentID, name)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parentID = childID
|
||||
}
|
||||
return parentID, nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) findChildFolderID(ctx context.Context, parentID, name string) (string, error) {
|
||||
pageSize := d.PageSize
|
||||
if pageSize <= 0 {
|
||||
pageSize = 100
|
||||
}
|
||||
|
||||
const maxPage = 10000
|
||||
seen := 0
|
||||
for page := 0; page < maxPage; page++ {
|
||||
var resp listResp
|
||||
body := map[string]any{
|
||||
"parentId": parentID,
|
||||
"page": page,
|
||||
"pageSize": pageSize,
|
||||
"orderBy": d.OrderBy,
|
||||
"sortType": d.SortType,
|
||||
}
|
||||
if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_file_list", body, &resp); err != nil {
|
||||
return "", err
|
||||
}
|
||||
for _, item := range resp.Data.List {
|
||||
seen++
|
||||
if item.ResType == 2 && item.FileName == name {
|
||||
return item.FileID, nil
|
||||
}
|
||||
}
|
||||
if len(resp.Data.List) < pageSize {
|
||||
break
|
||||
}
|
||||
if resp.Data.Total > 0 && seen >= resp.Data.Total {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if parentID == "" {
|
||||
return "", fmt.Errorf("resolve root folder path failed: folder %q not found under /", name)
|
||||
}
|
||||
return "", fmt.Errorf("resolve root folder path failed: folder %q not found under parent %s", name, parentID)
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) ensureAccessToken(ctx context.Context) error {
|
||||
if strings.TrimSpace(d.AccessToken) != "" {
|
||||
return nil
|
||||
}
|
||||
if strings.TrimSpace(d.RefreshToken) == "" {
|
||||
return errors.New("not logged in, please re-init storage")
|
||||
}
|
||||
return d.refreshToken(ctx)
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) validateToken(ctx context.Context) error {
|
||||
var me userMeResp
|
||||
resp, err := d.accountClient.R().
|
||||
SetContext(ctx).
|
||||
SetHeader("Authorization", "Bearer "+d.AccessToken).
|
||||
SetResult(&me).
|
||||
Get("/v1/user/me")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.IsError() {
|
||||
return fmt.Errorf("validate token failed: status=%d body=%s", resp.StatusCode(), resp.String())
|
||||
}
|
||||
if strings.TrimSpace(me.Sub) == "" {
|
||||
return errors.New("validate token failed: empty user sub")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) refreshToken(ctx context.Context) error {
|
||||
if strings.TrimSpace(d.RefreshToken) == "" {
|
||||
return errors.New("refresh_token is empty")
|
||||
}
|
||||
|
||||
d.refreshMu.Lock()
|
||||
defer d.refreshMu.Unlock()
|
||||
|
||||
// Double-check after acquiring lock (may have been refreshed by another goroutine)
|
||||
if strings.TrimSpace(d.AccessToken) != "" {
|
||||
if err := d.validateToken(ctx); err == nil {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
var out tokenResp
|
||||
resp, err := d.accountClient.R().
|
||||
SetContext(ctx).
|
||||
SetBody(map[string]any{
|
||||
"client_id": d.ClientID,
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": d.RefreshToken,
|
||||
}).
|
||||
SetResult(&out).
|
||||
Post("/v1/auth/token")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.IsError() || out.Error != "" || strings.TrimSpace(out.AccessToken) == "" {
|
||||
errMsg := strings.TrimSpace(out.ErrorDesc)
|
||||
if errMsg == "" {
|
||||
errMsg = strings.TrimSpace(out.Error)
|
||||
}
|
||||
if errMsg == "" {
|
||||
errMsg = strings.TrimSpace(resp.String())
|
||||
}
|
||||
if errMsg == "" {
|
||||
errMsg = fmt.Sprintf("status=%d", resp.StatusCode())
|
||||
}
|
||||
return fmt.Errorf("refresh token failed: %s", errMsg)
|
||||
}
|
||||
|
||||
d.AccessToken = strings.TrimSpace(out.AccessToken)
|
||||
if strings.TrimSpace(out.RefreshToken) != "" {
|
||||
d.RefreshToken = strings.TrimSpace(out.RefreshToken)
|
||||
}
|
||||
op.MustSaveDriverStorage(d)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) canSMSLogin() bool {
|
||||
return d.PhoneNumber != "" && d.VerifyCode != ""
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) loginBySMSCode(ctx context.Context) error {
|
||||
verificationID := strings.TrimSpace(d.VerificationID)
|
||||
if verificationID == "" {
|
||||
var err error
|
||||
verificationID, err = d.requestVerificationID(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
var step2 verifyResp
|
||||
resp, err := d.accountClient.R().
|
||||
SetContext(ctx).
|
||||
SetBody(map[string]any{
|
||||
"verification_id": verificationID,
|
||||
"verification_code": d.VerifyCode,
|
||||
"client_id": d.ClientID,
|
||||
}).
|
||||
SetResult(&step2).
|
||||
Post("/v1/auth/verification/verify")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.IsError() || step2.Error != "" || strings.TrimSpace(step2.VerificationToken) == "" {
|
||||
return fmt.Errorf("verify code failed: %s", d.accountErr(step2.ErrorDesc, step2.Error, resp))
|
||||
}
|
||||
|
||||
var out tokenResp
|
||||
resp, err = d.accountClient.R().
|
||||
SetContext(ctx).
|
||||
SetBody(map[string]any{
|
||||
"verification_code": d.VerifyCode,
|
||||
"verification_token": step2.VerificationToken,
|
||||
"username": normalizePhoneE164(d.PhoneNumber),
|
||||
"client_id": d.ClientID,
|
||||
}).
|
||||
SetResult(&out).
|
||||
Post("/v1/auth/signin")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.IsError() || out.Error != "" || strings.TrimSpace(out.AccessToken) == "" {
|
||||
return fmt.Errorf("signin failed: %s", d.accountErr(out.ErrorDesc, out.Error, resp))
|
||||
}
|
||||
|
||||
d.AccessToken = strings.TrimSpace(out.AccessToken)
|
||||
d.RefreshToken = strings.TrimSpace(out.RefreshToken)
|
||||
d.VerificationID = ""
|
||||
// One-time SMS code should not be reused after successful login.
|
||||
d.VerifyCode = ""
|
||||
op.MustSaveDriverStorage(d)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) prepareSMSCode(ctx context.Context) error {
|
||||
// Explicit send action should always refresh verification_id.
|
||||
d.VerificationID = ""
|
||||
if err := d.ensureCaptchaToken(ctx, false); err != nil {
|
||||
return err
|
||||
}
|
||||
verificationID, err := d.requestVerificationID(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
d.VerificationID = verificationID
|
||||
d.SendCode = false
|
||||
op.MustSaveDriverStorage(d)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) requestVerificationID(ctx context.Context) (string, error) {
|
||||
req := d.accountClient.R().SetContext(ctx)
|
||||
if d.CaptchaToken != "" {
|
||||
req.SetHeader("X-Captcha-Token", d.CaptchaToken)
|
||||
}
|
||||
|
||||
var step1 verificationResp
|
||||
resp, err := req.
|
||||
SetBody(map[string]any{
|
||||
"phone_number": normalizePhoneE164(d.PhoneNumber),
|
||||
"target": "ANY",
|
||||
"client_id": d.ClientID,
|
||||
}).
|
||||
SetResult(&step1).
|
||||
Post("/v1/auth/verification")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if resp.IsError() || step1.Error != "" || strings.TrimSpace(step1.VerificationID) == "" {
|
||||
// If captcha token is expired/invalid, refresh it once and retry.
|
||||
if strings.Contains(step1.Error, "captcha_invalid") || strings.Contains(step1.ErrorDesc, "captcha_token expired") {
|
||||
if err := d.ensureCaptchaToken(ctx, true); err == nil {
|
||||
return d.requestVerificationID(ctx)
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("request verification failed: %s", d.accountErr(step1.ErrorDesc, step1.Error, resp))
|
||||
}
|
||||
return strings.TrimSpace(step1.VerificationID), nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) ensureCaptchaToken(ctx context.Context, force bool) error {
|
||||
if !force && d.CaptchaToken != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
var out captchaInitResp
|
||||
req := d.accountClient.R().SetContext(ctx)
|
||||
if d.CaptchaToken != "" {
|
||||
req.SetHeader("X-Captcha-Token", d.CaptchaToken)
|
||||
}
|
||||
resp, err := req.
|
||||
SetBody(map[string]any{
|
||||
"client_id": d.ClientID,
|
||||
"action": "POST:/v1/auth/verification",
|
||||
"device_id": d.DeviceID,
|
||||
"meta": map[string]any{
|
||||
"username": normalizePhoneE164(d.PhoneNumber),
|
||||
"phone_number": normalizePhoneE164(d.PhoneNumber),
|
||||
"VERIFICATION_PHONE": normalizePhoneE164(d.PhoneNumber),
|
||||
},
|
||||
}).
|
||||
SetResult(&out).
|
||||
Post("/v1/shield/captcha/init")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.IsError() || out.Error != "" || strings.TrimSpace(out.CaptchaToken) == "" {
|
||||
return fmt.Errorf("init captcha token failed: %s", d.accountErr(out.ErrorDesc, out.Error, resp))
|
||||
}
|
||||
d.CaptchaToken = strings.TrimSpace(out.CaptchaToken)
|
||||
op.MustSaveDriverStorage(d)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Interface compliance checks
|
||||
var (
|
||||
_ driver.Driver = (*GuangYaPan)(nil)
|
||||
_ driver.GetRooter = (*GuangYaPan)(nil)
|
||||
_ driver.Mkdir = (*GuangYaPan)(nil)
|
||||
_ driver.Move = (*GuangYaPan)(nil)
|
||||
_ driver.Copy = (*GuangYaPan)(nil)
|
||||
_ driver.Rename = (*GuangYaPan)(nil)
|
||||
_ driver.Remove = (*GuangYaPan)(nil)
|
||||
_ driver.PutResult = (*GuangYaPan)(nil)
|
||||
_ driver.WithDetails = (*GuangYaPan)(nil)
|
||||
)
|
||||
@@ -0,0 +1,37 @@
|
||||
package guangyapan
|
||||
|
||||
import (
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
)
|
||||
|
||||
type Addition struct {
|
||||
RootPath string `json:"root_path" help:"Full path in GuangYaPan cloud drive"`
|
||||
PhoneNumber string `json:"phone_number" type:"text" help:"Phone number for SMS login, e.g. +86 13800000000"`
|
||||
CaptchaToken string `json:"captcha_token" help:"Captcha token required by /v1/auth/verification"`
|
||||
SendCode bool `json:"send_code" type:"bool" help:"Set true and save to send SMS code, it auto-resets to false after sending"`
|
||||
VerifyCode string `json:"verify_code" type:"text" help:"SMS verification code used with phone_number; fill then save to finish login"`
|
||||
VerificationID string `json:"verification_id" type:"text" help:"Auto-generated after sending SMS code; do not edit manually"`
|
||||
AccessToken string `json:"access_token" help:"Bearer access token (optional if refresh_token is provided)"`
|
||||
RefreshToken string `json:"refresh_token" help:"Refresh token for auto-login/auto-refresh"`
|
||||
ClientID string `json:"client_id" required:"true" help:"Client ID for GuangYaPan API, must be provided"`
|
||||
DeviceID string `json:"device_id" help:"Optional custom device id (32 hex chars), auto-generated when empty"`
|
||||
DeviceSign string `json:"device_sign" help:"Optional custom X-Device-Sign header (generated from device_id when empty)"`
|
||||
PageSize int `json:"page_size" type:"number" default:"100"`
|
||||
OrderBy int `json:"order_by" type:"number" options:"0,1,2,3,4" default:"3" help:"Sort field used by the file list"`
|
||||
SortType int `json:"sort_type" type:"number" options:"0,1" default:"1" help:"Sort direction used by the file list"`
|
||||
}
|
||||
|
||||
var config = driver.Config{
|
||||
Name: "GuangYaPan",
|
||||
DefaultRoot: "",
|
||||
CheckStatus: true,
|
||||
Alert: "info|Two-stage SMS login: (1) fill phone_number (+ captcha_token if needed), set send_code=true and save; (2) fill verify_code and save to finish login and auto-save access_token/refresh_token.",
|
||||
NoOverwriteUpload: true,
|
||||
}
|
||||
|
||||
func init() {
|
||||
op.RegisterDriver(func() driver.Driver {
|
||||
return &GuangYaPan{}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package guangyapan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
stdpath "path"
|
||||
"strings"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
)
|
||||
|
||||
func (d *GuangYaPan) ResolveOfflineResource(ctx context.Context, fileURL string) (*OfflineResolveData, error) {
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fileURL = strings.TrimSpace(fileURL)
|
||||
if fileURL == "" {
|
||||
return nil, errors.New("offline url is empty")
|
||||
}
|
||||
|
||||
var resp offlineResolveResp
|
||||
if err := d.postAPI(ctx, "/cloudcollection/v1/resolve_res", map[string]any{
|
||||
"url": fileURL,
|
||||
}, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !isSuccessMsg(resp.Msg) {
|
||||
return nil, fmt.Errorf("resolve offline resource failed: %s", strings.TrimSpace(resp.Msg))
|
||||
}
|
||||
return &resp.Data, nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) OfflineDownload(ctx context.Context, fileURL string, parentDir model.Obj, fileName string) (*OfflineTask, error) {
|
||||
resolved, err := d.ResolveOfflineResource(ctx, fileURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
parentID := parentDir.GetID()
|
||||
rootID, err := d.getRootFolderID(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if parentID == rootID {
|
||||
parentID = ""
|
||||
}
|
||||
|
||||
taskURL := strings.TrimSpace(resolved.URL)
|
||||
if taskURL == "" {
|
||||
taskURL = strings.TrimSpace(fileURL)
|
||||
}
|
||||
name := strings.TrimSpace(fileName)
|
||||
if name == "" {
|
||||
name = resolved.defaultName(taskURL)
|
||||
}
|
||||
|
||||
body := map[string]any{
|
||||
"url": taskURL,
|
||||
"parentId": parentID,
|
||||
"newName": name,
|
||||
}
|
||||
if indexes := resolved.fileIndexes(); len(indexes) > 0 {
|
||||
body["fileIndexes"] = indexes
|
||||
}
|
||||
|
||||
var resp offlineCreateResp
|
||||
if err := d.postAPI(ctx, "/cloudcollection/v1/create_task", body, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !isSuccessMsg(resp.Msg) {
|
||||
return nil, fmt.Errorf("create offline task failed: %s", strings.TrimSpace(resp.Msg))
|
||||
}
|
||||
taskID := strings.TrimSpace(resp.Data.TaskID)
|
||||
if taskID == "" {
|
||||
return nil, errors.New("create offline task failed: empty task id")
|
||||
}
|
||||
return &OfflineTask{
|
||||
TaskID: taskID,
|
||||
FileName: name,
|
||||
Res: taskURL,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) OfflineList(ctx context.Context, taskIDs []string, statuses []int, cursor string, pageSize int) ([]OfflineTask, error) {
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body := map[string]any{}
|
||||
if len(taskIDs) > 0 {
|
||||
body["taskIds"] = taskIDs
|
||||
}
|
||||
if len(statuses) > 0 {
|
||||
body["status"] = statuses
|
||||
}
|
||||
if cursor = strings.TrimSpace(cursor); cursor != "" {
|
||||
body["cursor"] = cursor
|
||||
}
|
||||
if pageSize > 0 {
|
||||
body["pageSize"] = pageSize
|
||||
}
|
||||
|
||||
var resp offlineListResp
|
||||
if err := d.postAPI(ctx, "/cloudcollection/v1/list_task", body, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !isSuccessMsg(resp.Msg) {
|
||||
return nil, fmt.Errorf("list offline tasks failed: %s", strings.TrimSpace(resp.Msg))
|
||||
}
|
||||
return resp.Data.List, nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) DeleteOfflineTasks(ctx context.Context, taskIDs []string) error {
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(taskIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var resp offlineDeleteResp
|
||||
if err := d.postAPI(ctx, "/cloudcollection/v2/delete_task", map[string]any{
|
||||
"taskIds": taskIDs,
|
||||
}, &resp); err != nil {
|
||||
return err
|
||||
}
|
||||
if !isSuccessMsg(resp.Msg) {
|
||||
return fmt.Errorf("delete offline tasks failed: %s", strings.TrimSpace(resp.Msg))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d OfflineResolveData) defaultName(fileURL string) string {
|
||||
if d.BTResInfo != nil && strings.TrimSpace(d.BTResInfo.FileName) != "" {
|
||||
return strings.TrimSpace(d.BTResInfo.FileName)
|
||||
}
|
||||
u, err := url.Parse(fileURL)
|
||||
if err == nil {
|
||||
name := strings.TrimSpace(stdpath.Base(u.Path))
|
||||
if name != "" && name != "." && name != "/" {
|
||||
if decoded, err := url.PathUnescape(name); err == nil {
|
||||
name = decoded
|
||||
}
|
||||
return name
|
||||
}
|
||||
}
|
||||
return "offline_download"
|
||||
}
|
||||
|
||||
func (d OfflineResolveData) fileIndexes() []int {
|
||||
if d.BTResInfo == nil || len(d.BTResInfo.Subfiles) == 0 {
|
||||
return nil
|
||||
}
|
||||
indexes := make([]int, 0, len(d.BTResInfo.Subfiles))
|
||||
for i, file := range d.BTResInfo.Subfiles {
|
||||
if file.FileIndex != nil {
|
||||
indexes = append(indexes, *file.FileIndex)
|
||||
continue
|
||||
}
|
||||
indexes = append(indexes, i)
|
||||
}
|
||||
return indexes
|
||||
}
|
||||
|
||||
func isSuccessMsg(msg string) bool {
|
||||
msg = strings.TrimSpace(msg)
|
||||
return msg == "" || strings.EqualFold(msg, "success")
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
package guangyapan
|
||||
|
||||
import "time"
|
||||
|
||||
type tokenResp struct {
|
||||
AccessToken string `json:"access_token"`
|
||||
RefreshToken string `json:"refresh_token"`
|
||||
TokenType string `json:"token_type"`
|
||||
ExpiresIn int64 `json:"expires_in"`
|
||||
Sub string `json:"sub"`
|
||||
Error string `json:"error"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
ErrorDesc string `json:"error_description"`
|
||||
}
|
||||
|
||||
type verificationResp struct {
|
||||
VerificationID string `json:"verification_id"`
|
||||
Error string `json:"error"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
ErrorDesc string `json:"error_description"`
|
||||
}
|
||||
|
||||
type captchaInitResp struct {
|
||||
CaptchaToken string `json:"captcha_token"`
|
||||
ExpiresIn int64 `json:"expires_in"`
|
||||
Error string `json:"error"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
ErrorDesc string `json:"error_description"`
|
||||
}
|
||||
|
||||
type verifyResp struct {
|
||||
VerificationToken string `json:"verification_token"`
|
||||
Error string `json:"error"`
|
||||
ErrorCode int `json:"error_code"`
|
||||
ErrorDesc string `json:"error_description"`
|
||||
}
|
||||
|
||||
type userMeResp struct {
|
||||
Sub string `json:"sub"`
|
||||
}
|
||||
|
||||
type listResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
Total int `json:"total"`
|
||||
List []fileItem `json:"list"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type fileItem struct {
|
||||
FileID string `json:"fileId"`
|
||||
ParentID string `json:"parentId"`
|
||||
FileName string `json:"fileName"`
|
||||
FileSize int64 `json:"fileSize"`
|
||||
ResType int `json:"resType"`
|
||||
CTime int64 `json:"ctime"`
|
||||
UTime int64 `json:"utime"`
|
||||
}
|
||||
|
||||
type downloadResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
SignedURL string `json:"signedURL"`
|
||||
DownloadURL string `json:"downloadUrl"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type createDirResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
FileID string `json:"fileId"`
|
||||
FileName string `json:"fileName"`
|
||||
ResType int `json:"resType"`
|
||||
CTime int64 `json:"ctime"`
|
||||
UTime int64 `json:"utime"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type commonResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
}
|
||||
|
||||
// taskResp is used for async operations (delete, move, copy) that return a task ID.
|
||||
type taskResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
TaskID string `json:"taskId"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type taskStatusResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
Status int `json:"status"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type uploadTokenResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data uploadTokenData `json:"data"`
|
||||
}
|
||||
|
||||
type uploadTokenData struct {
|
||||
TaskID string `json:"taskId"`
|
||||
AlreadyDone bool `json:"-"`
|
||||
ObjectPath string `json:"objectPath"`
|
||||
Provider any `json:"provider"`
|
||||
Region string `json:"region"`
|
||||
BucketName string `json:"bucketName"`
|
||||
EndPoint string `json:"endPoint"`
|
||||
FullEndPoint string `json:"fullEndPoint"`
|
||||
CallbackVar string `json:"callbackVar"`
|
||||
AccessKeyID string `json:"accessKeyID"`
|
||||
SecretAccessKey string `json:"secretAccessKey"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
Creds struct {
|
||||
AccessKeyID string `json:"accessKeyID"`
|
||||
SecretAccessKey string `json:"secretAccessKey"`
|
||||
SessionToken string `json:"sessionToken"`
|
||||
} `json:"creds"`
|
||||
}
|
||||
|
||||
type taskInfoResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
FileID string `json:"fileId"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
// assetsInfoResp is the response for storage details query.
|
||||
type assetsInfoResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
TotalSpaceSize int64 `json:"totalSpaceSize"`
|
||||
UsedSpaceSize int64 `json:"usedSpaceSize"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
func (r assetsInfoResp) IsSuccess() bool {
|
||||
return isSuccessMsg(r.Msg)
|
||||
}
|
||||
|
||||
// Offline download types
|
||||
|
||||
type offlineResolveResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data OfflineResolveData `json:"data"`
|
||||
}
|
||||
|
||||
type OfflineResolveData struct {
|
||||
ResType int `json:"resType"`
|
||||
BTResInfo *OfflineBTResInfo `json:"btResInfo"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type OfflineBTResInfo struct {
|
||||
InfoHash string `json:"infoHash"`
|
||||
FileName string `json:"fileName"`
|
||||
FileSize int64 `json:"fileSize"`
|
||||
SubfilesNum int `json:"subfilesNum"`
|
||||
Subfiles []OfflineSubfile `json:"subfiles"`
|
||||
CreateTime int64 `json:"createTime"`
|
||||
ExcludeIndices []int `json:"excludeIndices"`
|
||||
}
|
||||
|
||||
type OfflineSubfile struct {
|
||||
FileName string `json:"fileName"`
|
||||
FileIndex *int `json:"fileIndex"`
|
||||
FileSize int64 `json:"fileSize"`
|
||||
}
|
||||
|
||||
type offlineCreateResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
TaskID string `json:"taskId"`
|
||||
URL string `json:"url"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type offlineDeleteResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
TaskIDs []string `json:"taskIds"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type offlineListResp struct {
|
||||
Code int `json:"code"`
|
||||
Msg string `json:"msg"`
|
||||
Data struct {
|
||||
StatusCounts []struct {
|
||||
Status int `json:"status"`
|
||||
Count int `json:"count"`
|
||||
} `json:"statusCounts"`
|
||||
Cursor string `json:"cursor"`
|
||||
List []OfflineTask `json:"list"`
|
||||
Total int `json:"total"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
type OfflineTask struct {
|
||||
TaskID string `json:"taskId"`
|
||||
FileName string `json:"fileName"`
|
||||
TotalSize int64 `json:"totalSize"`
|
||||
Status int `json:"status"`
|
||||
CreateTime int64 `json:"createTime"`
|
||||
Res string `json:"res"`
|
||||
ResType int `json:"resType"`
|
||||
Progress int `json:"progress"`
|
||||
FileID string `json:"fileId"`
|
||||
IsDir bool `json:"isDir"`
|
||||
Exist bool `json:"exist"`
|
||||
}
|
||||
|
||||
func unixOrZero(v int64) time.Time {
|
||||
if v <= 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.Unix(v, 0)
|
||||
}
|
||||
@@ -0,0 +1,407 @@
|
||||
package guangyapan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
streamPkg "github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/aliyun/aliyun-oss-go-sdk/oss"
|
||||
"github.com/avast/retry-go"
|
||||
"github.com/go-resty/resty/v2"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
// --- HTTP request helpers ---
|
||||
|
||||
func (d *GuangYaPan) accountErr(desc, short string, resp *resty.Response) string {
|
||||
msg := strings.TrimSpace(desc)
|
||||
if msg == "" {
|
||||
msg = strings.TrimSpace(short)
|
||||
}
|
||||
if msg == "" && resp != nil {
|
||||
msg = strings.TrimSpace(resp.String())
|
||||
}
|
||||
if msg == "" && resp != nil {
|
||||
msg = fmt.Sprintf("status=%d", resp.StatusCode())
|
||||
}
|
||||
if msg == "" {
|
||||
msg = "unknown error"
|
||||
}
|
||||
return msg
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) apiRateLimitWait(ctx context.Context, path string) error {
|
||||
value, _ := d.apiRateLimit.LoadOrStore(path, rate.NewLimiter(rate.Every(apiRateInterval), 1))
|
||||
return value.(*rate.Limiter).Wait(ctx)
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) postAPI(ctx context.Context, path string, body any, out any) error {
|
||||
if err := d.ensureAccessToken(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := d.apiRateLimitWait(ctx, path); err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := d.apiClient.R().
|
||||
SetContext(ctx).
|
||||
SetHeader("Authorization", "Bearer "+d.AccessToken).
|
||||
SetBody(body).
|
||||
SetResult(out).
|
||||
Post(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode() == 401 || resp.StatusCode() == 403 {
|
||||
if strings.TrimSpace(d.RefreshToken) == "" {
|
||||
return fmt.Errorf("request failed: status=%d body=%s", resp.StatusCode(), resp.String())
|
||||
}
|
||||
if err := d.refreshToken(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err = d.apiClient.R().
|
||||
SetContext(ctx).
|
||||
SetHeader("Authorization", "Bearer "+d.AccessToken).
|
||||
SetBody(body).
|
||||
SetResult(out).
|
||||
Post(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if resp.IsError() {
|
||||
return fmt.Errorf("request failed: status=%d body=%s", resp.StatusCode(), resp.String())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) setTempStatus(status string) {
|
||||
if d.statusTimer != nil {
|
||||
d.statusTimer.Stop()
|
||||
}
|
||||
// initStorage sets status to WORK after Init returns, so we update it shortly after.
|
||||
d.statusTimer = time.AfterFunc(200*time.Millisecond, func() {
|
||||
d.GetStorage().SetStatus(status)
|
||||
op.MustSaveDriverStorage(d)
|
||||
})
|
||||
}
|
||||
|
||||
// --- Task polling helpers ---
|
||||
|
||||
func (d *GuangYaPan) waitTaskDone(ctx context.Context, taskID string) error {
|
||||
const (
|
||||
maxTry = 30
|
||||
interval = 300 * time.Millisecond
|
||||
)
|
||||
for i := 0; i < maxTry; i++ {
|
||||
var out taskStatusResp
|
||||
if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_task_status", map[string]any{
|
||||
"taskId": taskID,
|
||||
}, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if !isSuccessMsg(out.Msg) {
|
||||
return fmt.Errorf("get task status failed: %s", strings.TrimSpace(out.Msg))
|
||||
}
|
||||
switch out.Data.Status {
|
||||
case 2:
|
||||
return nil
|
||||
case -1, 3:
|
||||
return fmt.Errorf("task %s failed with status=%d", taskID, out.Data.Status)
|
||||
}
|
||||
if i == maxTry-1 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(interval):
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("task %s timeout", taskID)
|
||||
}
|
||||
|
||||
// --- Upload helpers ---
|
||||
|
||||
func (d *GuangYaPan) getUploadToken(ctx context.Context, parentID, name string, size int64) (*uploadTokenData, int, error) {
|
||||
var out uploadTokenResp
|
||||
err := d.postAPI(ctx, "/nd.bizuserres.s/v1/get_res_center_token", map[string]any{
|
||||
"capacity": 2,
|
||||
"name": name,
|
||||
"parentId": parentID,
|
||||
"res": map[string]any{
|
||||
"fileSize": size,
|
||||
},
|
||||
}, &out)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
msg := strings.TrimSpace(out.Msg)
|
||||
if !isSuccessMsg(msg) && !isUploadAlreadyDone(msg) {
|
||||
return nil, out.Code, fmt.Errorf("get upload token failed: %s", msg)
|
||||
}
|
||||
if out.Data.TaskID == "" {
|
||||
return nil, out.Code, errors.New("get upload token failed: empty task id")
|
||||
}
|
||||
// When the backend reports the file is already uploaded/instant-uploaded,
|
||||
// it returns a valid TaskID without OSS credentials.
|
||||
// Mark it so the caller can skip the real upload and just wait for the task.
|
||||
if out.Code == 156 || isUploadAlreadyDone(msg) {
|
||||
out.Data.AlreadyDone = true
|
||||
}
|
||||
if out.Data.AccessKeyID == "" {
|
||||
out.Data.AccessKeyID = out.Data.Creds.AccessKeyID
|
||||
}
|
||||
if out.Data.SecretAccessKey == "" {
|
||||
out.Data.SecretAccessKey = out.Data.Creds.SecretAccessKey
|
||||
}
|
||||
if out.Data.SessionToken == "" {
|
||||
out.Data.SessionToken = out.Data.Creds.SessionToken
|
||||
}
|
||||
if strings.TrimSpace(out.Data.EndPoint) == "" {
|
||||
out.Data.EndPoint = strings.TrimSpace(out.Data.FullEndPoint)
|
||||
}
|
||||
if strings.TrimSpace(out.Data.EndPoint) != "" && !strings.HasPrefix(out.Data.EndPoint, "http://") && !strings.HasPrefix(out.Data.EndPoint, "https://") {
|
||||
if strings.TrimSpace(out.Data.FullEndPoint) != "" {
|
||||
out.Data.EndPoint = strings.TrimSpace(out.Data.FullEndPoint)
|
||||
} else if strings.TrimSpace(out.Data.BucketName) != "" {
|
||||
host := strings.TrimSpace(out.Data.EndPoint)
|
||||
prefix := strings.TrimSpace(out.Data.BucketName) + "."
|
||||
if strings.HasPrefix(host, prefix) {
|
||||
out.Data.EndPoint = "https://" + host
|
||||
} else {
|
||||
out.Data.EndPoint = "https://" + strings.TrimSpace(out.Data.BucketName) + "." + host
|
||||
}
|
||||
} else {
|
||||
out.Data.EndPoint = "https://" + strings.TrimSpace(out.Data.EndPoint)
|
||||
}
|
||||
}
|
||||
return &out.Data, out.Code, nil
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) waitUploadTaskInfo(ctx context.Context, taskID string) error {
|
||||
const (
|
||||
maxTry = 300
|
||||
interval = 1 * time.Second
|
||||
)
|
||||
for i := 0; i < maxTry; i++ {
|
||||
var out taskInfoResp
|
||||
if err := d.postAPI(ctx, "/nd.bizuserres.s/v1/file/get_info_by_task_id", map[string]any{
|
||||
"taskId": taskID,
|
||||
}, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
if out.Data.FileID != "" {
|
||||
return nil
|
||||
}
|
||||
switch out.Code {
|
||||
case 145, 146, 147, 155, 163, 0:
|
||||
// uploading/verifying/processing
|
||||
default:
|
||||
if strings.TrimSpace(out.Msg) != "" {
|
||||
return fmt.Errorf("upload task failed: code=%d msg=%s", out.Code, strings.TrimSpace(out.Msg))
|
||||
}
|
||||
}
|
||||
if i == maxTry-1 {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(interval):
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("upload task %s timeout", taskID)
|
||||
}
|
||||
|
||||
func (d *GuangYaPan) multipartUploadToOSS(ctx context.Context, bucket *oss.Bucket, objectPath string, file model.FileStreamer, up driver.UpdateProgress) error {
|
||||
partSize := calcUploadPartSize(file.GetSize())
|
||||
imur, err := bucket.InitiateMultipartUpload(objectPath, oss.Sequential())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
total := file.GetSize()
|
||||
partCount := int((total + partSize - 1) / partSize)
|
||||
|
||||
// Use StreamSectionReader for seekable, retryable chunk reads (hybrid cache).
|
||||
ss, err := streamPkg.NewStreamSectionReader(file, int(partSize), &up)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
parts := make([]oss.UploadPart, 0, partCount)
|
||||
for i := 0; i < partCount; i++ {
|
||||
if utils.IsCanceled(ctx) {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
offset := int64(i) * partSize
|
||||
length := partSize
|
||||
if remain := total - offset; length > remain {
|
||||
length = remain
|
||||
}
|
||||
|
||||
rd, err := ss.GetSectionReader(offset, length)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var part oss.UploadPart
|
||||
err = retry.Do(func() error {
|
||||
rd.Seek(0, io.SeekStart)
|
||||
var uploadErr error
|
||||
part, uploadErr = bucket.UploadPart(imur, driver.NewLimitedUploadStream(ctx, rd), length, i+1)
|
||||
return uploadErr
|
||||
},
|
||||
retry.Context(ctx),
|
||||
retry.Attempts(3),
|
||||
retry.DelayType(retry.BackOffDelay),
|
||||
retry.Delay(time.Second))
|
||||
ss.FreeSectionReader(rd)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to upload part %d: %w", i+1, err)
|
||||
}
|
||||
parts = append(parts, part)
|
||||
}
|
||||
|
||||
_, err = bucket.CompleteMultipartUpload(imur, parts)
|
||||
return err
|
||||
}
|
||||
|
||||
// --- Normalization helpers ---
|
||||
|
||||
func normalizeOSSEndpoint(endpoint, bucket string) string {
|
||||
ep := strings.TrimSpace(endpoint)
|
||||
if ep == "" {
|
||||
return ep
|
||||
}
|
||||
if !strings.HasPrefix(ep, "http://") && !strings.HasPrefix(ep, "https://") {
|
||||
ep = "https://" + ep
|
||||
}
|
||||
u, err := url.Parse(ep)
|
||||
if err != nil || u.Host == "" {
|
||||
return ep
|
||||
}
|
||||
host := u.Host
|
||||
prefix := strings.TrimSpace(bucket)
|
||||
if prefix != "" && strings.HasPrefix(host, prefix+".") {
|
||||
host = strings.TrimPrefix(host, prefix+".")
|
||||
}
|
||||
u.Host = host
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func normalizeDeviceID(v string) string {
|
||||
v = strings.ToLower(strings.TrimSpace(v))
|
||||
v = strings.ReplaceAll(v, "-", "")
|
||||
if len(v) != 32 {
|
||||
return ""
|
||||
}
|
||||
for _, ch := range v {
|
||||
if (ch < '0' || ch > '9') && (ch < 'a' || ch > 'f') {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func randomDeviceID() string {
|
||||
b := make([]byte, 16)
|
||||
if _, err := rand.Read(b); err != nil {
|
||||
return "0123456789abcdef0123456789abcdef"
|
||||
}
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
func normalizeCaptchaUsername(phone string) string {
|
||||
p := strings.TrimSpace(phone)
|
||||
p = strings.ReplaceAll(p, " ", "")
|
||||
p = strings.TrimPrefix(p, "+")
|
||||
// Keep only digits.
|
||||
b := make([]rune, 0, len(p))
|
||||
for _, ch := range p {
|
||||
if ch >= '0' && ch <= '9' {
|
||||
b = append(b, ch)
|
||||
}
|
||||
}
|
||||
digits := string(b)
|
||||
// Mainland number normalization: +86xxxxxxxxxxx -> xxxxxxxxxxx
|
||||
if strings.HasPrefix(digits, "86") && len(digits) > 11 {
|
||||
digits = digits[2:]
|
||||
}
|
||||
return digits
|
||||
}
|
||||
|
||||
func normalizePhoneE164(phone string) string {
|
||||
p := strings.TrimSpace(phone)
|
||||
if p == "" {
|
||||
return ""
|
||||
}
|
||||
p = strings.ReplaceAll(p, " ", "")
|
||||
if strings.HasPrefix(p, "+") {
|
||||
// Format as "+86 1xxxxxxxxxx" to match browser payload expectations.
|
||||
if strings.HasPrefix(p, "+86") && len(p) > 3 {
|
||||
rest := strings.TrimPrefix(p, "+86")
|
||||
return "+86 " + rest
|
||||
}
|
||||
return p
|
||||
}
|
||||
// If raw mainland number is provided, normalize with +86 prefix.
|
||||
digits := normalizeCaptchaUsername(p)
|
||||
if len(digits) == 11 {
|
||||
return "+86 " + digits
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func calcUploadPartSize(size int64) int64 {
|
||||
const (
|
||||
mb = int64(1024 * 1024)
|
||||
gb = int64(1024 * 1024 * 1024)
|
||||
)
|
||||
switch {
|
||||
case size <= 100*mb:
|
||||
return 1 * mb
|
||||
case size <= 16*gb:
|
||||
return 2 * mb
|
||||
case size <= 160*gb:
|
||||
return 4 * mb
|
||||
default:
|
||||
return 8 * mb
|
||||
}
|
||||
}
|
||||
|
||||
// isUploadAlreadyDone reports whether the upload-token response indicates the
|
||||
// file was already uploaded (instant upload). In that case the backend returns
|
||||
// a valid TaskID but no OSS credentials, and we should just wait for the task
|
||||
// instead of starting a real upload.
|
||||
func isUploadAlreadyDone(msg string) bool {
|
||||
msg = strings.TrimSpace(msg)
|
||||
if msg == "" {
|
||||
return false
|
||||
}
|
||||
if strings.EqualFold(msg, "上传已完成") {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(msg, "upload completed") {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(msg, "already uploaded") {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(msg, "秒传成功") {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -304,7 +304,7 @@ var findFromReg = regexp.MustCompile(`data : '(.+?)'`) // 查找from字符串
|
||||
func htmlFormToMap(html string) (map[string]string, error) {
|
||||
forms := findFromReg.FindStringSubmatch(html)
|
||||
if len(forms) != 2 {
|
||||
return nil, fmt.Errorf("not find file sgin")
|
||||
return nil, fmt.Errorf("not find file sign")
|
||||
}
|
||||
return formToMap(forms[1]), nil
|
||||
}
|
||||
|
||||
@@ -132,7 +132,7 @@ type FileOrFolderByShareUrl struct {
|
||||
//T int `json:"t"`
|
||||
|
||||
// 文件夹特有
|
||||
IsFloder bool `json:"-"`
|
||||
IsFolder bool `json:"-"`
|
||||
|
||||
//
|
||||
Url string `json:"-"`
|
||||
@@ -162,7 +162,7 @@ func (f *FileOrFolderByShareUrl) GetSize() int64 {
|
||||
}
|
||||
return *f.size
|
||||
}
|
||||
func (f *FileOrFolderByShareUrl) IsDir() bool { return f.IsFloder }
|
||||
func (f *FileOrFolderByShareUrl) IsDir() bool { return f.IsFolder }
|
||||
func (f *FileOrFolderByShareUrl) ModTime() time.Time {
|
||||
if f.time == nil {
|
||||
time := MustParseTime(f.Time)
|
||||
|
||||
+42
-24
@@ -157,25 +157,43 @@ func (d *LanZou) request(url string, method string, callback base.ReqCallback, u
|
||||
}
|
||||
|
||||
func (d *LanZou) Login() ([]*http.Cookie, error) {
|
||||
resp, err := base.NewRestyClient().SetRedirectPolicy(resty.NoRedirectPolicy()).
|
||||
R().SetFormData(map[string]string{
|
||||
"task": "3",
|
||||
"uid": d.Account,
|
||||
"pwd": d.Password,
|
||||
"setSessionId": "",
|
||||
"setSig": "",
|
||||
"setScene": "",
|
||||
"setTocen": "",
|
||||
"formhash": "",
|
||||
}).Post("https://up.woozooo.com/mlogin.php")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var vs string
|
||||
for retry := 0; retry < 3; retry++ {
|
||||
req := base.NewRestyClient().SetRedirectPolicy(resty.NoRedirectPolicy()).R()
|
||||
|
||||
// 如果已计算出 acw_sc__v2,通过 cookie 携带
|
||||
if vs != "" {
|
||||
req.SetHeader("cookie", "acw_sc__v2="+vs)
|
||||
}
|
||||
|
||||
resp, err := req.SetFormData(map[string]string{
|
||||
"task": "3",
|
||||
"uid": d.Account,
|
||||
"pwd": d.Password,
|
||||
"setSessionId": "",
|
||||
"setSig": "",
|
||||
"setScene": "",
|
||||
"setTocen": "",
|
||||
"formhash": "",
|
||||
}).Post("https://up.woozooo.com/mlogin.php")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
bodyStr := resp.String()
|
||||
if strings.Contains(bodyStr, "acw_sc__v2") {
|
||||
vs, err = CalcAcwScV2(bodyStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
if utils.Json.Get(resp.Body(), "zt").ToInt() != 1 {
|
||||
return nil, fmt.Errorf("login err: %s", resp.Body())
|
||||
}
|
||||
d.Cookie = CookieToString(resp.Cookies())
|
||||
return resp.Cookies(), nil
|
||||
}
|
||||
if utils.Json.Get(resp.Body(), "zt").ToInt() != 1 {
|
||||
return nil, fmt.Errorf("login err: %s", resp.Body())
|
||||
}
|
||||
d.Cookie = CookieToString(resp.Cookies())
|
||||
return resp.Cookies(), nil
|
||||
return nil, errors.New("acw_sc__v2 validation error")
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -572,14 +590,14 @@ func (d *LanZou) getFolderByShareUrl(pwd string, sharePageData string) ([]FileOr
|
||||
|
||||
files := make([]FileOrFolderByShareUrl, 0)
|
||||
// vip获取文件夹
|
||||
floders := findSubFolderReg.FindAllStringSubmatch(sharePageData, -1)
|
||||
for _, floder := range floders {
|
||||
if len(floder) == 3 {
|
||||
folders := findSubFolderReg.FindAllStringSubmatch(sharePageData, -1)
|
||||
for _, folder := range folders {
|
||||
if len(folder) == 3 {
|
||||
files = append(files, FileOrFolderByShareUrl{
|
||||
// Pwd: pwd, // 子文件夹不加密
|
||||
ID: floder[1],
|
||||
NameAll: floder[2],
|
||||
IsFloder: true,
|
||||
ID: folder[1],
|
||||
NameAll: folder[2],
|
||||
IsFolder: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,8 +5,8 @@ go 1.25.0
|
||||
toolchain go1.26.4
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0
|
||||
github.com/KarpelesLab/reflink v1.0.2
|
||||
github.com/KirCute/zip v1.0.1
|
||||
github.com/OpenListTeam/go-cache v0.1.0
|
||||
@@ -16,7 +16,7 @@ require (
|
||||
github.com/OpenListTeam/wopan-sdk-go v0.1.5
|
||||
github.com/ProtonMail/go-crypto v1.4.1
|
||||
github.com/ProtonMail/gopenpgp/v2 v2.10.0
|
||||
github.com/SheltonZhu/115driver v1.3.4
|
||||
github.com/SheltonZhu/115driver v1.3.5
|
||||
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible
|
||||
github.com/antchfx/htmlquery v1.3.6
|
||||
github.com/antchfx/xpath v1.3.6
|
||||
@@ -65,7 +65,7 @@ require (
|
||||
github.com/pquerna/otp v1.5.0
|
||||
github.com/quic-go/quic-go v0.60.0
|
||||
github.com/rclone/rclone v1.74.4
|
||||
github.com/shirou/gopsutil/v4 v4.26.6
|
||||
github.com/shirou/gopsutil/v4 v4.26.7
|
||||
github.com/sirupsen/logrus v1.9.4
|
||||
github.com/spf13/afero v1.15.0
|
||||
github.com/spf13/cobra v1.10.2
|
||||
@@ -74,23 +74,23 @@ require (
|
||||
github.com/tchap/go-patricia/v2 v2.3.3
|
||||
github.com/u2takey/ffmpeg-go v0.5.0
|
||||
github.com/upyun/go-sdk/v3 v3.0.4
|
||||
github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286
|
||||
github.com/zzzhr1990/go-common-entity v0.0.0-20250202070650-1a200048f0d3
|
||||
golang.org/x/crypto v0.53.0
|
||||
golang.org/x/crypto v0.54.0
|
||||
golang.org/x/image v0.43.0
|
||||
golang.org/x/net v0.56.0
|
||||
golang.org/x/net v0.57.0
|
||||
golang.org/x/oauth2 v0.36.0
|
||||
golang.org/x/time v0.15.0
|
||||
google.golang.org/appengine v1.6.8
|
||||
gopkg.in/ldap.v3 v3.1.0
|
||||
gorm.io/driver/mysql v1.5.7
|
||||
gorm.io/driver/mysql v1.6.0
|
||||
gorm.io/driver/postgres v1.6.0
|
||||
gorm.io/driver/sqlite v1.6.0
|
||||
gorm.io/gorm v1.31.2
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect
|
||||
filippo.io/edwards25519 v1.1.0 // indirect
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect
|
||||
github.com/BurntSushi/toml v1.6.0 // indirect
|
||||
github.com/ProtonMail/bcrypt v0.0.0-20211005172633-e235017c1baf // indirect
|
||||
github.com/ProtonMail/gluon v0.17.1-0.20230724134000-308be39be96e // indirect
|
||||
@@ -111,7 +111,7 @@ require (
|
||||
github.com/coreos/go-systemd/v22 v22.6.0 // indirect
|
||||
github.com/cronokirby/saferith v0.33.1-0.20250226174546-1f11f94ce488 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.10.0 // indirect
|
||||
github.com/ebitengine/purego v0.10.2 // indirect
|
||||
github.com/emersion/go-message v0.18.2 // indirect
|
||||
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff // indirect
|
||||
github.com/geoffgarside/ber v1.2.0 // indirect
|
||||
@@ -174,7 +174,7 @@ require (
|
||||
github.com/klauspost/pgzip v1.2.6 // indirect
|
||||
github.com/matoous/go-nanoid/v2 v2.1.0 // indirect
|
||||
github.com/microcosm-cc/bluemonday v1.0.27
|
||||
github.com/nwaples/rardecode/v2 v2.2.5
|
||||
github.com/nwaples/rardecode/v2 v2.3.0
|
||||
github.com/sorairolake/lzip-go v0.3.8 // indirect
|
||||
github.com/taruti/bytepool v0.0.0-20160310082835-5e3a9ea56543 // indirect
|
||||
github.com/ulikunitz/xz v0.5.15 // indirect
|
||||
@@ -225,7 +225,7 @@ require (
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.30.1 // indirect
|
||||
github.com/go-sql-driver/mysql v1.7.0 // indirect
|
||||
github.com/go-sql-driver/mysql v1.8.1 // indirect
|
||||
github.com/go-webauthn/x v0.1.23 // indirect
|
||||
github.com/goccy/go-json v0.10.5 // indirect
|
||||
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
|
||||
@@ -237,7 +237,7 @@ require (
|
||||
github.com/hashicorp/go-version v1.6.0 // indirect
|
||||
github.com/henrybear327/Proton-API-Bridge v1.0.0
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/ipfs/go-cid v0.5.0
|
||||
github.com/ipfs/go-cid v0.6.2
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/pgx/v5 v5.6.0 // indirect
|
||||
@@ -266,7 +266,7 @@ require (
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/mr-tron/base58 v1.2.0 // indirect
|
||||
github.com/mr-tron/base58 v1.3.0 // indirect
|
||||
github.com/mschoch/smat v0.2.0 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
@@ -274,11 +274,11 @@ require (
|
||||
github.com/multiformats/go-base32 v0.1.0 // indirect
|
||||
github.com/multiformats/go-base36 v0.2.0 // indirect
|
||||
github.com/multiformats/go-multiaddr v0.9.0 // indirect
|
||||
github.com/multiformats/go-multibase v0.2.0 // indirect
|
||||
github.com/multiformats/go-multibase v0.3.0 // indirect
|
||||
github.com/multiformats/go-multicodec v0.9.0 // indirect
|
||||
github.com/multiformats/go-multihash v0.2.3 // indirect
|
||||
github.com/multiformats/go-multistream v0.4.1 // indirect
|
||||
github.com/multiformats/go-varint v0.0.7 // indirect
|
||||
github.com/multiformats/go-varint v0.1.0 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.27 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
@@ -305,8 +305,8 @@ require (
|
||||
go.etcd.io/bbolt v1.4.3 // indirect
|
||||
golang.org/x/arch v0.23.0 // indirect
|
||||
golang.org/x/sync v0.22.0
|
||||
golang.org/x/sys v0.46.0
|
||||
golang.org/x/term v0.44.0 // indirect
|
||||
golang.org/x/sys v0.47.0
|
||||
golang.org/x/term v0.45.0 // indirect
|
||||
golang.org/x/text v0.40.0
|
||||
golang.org/x/tools v0.47.0 // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
|
||||
|
||||
@@ -5,22 +5,30 @@ cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIi
|
||||
cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c=
|
||||
cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs=
|
||||
cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10=
|
||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1 h1:Wc1ml6QlJs2BHQ/9Bqu1jiyggbsSjramq2oUmp5WeIo=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.1/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0 h1:fou+2+WFTib47nS+nz/ozhEBnvU96bKHy6LjRsY4E28=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.21.0/go.mod h1:t76Ruy8AHvUAC8GfMWJMa0ElSbuIcO03NLpynfbgsPA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0 h1:aokoqcHvaGjiM3VpjKDfMMnF/8epJ+Q1HLJ7CudztqE=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.22.0/go.mod h1:/WYEx9pcM9Y+Dd/APJaNlSvVSvzl54rrMdZT5+Oi2LM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2 h1:FwladfywkNirM+FZYLBR2kBz5C8Tg0fw5w5Y7meRXWI=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.2/go.mod h1:vv5Ad0RrIoT1lJFdWBZwt4mB1+j+V8DUroixmKDTCdk=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4 h1:jWQK1GI+LeGGUKBADtcH2rRqPxYB1Ljwms5gFA2LqrM=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.4/go.mod h1:8mwH4klAm9DUgR2EEHyEEAQlRDvLPyg5fQry3y+cDew=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw=
|
||||
github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=
|
||||
github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
|
||||
github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
|
||||
@@ -77,6 +85,8 @@ github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4=
|
||||
github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk=
|
||||
github.com/SheltonZhu/115driver v1.3.4 h1:joUFKN591n7KiTEGnBw0ZHjPzhl3G1PpZrwv2YHhoC8=
|
||||
github.com/SheltonZhu/115driver v1.3.4/go.mod h1:OujS7azslg1/bn85sPSHnNsp4/WBI9/TiijtZL9kuSQ=
|
||||
github.com/SheltonZhu/115driver v1.3.5 h1:NlJA61RskdwHhIZyg4519696lCk1n4aid0lKARsMwFg=
|
||||
github.com/SheltonZhu/115driver v1.3.5/go.mod h1:ag0B8Vs6TPFQjTNFLVVxA7dINPt4q+WR47+JIPJhmTk=
|
||||
github.com/abbot/go-http-auth v0.4.0 h1:QjmvZ5gSC7jm3Zg54DqWE/T5m1t2AfDu6QlXJT0EVT0=
|
||||
github.com/abbot/go-http-auth v0.4.0/go.mod h1:Cz6ARTIzApMJDzh5bRMSUou6UMSp0IEXg9km/ci7TJM=
|
||||
github.com/adrg/xdg v0.5.3 h1:xRnxJXne7+oWDatRhR1JLnvuccuIeCoBu2rtuLqQB78=
|
||||
@@ -280,6 +290,8 @@ github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564 h1:I6KUy4CI6hHjqnyJL
|
||||
github.com/dustinxie/ecc v0.0.0-20210511000915-959544187564/go.mod h1:yekO+3ZShy19S+bsmnERmznGy9Rfg6dWWWpiGJjNAz8=
|
||||
github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU=
|
||||
github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/ebitengine/purego v0.10.2 h1:W809HbnvzAxgdm+aOvlSekrM16wGCdT/e76+9tS7gzE=
|
||||
github.com/ebitengine/purego v0.10.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
|
||||
github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
|
||||
github.com/emersion/go-vcard v0.0.0-20241024213814-c9703dde27ff h1:4N8wnS3f1hNHSmFD5zgFkWCyA4L1kCDkImPAtK7D6tg=
|
||||
@@ -345,6 +357,8 @@ github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAy
|
||||
github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
|
||||
github.com/go-sql-driver/mysql v1.7.0 h1:ueSltNNllEqE3qcWBTD0iQd3IpL/6U+mJxLkazJ7YPc=
|
||||
github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
|
||||
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
|
||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||
github.com/go-webauthn/webauthn v0.13.4 h1:q68qusWPcqHbg9STSxBLBHnsKaLxNO0RnVKaAqMuAuQ=
|
||||
github.com/go-webauthn/webauthn v0.13.4/go.mod h1:MglN6OH9ECxvhDqoq1wMoF6P6JRYDiQpC9nc5OomQmI=
|
||||
github.com/go-webauthn/x v0.1.23 h1:9lEO0s+g8iTyz5Vszlg/rXTGrx3CjcD0RZQ1GPZCaxI=
|
||||
@@ -427,6 +441,8 @@ github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ=
|
||||
github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg=
|
||||
github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg=
|
||||
github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk=
|
||||
github.com/ipfs/go-cid v0.6.2 h1:VuGwJd+KJTaMJ4S4d5EEf9SXc17YUblS5axCbocn9YE=
|
||||
github.com/ipfs/go-cid v0.6.2/go.mod h1:Xhwg8NzHeK9xPCEZkCw4idzPiuNMpX3fARuI5Iwj1Lo=
|
||||
github.com/ipfs/go-ipfs-api v0.7.0 h1:CMBNCUl0b45coC+lQCXEVpMhwoqjiaCwUIrM+coYW2Q=
|
||||
github.com/ipfs/go-ipfs-api v0.7.0/go.mod h1:AIxsTNB0+ZhkqIfTZpdZ0VR/cpX5zrXjATa3prSay3g=
|
||||
github.com/itsHenry35/gofakes3 v0.0.8 h1:1AgOl04IgoUV5r/WSK7ycnvwfpgharYLfVTmnzk5miw=
|
||||
@@ -556,6 +572,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G
|
||||
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
|
||||
github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o=
|
||||
github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc=
|
||||
github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI=
|
||||
github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8=
|
||||
github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM=
|
||||
github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
|
||||
@@ -572,6 +590,8 @@ github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sgg
|
||||
github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0=
|
||||
github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g=
|
||||
github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk=
|
||||
github.com/multiformats/go-multibase v0.3.0 h1:8helZD2+4Db7NNWFiktk2NePbF0boolBe6bDQvM4r68=
|
||||
github.com/multiformats/go-multibase v0.3.0/go.mod h1:MoBLQPCkRTOL3eveIPO81860j2AQY8JwcnNlRkGRUfI=
|
||||
github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg=
|
||||
github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k=
|
||||
github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U=
|
||||
@@ -580,6 +600,8 @@ github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3d
|
||||
github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q=
|
||||
github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8=
|
||||
github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU=
|
||||
github.com/multiformats/go-varint v0.1.0 h1:i2wqFp4sdl3IcIxfAonHQV9qU5OsZ4Ts9IOoETFs5dI=
|
||||
github.com/multiformats/go-varint v0.1.0/go.mod h1:5KVAVXegtfmNQQm/lCY+ATvDzvJJhSkUlGQV9wgObdI=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
|
||||
github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
|
||||
@@ -588,6 +610,8 @@ github.com/ncw/swift/v2 v2.0.5 h1:9o5Gsd7bInAFEqsGPcaUdsboMbqf8lnNtxqWKFT9iz8=
|
||||
github.com/ncw/swift/v2 v2.0.5/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk=
|
||||
github.com/nwaples/rardecode/v2 v2.2.5 h1:L5doqgGfQwI7qADJMqnkrSB86rpPsqQDrHeO0HWa5JY=
|
||||
github.com/nwaples/rardecode/v2 v2.2.5/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
|
||||
github.com/nwaples/rardecode/v2 v2.3.0 h1:CtgyxWm8ClLcSh1u4M58fOz6lmeb/j4V7KpaEi/6UtM=
|
||||
github.com/nwaples/rardecode/v2 v2.3.0/go.mod h1:7uz379lSxPe6j9nvzxUZ+n7mnJNgjsRNb6IbvGVHRmw=
|
||||
github.com/panjf2000/ants/v2 v2.4.2/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
@@ -659,6 +683,8 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIG
|
||||
github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs=
|
||||
github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ=
|
||||
github.com/shirou/gopsutil/v4 v4.26.7 h1:IXzpHz/dkMRYAhKkOXr1HB6SuzWU3eoyyeWe7g3bNZc=
|
||||
github.com/shirou/gopsutil/v4 v4.26.7/go.mod h1:5O9FjBiXoTDFatIWjZZosqj4pV0DRtLx598xGbBehzM=
|
||||
github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
|
||||
github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
|
||||
@@ -724,10 +750,6 @@ github.com/unknwon/goconfig v1.0.0 h1:rS7O+CmUdli1T+oDm7fYj1MwqNWtEJfNj+FqcUHML8
|
||||
github.com/unknwon/goconfig v1.0.0/go.mod h1:qu2ZQ/wcC/if2u32263HTVC39PeOQRSmidQk3DuDFQ8=
|
||||
github.com/upyun/go-sdk/v3 v3.0.4 h1:2DCJa/Yi7/3ZybT9UCPATSzvU3wpPPxhXinNlb1Hi8Q=
|
||||
github.com/upyun/go-sdk/v3 v3.0.4/go.mod h1:P/SnuuwhrIgAVRd/ZpzDWqCsBAf/oHg7UggbAxyZa0E=
|
||||
github.com/winfsp/cgofuse v1.6.0 h1:re3W+HTd0hj4fISPBqfsrwyvPFpzqhDu8doJ9nOPDB0=
|
||||
github.com/winfsp/cgofuse v1.6.0/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I=
|
||||
github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286 h1:tw5GqRXqExB/xghPoPLtVujBe9w9Pg1G78tvXCJNJAA=
|
||||
github.com/winfsp/cgofuse v1.6.1-0.20260126094232-f2c4fccdb286/go.mod h1:uxjoF2jEYT3+x+vC2KJddEGdk/LU8pRowXmyVMHSV5I=
|
||||
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
|
||||
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
@@ -791,6 +813,8 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
|
||||
golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476 h1:bsqhLWFR6G6xiQcb+JoGqdKdRU6WzPWmK8E0jxTjzo4=
|
||||
golang.org/x/exp v0.0.0-20250606033433-dcc06ee1d476/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
|
||||
@@ -821,6 +845,8 @@ golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
|
||||
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
|
||||
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@@ -855,6 +881,8 @@ golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
|
||||
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
@@ -867,6 +895,8 @@ golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
|
||||
golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
@@ -935,6 +965,8 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo=
|
||||
gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM=
|
||||
gorm.io/driver/mysql v1.6.0 h1:eNbLmNTpPpTOVZi8MMxCi2aaIm0ZpInbORNXDwyLGvg=
|
||||
gorm.io/driver/mysql v1.6.0/go.mod h1:D/oCC2GWK3M/dqoLxnOlaNKmXz8WNTfcS9y5ovaSqKo=
|
||||
gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
|
||||
gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
|
||||
gorm.io/driver/sqlite v1.6.0 h1:WHRRrIiulaPiPFmDcod6prc4l2VGVWHz80KspNsxSfQ=
|
||||
|
||||
@@ -249,6 +249,8 @@ func InitialSettings() []model.SettingItem {
|
||||
{Key: conf.StreamMaxClientUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE},
|
||||
{Key: conf.StreamMaxServerDownloadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE},
|
||||
{Key: conf.StreamMaxServerUploadSpeed, Value: "-1", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PRIVATE},
|
||||
{Key: conf.MultipartEnabled, Value: "true", Type: conf.TypeBool, Group: model.TRAFFIC, Flag: model.PUBLIC},
|
||||
{Key: conf.MultipartChunkSize, Value: "10", Type: conf.TypeNumber, Group: model.TRAFFIC, Flag: model.PUBLIC, Help: `chunk size of multipart upload in MB (positive integer), keep it under your CDN's request body limit; each active session buffers up to 8 chunks on the server's disk`},
|
||||
}
|
||||
additionalSettingItems := tool.Tools.Items()
|
||||
// 固定顺序
|
||||
|
||||
@@ -95,6 +95,9 @@ const (
|
||||
// thunder_browser
|
||||
ThunderBrowserTempDir = "thunder_browser_temp_dir"
|
||||
|
||||
// guangyapan
|
||||
GuangYaPanTempDir = "guangyapan_temp_dir"
|
||||
|
||||
// single
|
||||
Token = "token"
|
||||
IndexProgress = "index_progress"
|
||||
@@ -161,6 +164,8 @@ const (
|
||||
StreamMaxClientUploadSpeed = "max_client_upload_speed"
|
||||
StreamMaxServerDownloadSpeed = "max_server_download_speed"
|
||||
StreamMaxServerUploadSpeed = "max_server_upload_speed"
|
||||
MultipartEnabled = "multipart_enabled"
|
||||
MultipartChunkSize = "multipart_chunk_size"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -1,170 +0,0 @@
|
||||
package fuse
|
||||
|
||||
import "github.com/winfsp/cgofuse/fuse"
|
||||
|
||||
type Fs struct {
|
||||
RootFolder string
|
||||
fuse.FileSystemBase
|
||||
}
|
||||
|
||||
func (fs *Fs) Init() {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Destroy() {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Statfs(path string, stat *fuse.Statfs_t) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Mknod(path string, mode uint32, dev uint64) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Mkdir(path string, mode uint32) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Unlink(path string) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Rmdir(path string) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Link(oldpath string, newpath string) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Symlink(target string, newpath string) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Readlink(path string) (int, string) {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Rename(oldpath string, newpath string) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Chmod(path string, mode uint32) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Chown(path string, uid uint32, gid uint32) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Utimens(path string, tmsp []fuse.Timespec) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Access(path string, mask uint32) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Create(path string, flags int, mode uint32) (int, uint64) {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Open(path string, flags int) (int, uint64) {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Getattr(path string, stat *fuse.Stat_t, fh uint64) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Truncate(path string, size int64, fh uint64) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Read(path string, buff []byte, ofst int64, fh uint64) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Write(path string, buff []byte, ofst int64, fh uint64) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Flush(path string, fh uint64) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Release(path string, fh uint64) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Fsync(path string, datasync bool, fh uint64) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Opendir(path string) (int, uint64) {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Readdir(path string, fill func(name string, stat *fuse.Stat_t, ofst int64) bool, ofst int64, fh uint64) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Releasedir(path string, fh uint64) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Fsyncdir(path string, datasync bool, fh uint64) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Setxattr(path string, name string, value []byte, flags int) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Getxattr(path string, name string) (int, []byte) {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Removexattr(path string, name string) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (fs *Fs) Listxattr(path string, fill func(name string) bool) int {
|
||||
//TODO implement me
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
var _ fuse.FileSystemInterface = (*Fs)(nil)
|
||||
@@ -1,9 +0,0 @@
|
||||
package fuse
|
||||
|
||||
import "github.com/winfsp/cgofuse/fuse"
|
||||
|
||||
func Mount(mountSrc, mountDst string, opts []string) {
|
||||
fs := &Fs{RootFolder: mountSrc}
|
||||
host := fuse.NewFileSystemHost(fs)
|
||||
go host.Mount(mountDst, opts)
|
||||
}
|
||||
@@ -0,0 +1,607 @@
|
||||
package multipart
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
stdpath "path"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type State string
|
||||
|
||||
const (
|
||||
StateReceiving State = "receiving"
|
||||
StateCompleted State = "completed"
|
||||
StateFailedRetriable State = "failed_retriable"
|
||||
StateFailedPermanent State = "failed_permanent"
|
||||
StateAborted State = "aborted"
|
||||
)
|
||||
|
||||
// WindowSlots bounds the per-session disk footprint to WindowSlots*ChunkSize.
|
||||
var WindowSlots = 8
|
||||
|
||||
const (
|
||||
// defaultSessionTTL is the sliding inactivity timeout; it also serves as the
|
||||
// grace period during which finished sessions remain queryable.
|
||||
defaultSessionTTL = 30 * time.Minute
|
||||
gcInterval = time.Minute
|
||||
)
|
||||
|
||||
var (
|
||||
ErrSessionNotFound = errors.New("multipart upload session not found")
|
||||
ErrNotOwner = errors.New("multipart upload session belongs to another user")
|
||||
// errAborted wraps context.Canceled so a driver blocked on the stream sees
|
||||
// the abort as a canceled request and runs its cancellation cleanup.
|
||||
errAborted = fmt.Errorf("multipart upload aborted: %w", context.Canceled)
|
||||
)
|
||||
|
||||
// putFile is the pipeline tail: resolve the storage and run the regular upload
|
||||
// path. It mirrors the checks of fs.putDirectly (internal/fs/put.go) but calls
|
||||
// op.Put directly so the driver's progress callback can be observed.
|
||||
// It is a variable so session tests can stub the storage layer out.
|
||||
var putFile = func(ctx context.Context, dstDirPath string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
storage, dstDirActualPath, err := op.GetStorageAndActualPath(dstDirPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if storage.Config().NoUpload {
|
||||
return errs.UploadNotSupported
|
||||
}
|
||||
return op.Put(ctx, storage, dstDirActualPath, fs, up)
|
||||
}
|
||||
|
||||
// Session is one multipart upload: metadata survives pipeline attempts, the
|
||||
// window (chunk data) does not.
|
||||
type Session struct {
|
||||
ID string
|
||||
Path string // full destination path (dir + name), already user-joined
|
||||
DstDir string
|
||||
Name string
|
||||
Size int64
|
||||
ChunkSize int64
|
||||
Total int
|
||||
Mimetype string
|
||||
Modified time.Time
|
||||
Hashes map[*utils.HashType]string
|
||||
Creator *model.User
|
||||
|
||||
mu sync.Mutex
|
||||
state State
|
||||
err error
|
||||
attempt int
|
||||
win *Window
|
||||
done chan struct{}
|
||||
cancel context.CancelFunc
|
||||
prevCRCs []uint32
|
||||
prevSet []bool
|
||||
|
||||
storagePct atomic.Uint64 // math.Float64bits of the driver progress (0-100)
|
||||
lastActive atomic.Int64 // unix nano
|
||||
}
|
||||
|
||||
func (s *Session) touch() { s.lastActive.Store(time.Now().UnixNano()) }
|
||||
|
||||
func (s *Session) setStoragePct(p float64) { s.storagePct.Store(math.Float64bits(p)) }
|
||||
|
||||
// Snapshot is the wire representation of a session used by all endpoints.
|
||||
type SessionSnapshot struct {
|
||||
ID string `json:"upload_id"`
|
||||
State State `json:"state"`
|
||||
Attempt int `json:"attempt"`
|
||||
Path string `json:"path"`
|
||||
Size int64 `json:"size"`
|
||||
ChunkSize int64 `json:"chunk_size"`
|
||||
TotalChunks int `json:"total_chunks"`
|
||||
Received [][2]int `json:"received"`
|
||||
ReceivedBytes int64 `json:"received_bytes"`
|
||||
Frontier int `json:"frontier"`
|
||||
StorageProgress float64 `json:"storage_progress"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Session) snapshotLocked() SessionSnapshot {
|
||||
snap := SessionSnapshot{
|
||||
ID: s.ID,
|
||||
State: s.state,
|
||||
Attempt: s.attempt,
|
||||
Path: s.Path,
|
||||
Size: s.Size,
|
||||
ChunkSize: s.ChunkSize,
|
||||
TotalChunks: s.Total,
|
||||
Received: [][2]int{},
|
||||
StorageProgress: math.Float64frombits(s.storagePct.Load()),
|
||||
}
|
||||
if s.err != nil {
|
||||
snap.Error = s.err.Error()
|
||||
}
|
||||
switch {
|
||||
case s.state == StateCompleted:
|
||||
snap.Received = [][2]int{{0, s.Total - 1}}
|
||||
snap.ReceivedBytes = s.Size
|
||||
snap.Frontier = s.Total
|
||||
snap.StorageProgress = 100
|
||||
case s.win != nil:
|
||||
ws := s.win.Snapshot()
|
||||
snap.Received = ws.Received
|
||||
snap.ReceivedBytes = ws.ReceivedBytes
|
||||
snap.Frontier = ws.Frontier
|
||||
}
|
||||
return snap
|
||||
}
|
||||
|
||||
func (s *Session) Snapshot() SessionSnapshot {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.snapshotLocked()
|
||||
}
|
||||
|
||||
// InitReq carries everything the handler parsed from the init request.
|
||||
type InitReq struct {
|
||||
User *model.User
|
||||
Path string // full destination path, already user-joined
|
||||
Size int64
|
||||
ChunkSize int64 // final chunk size in bytes, already clamped by the handler
|
||||
Mimetype string
|
||||
Modified time.Time
|
||||
Hashes map[*utils.HashType]string
|
||||
}
|
||||
|
||||
// Manager owns all live sessions. Sessions are in-memory only (aligned with
|
||||
// upload tasks not being persisted); a restart drops them and the ring files
|
||||
// are swept on the next start.
|
||||
type Manager struct {
|
||||
mu sync.Mutex
|
||||
byID map[string]*Session
|
||||
byKey map[string]string
|
||||
gcOnce sync.Once
|
||||
ttl time.Duration // 0 means defaultSessionTTL; tests shrink it per instance
|
||||
}
|
||||
|
||||
func (m *Manager) sessionTTL() time.Duration {
|
||||
if m.ttl > 0 {
|
||||
return m.ttl
|
||||
}
|
||||
return defaultSessionTTL
|
||||
}
|
||||
|
||||
var DefaultManager = &Manager{
|
||||
byID: make(map[string]*Session),
|
||||
byKey: make(map[string]string),
|
||||
}
|
||||
|
||||
func (m *Manager) dir() string {
|
||||
return filepath.Join(conf.Conf.TempDir, "multipart")
|
||||
}
|
||||
|
||||
func sessionKey(userID uint, path string, size int64) string {
|
||||
return fmt.Sprintf("%d|%s|%d", userID, path, size)
|
||||
}
|
||||
|
||||
// hashesQualifyResume reports whether two hash sets prove the client is
|
||||
// re-uploading the same file: they must share at least one hash type and
|
||||
// agree on every shared one. Path+size alone is NOT enough to resume into a
|
||||
// receiving session — buffered chunks of a different same-sized file would be
|
||||
// silently mixed into the result.
|
||||
func hashesQualifyResume(old, new map[*utils.HashType]string) bool {
|
||||
shared := false
|
||||
for t, ov := range old {
|
||||
if nv, ok := new[t]; ok {
|
||||
if ov != nv {
|
||||
return false
|
||||
}
|
||||
shared = true
|
||||
}
|
||||
}
|
||||
return shared
|
||||
}
|
||||
|
||||
// StartGC sweeps ring files orphaned by a previous run and starts the expiry
|
||||
// loop. It is called at server startup so orphans are reclaimed even if no
|
||||
// multipart upload ever happens again; Init also calls it, so embedders that
|
||||
// skip the server wiring still get GC lazily.
|
||||
func (m *Manager) StartGC() {
|
||||
m.ensureGC()
|
||||
}
|
||||
|
||||
func (m *Manager) ensureGC() {
|
||||
m.gcOnce.Do(func() {
|
||||
// sweep ring files orphaned by a previous run; bootstrap's CleanTempDir
|
||||
// only runs when no transfer tasks are pending, so do not rely on it
|
||||
_ = os.RemoveAll(m.dir())
|
||||
go func() {
|
||||
ticker := time.NewTicker(gcInterval)
|
||||
for range ticker.C {
|
||||
m.gc()
|
||||
}
|
||||
}()
|
||||
})
|
||||
}
|
||||
|
||||
func (m *Manager) gc() {
|
||||
deadline := time.Now().Add(-m.sessionTTL()).UnixNano()
|
||||
m.mu.Lock()
|
||||
var expired []*Session
|
||||
for _, s := range m.byID {
|
||||
if s.lastActive.Load() < deadline {
|
||||
expired = append(expired, s)
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
for _, s := range expired {
|
||||
m.terminate(s, errors.New("multipart upload session expired"))
|
||||
}
|
||||
}
|
||||
|
||||
// terminate aborts a session (if still receiving) and drops it from the maps.
|
||||
func (m *Manager) terminate(s *Session, cause error) {
|
||||
s.mu.Lock()
|
||||
if s.state == StateReceiving {
|
||||
s.state = StateAborted
|
||||
s.err = cause
|
||||
}
|
||||
s.killAttemptLocked()
|
||||
s.mu.Unlock()
|
||||
m.remove(s)
|
||||
}
|
||||
|
||||
// killAttemptLocked stops the running pipeline attempt: the context cancel
|
||||
// interrupts drivers blocked on network I/O, and closing the window wakes a
|
||||
// driver blocked in Read (context cancellation cannot interrupt cond.Wait).
|
||||
// The caller must hold s.mu and must have set the final state first.
|
||||
func (s *Session) killAttemptLocked() {
|
||||
if s.cancel != nil {
|
||||
s.cancel()
|
||||
}
|
||||
if s.win != nil {
|
||||
_ = s.win.CloseWithError(errAborted)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) remove(s *Session) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
delete(m.byID, s.ID)
|
||||
key := sessionKey(s.Creator.ID, s.Path, s.Size)
|
||||
if m.byKey[key] == s.ID {
|
||||
delete(m.byKey, key)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) get(user *model.User, id string) (*Session, error) {
|
||||
m.mu.Lock()
|
||||
s, ok := m.byID[id]
|
||||
m.mu.Unlock()
|
||||
if !ok {
|
||||
return nil, ErrSessionNotFound
|
||||
}
|
||||
if s.Creator.ID != user.ID {
|
||||
return nil, ErrNotOwner
|
||||
}
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Init creates a session and starts its pipeline, or returns the live session
|
||||
// for the same (user, path, size) so an interrupted client resumes implicitly.
|
||||
func (m *Manager) Init(req InitReq) (SessionSnapshot, bool, error) {
|
||||
m.ensureGC()
|
||||
if req.Size <= 0 {
|
||||
return SessionSnapshot{}, false, fmt.Errorf("multipart upload requires a positive X-File-Size, got %d", req.Size)
|
||||
}
|
||||
if req.ChunkSize <= 0 {
|
||||
return SessionSnapshot{}, false, fmt.Errorf("invalid chunk size %d", req.ChunkSize)
|
||||
}
|
||||
key := sessionKey(req.User.ID, req.Path, req.Size)
|
||||
|
||||
m.mu.Lock()
|
||||
if id, ok := m.byKey[key]; ok {
|
||||
if s, ok := m.byID[id]; ok {
|
||||
s.mu.Lock()
|
||||
st := s.state
|
||||
s.mu.Unlock()
|
||||
// failed_retriable resumes unconditionally: nothing of the old
|
||||
// attempt survives except CRCs, and the re-fill CRC check catches
|
||||
// a changed file. A receiving session still holds data, so it only
|
||||
// resumes when hashes prove it is the same file.
|
||||
if st == StateFailedRetriable ||
|
||||
(st == StateReceiving && hashesQualifyResume(s.Hashes, req.Hashes)) {
|
||||
m.mu.Unlock()
|
||||
s.touch()
|
||||
return s.Snapshot(), true, nil
|
||||
}
|
||||
// finished session, or same path+size without proof of identity:
|
||||
// drop the old session and start fresh
|
||||
m.mu.Unlock()
|
||||
m.terminate(s, errors.New("superseded by a new upload of the same path and size"))
|
||||
m.mu.Lock()
|
||||
}
|
||||
}
|
||||
m.mu.Unlock()
|
||||
|
||||
dstDir, name := stdpath.Split(req.Path)
|
||||
s := &Session{
|
||||
ID: uuid.NewString(),
|
||||
Path: req.Path,
|
||||
DstDir: dstDir,
|
||||
Name: name,
|
||||
Size: req.Size,
|
||||
ChunkSize: req.ChunkSize,
|
||||
Mimetype: req.Mimetype,
|
||||
Modified: req.Modified,
|
||||
Hashes: req.Hashes,
|
||||
Creator: req.User,
|
||||
state: StateReceiving,
|
||||
}
|
||||
s.touch()
|
||||
|
||||
s.mu.Lock()
|
||||
if err := m.startAttemptLocked(s); err != nil {
|
||||
s.mu.Unlock()
|
||||
return SessionSnapshot{}, false, err
|
||||
}
|
||||
s.Total = s.win.TotalChunks()
|
||||
snap := s.snapshotLocked()
|
||||
s.mu.Unlock()
|
||||
|
||||
m.mu.Lock()
|
||||
m.byID[s.ID] = s
|
||||
m.byKey[key] = s.ID
|
||||
m.mu.Unlock()
|
||||
return snap, false, nil
|
||||
}
|
||||
|
||||
// startAttemptLocked builds a fresh window and spawns the pipeline goroutine.
|
||||
// The caller must hold s.mu.
|
||||
func (m *Manager) startAttemptLocked(s *Session) error {
|
||||
win, err := NewWindow(m.dir(), fmt.Sprintf("%s.%d", s.ID, s.attempt), s.ChunkSize, s.Size, WindowSlots)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.WithValue(context.Background(), conf.UserKey, s.Creator))
|
||||
done := make(chan struct{})
|
||||
s.win = win
|
||||
s.cancel = cancel
|
||||
s.done = done
|
||||
s.state = StateReceiving
|
||||
s.err = nil
|
||||
s.setStoragePct(0)
|
||||
|
||||
fileStream := &stream.FileStream{
|
||||
Obj: &model.Object{
|
||||
Name: s.Name,
|
||||
Size: s.Size,
|
||||
Modified: s.Modified,
|
||||
HashInfo: utils.NewHashInfoByMap(s.Hashes),
|
||||
},
|
||||
Reader: win,
|
||||
Mimetype: s.Mimetype,
|
||||
}
|
||||
fileStream.Add(win)
|
||||
|
||||
dstDir := s.DstDir
|
||||
put := putFile // capture: the seam must not be read after spawn
|
||||
go func() {
|
||||
err := put(ctx, dstDir, fileStream, s.setStoragePct)
|
||||
s.finishAttempt(win, err)
|
||||
close(done)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// finishAttempt records the pipeline outcome and harvests the CRC table for
|
||||
// re-fill verification. The window's data is gone at this point (op.Put closed
|
||||
// it); only metadata survives.
|
||||
func (s *Session) finishAttempt(win *Window, err error) {
|
||||
crcs, set := win.CRCs()
|
||||
_ = win.Close() // op.Put already closed it; make sure the ring file is gone anyway
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.touch()
|
||||
if s.win == win {
|
||||
s.win = nil
|
||||
}
|
||||
s.prevCRCs, s.prevSet = crcs, set
|
||||
switch {
|
||||
case err == nil:
|
||||
s.state = StateCompleted
|
||||
s.err = nil
|
||||
s.setStoragePct(100)
|
||||
case s.state != StateReceiving:
|
||||
// Abort/expiry already labeled this attempt; keep that state.
|
||||
if s.err == nil {
|
||||
s.err = err
|
||||
}
|
||||
case isPermanentPutError(err):
|
||||
s.state = StateFailedPermanent
|
||||
s.err = err
|
||||
default:
|
||||
s.state = StateFailedRetriable
|
||||
s.err = err
|
||||
s.attempt++
|
||||
}
|
||||
}
|
||||
|
||||
func isPermanentPutError(err error) bool {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return false
|
||||
}
|
||||
for _, target := range []error{
|
||||
errs.UploadNotSupported,
|
||||
errs.PermissionDenied,
|
||||
errs.StorageNotFound,
|
||||
errs.ObjectAlreadyExists,
|
||||
errs.RelativePath,
|
||||
errs.IgnoredSystemFile,
|
||||
} {
|
||||
if errors.Is(err, target) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Chunk feeds one chunk into the session. Re-sending chunk 0 to a
|
||||
// failed_retriable session re-fills it: a fresh window and pipeline attempt.
|
||||
func (m *Manager) Chunk(user *model.User, id string, idx int, body io.Reader) (SessionSnapshot, error) {
|
||||
s, err := m.get(user, id)
|
||||
if err != nil {
|
||||
return SessionSnapshot{}, err
|
||||
}
|
||||
s.touch()
|
||||
|
||||
s.mu.Lock()
|
||||
switch s.state {
|
||||
case StateReceiving:
|
||||
case StateFailedRetriable:
|
||||
if idx != 0 {
|
||||
snap := s.snapshotLocked()
|
||||
s.mu.Unlock()
|
||||
return snap, fmt.Errorf("upload attempt failed, resend from chunk 0 to retry: %w", s.err)
|
||||
}
|
||||
if err := m.startAttemptLocked(s); err != nil {
|
||||
snap := s.snapshotLocked()
|
||||
s.mu.Unlock()
|
||||
return snap, err
|
||||
}
|
||||
case StateCompleted:
|
||||
snap := s.snapshotLocked()
|
||||
s.mu.Unlock()
|
||||
return snap, nil // idempotent: stragglers after rapid-upload/finish succeed
|
||||
default:
|
||||
snap := s.snapshotLocked()
|
||||
s.mu.Unlock()
|
||||
return snap, fmt.Errorf("session is %s: %w", s.state, s.err)
|
||||
}
|
||||
win := s.win
|
||||
var prevCRC uint32
|
||||
hasPrev := false
|
||||
if idx < len(s.prevSet) && s.prevSet[idx] {
|
||||
prevCRC, hasPrev = s.prevCRCs[idx], true
|
||||
}
|
||||
s.mu.Unlock()
|
||||
|
||||
crc, err := win.WriteChunk(idx, body)
|
||||
if err != nil {
|
||||
// A closed window means the pipeline ended while this chunk was in
|
||||
// flight — rapid upload makes this the NORMAL case: the driver
|
||||
// succeeds off the hash alone with chunks still arriving. The window
|
||||
// closes (op.Put's defer) moments before the verdict is recorded, so
|
||||
// wait for the verdict instead of racing it, then absorb the chunk
|
||||
// idempotently if the upload in fact succeeded.
|
||||
if errors.Is(err, ErrClosed) || errors.Is(err, context.Canceled) {
|
||||
s.mu.Lock()
|
||||
done := s.done
|
||||
s.mu.Unlock()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(10 * time.Second): // pipeline teardown is µs-scale; never expected
|
||||
}
|
||||
}
|
||||
s.mu.Lock()
|
||||
completed := s.state == StateCompleted
|
||||
snap := s.snapshotLocked()
|
||||
s.mu.Unlock()
|
||||
if completed {
|
||||
return snap, nil
|
||||
}
|
||||
return snap, err
|
||||
}
|
||||
if hasPrev && crc != prevCRC {
|
||||
err := fmt.Errorf("chunk %d content changed between attempts, aborting", idx)
|
||||
s.mu.Lock()
|
||||
s.state = StateFailedPermanent
|
||||
s.err = err
|
||||
s.killAttemptLocked()
|
||||
s.mu.Unlock()
|
||||
return s.Snapshot(), err
|
||||
}
|
||||
return s.Snapshot(), nil
|
||||
}
|
||||
|
||||
// Complete waits for the pipeline outcome. It refuses to block while chunks
|
||||
// are still missing, so a buggy client cannot park a connection for the TTL.
|
||||
func (m *Manager) Complete(ctx context.Context, user *model.User, id string) (SessionSnapshot, error) {
|
||||
s, err := m.get(user, id)
|
||||
if err != nil {
|
||||
return SessionSnapshot{}, err
|
||||
}
|
||||
s.touch()
|
||||
for {
|
||||
s.mu.Lock()
|
||||
st := s.state
|
||||
done := s.done
|
||||
if st == StateReceiving && s.win != nil {
|
||||
if ws := s.win.Snapshot(); ws.ReceivedBytes < s.Size {
|
||||
snap := s.snapshotLocked()
|
||||
s.mu.Unlock()
|
||||
return snap, fmt.Errorf("cannot complete: %d of %d bytes received", ws.ReceivedBytes, s.Size)
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
if st != StateReceiving {
|
||||
break
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
case <-ctx.Done():
|
||||
return s.Snapshot(), ctx.Err()
|
||||
}
|
||||
}
|
||||
s.touch()
|
||||
s.mu.Lock()
|
||||
st, serr := s.state, s.err
|
||||
snap := s.snapshotLocked()
|
||||
s.mu.Unlock()
|
||||
if st == StateCompleted {
|
||||
m.remove(s) // served its purpose; frees the key for future uploads
|
||||
return snap, nil
|
||||
}
|
||||
return snap, fmt.Errorf("upload failed (%s): %w", st, serr)
|
||||
}
|
||||
|
||||
// Status looks a session up by id.
|
||||
func (m *Manager) Status(user *model.User, id string) (SessionSnapshot, error) {
|
||||
s, err := m.get(user, id)
|
||||
if err != nil {
|
||||
return SessionSnapshot{}, err
|
||||
}
|
||||
s.touch()
|
||||
return s.Snapshot(), nil
|
||||
}
|
||||
|
||||
// Find looks a live session up by destination path and size, for resume discovery.
|
||||
func (m *Manager) Find(user *model.User, path string, size int64) (SessionSnapshot, error) {
|
||||
m.mu.Lock()
|
||||
id, ok := m.byKey[sessionKey(user.ID, path, size)]
|
||||
m.mu.Unlock()
|
||||
if !ok {
|
||||
return SessionSnapshot{}, ErrSessionNotFound
|
||||
}
|
||||
return m.Status(user, id)
|
||||
}
|
||||
|
||||
// Abort cancels the pipeline and forgets the session immediately.
|
||||
func (m *Manager) Abort(user *model.User, id string) error {
|
||||
s, err := m.get(user, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.terminate(s, errors.New("multipart upload aborted by client"))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,650 @@
|
||||
package multipart
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/driver"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
// Tests in this file share the putFile seam and must not run in parallel.
|
||||
|
||||
func setupSessionTest(t *testing.T) *Manager {
|
||||
t.Helper()
|
||||
oldConf := conf.Conf
|
||||
conf.Conf = conf.DefaultConfig(t.TempDir())
|
||||
t.Cleanup(func() { conf.Conf = oldConf })
|
||||
return &Manager{byID: make(map[string]*Session), byKey: make(map[string]string)}
|
||||
}
|
||||
|
||||
func stubPut(t *testing.T, fn func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error) {
|
||||
t.Helper()
|
||||
orig := putFile
|
||||
putFile = fn
|
||||
t.Cleanup(func() { putFile = orig })
|
||||
}
|
||||
|
||||
func testUser() *model.User { return &model.User{ID: 7, Username: "tester"} }
|
||||
|
||||
func initReq(user *model.User, size, chunkSize int64) InitReq {
|
||||
return InitReq{
|
||||
User: user,
|
||||
Path: "/local/test.bin",
|
||||
Size: size,
|
||||
ChunkSize: chunkSize,
|
||||
Mimetype: "application/octet-stream",
|
||||
Modified: time.Unix(1700000000, 0),
|
||||
}
|
||||
}
|
||||
|
||||
func sendChunk(t *testing.T, m *Manager, user *model.User, id string, data []byte, idx int, chunkSize int64) SessionSnapshot {
|
||||
t.Helper()
|
||||
snap, err := m.Chunk(user, id, idx, bytes.NewReader(chunkOf(data, idx, chunkSize)))
|
||||
if err != nil {
|
||||
t.Fatalf("Chunk(%d): %v", idx, err)
|
||||
}
|
||||
return snap
|
||||
}
|
||||
|
||||
func waitState(t *testing.T, m *Manager, user *model.User, id string, want State) SessionSnapshot {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(10 * time.Second)
|
||||
for {
|
||||
snap, err := m.Status(user, id)
|
||||
if err != nil {
|
||||
t.Fatalf("Status: %v", err)
|
||||
}
|
||||
if snap.State == want {
|
||||
return snap
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("session state = %s, want %s (err: %s)", snap.State, want, snap.Error)
|
||||
}
|
||||
time.Sleep(2 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionHappyPath(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(3*chunkSize + 300)
|
||||
data := genData(totalSize)
|
||||
user := testUser()
|
||||
|
||||
got := make(chan []byte, 1)
|
||||
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
defer fs.Close()
|
||||
if dst != "/local/" {
|
||||
return fmt.Errorf("unexpected dst dir %q", dst)
|
||||
}
|
||||
if fs.GetName() != "test.bin" || fs.GetSize() != totalSize {
|
||||
return fmt.Errorf("unexpected stream meta %s/%d", fs.GetName(), fs.GetSize())
|
||||
}
|
||||
b, err := io.ReadAll(fs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
up(100)
|
||||
got <- b
|
||||
return nil
|
||||
})
|
||||
|
||||
snap, resumed, err := m.Init(initReq(user, totalSize, chunkSize))
|
||||
if err != nil || resumed {
|
||||
t.Fatalf("Init = (resumed=%v, err=%v)", resumed, err)
|
||||
}
|
||||
if snap.TotalChunks != 4 || snap.State != StateReceiving {
|
||||
t.Fatalf("init snapshot = %+v", snap)
|
||||
}
|
||||
for _, idx := range []int{1, 0, 3, 2} { // out of order on purpose
|
||||
sendChunk(t, m, user, snap.ID, data, idx, chunkSize)
|
||||
}
|
||||
final, err := m.Complete(context.Background(), user, snap.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Complete: %v", err)
|
||||
}
|
||||
if final.State != StateCompleted || final.StorageProgress != 100 {
|
||||
t.Fatalf("final snapshot = %+v", final)
|
||||
}
|
||||
if !bytes.Equal(<-got, data) {
|
||||
t.Fatal("driver received different bytes")
|
||||
}
|
||||
if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) {
|
||||
t.Fatalf("session should be removed after Complete, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionRetriableRefill(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(2 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
user := testUser()
|
||||
|
||||
attempts := 0
|
||||
got := make(chan []byte, 1)
|
||||
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
defer fs.Close()
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
buf := make([]byte, chunkSize)
|
||||
if _, err := io.ReadFull(fs, buf); err != nil {
|
||||
return err
|
||||
}
|
||||
return errors.New("transient storage hiccup")
|
||||
}
|
||||
b, err := io.ReadAll(fs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
got <- b
|
||||
return nil
|
||||
})
|
||||
|
||||
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
|
||||
if err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
|
||||
failed := waitState(t, m, user, snap.ID, StateFailedRetriable)
|
||||
if failed.Attempt != 1 {
|
||||
t.Fatalf("attempt = %d, want 1", failed.Attempt)
|
||||
}
|
||||
if len(failed.Received) != 0 {
|
||||
t.Fatalf("failed session must report nothing received, got %v", failed.Received)
|
||||
}
|
||||
|
||||
// chunks other than 0 are rejected until the client restarts the fill
|
||||
if _, err := m.Chunk(user, snap.ID, 1, bytes.NewReader(chunkOf(data, 1, chunkSize))); err == nil {
|
||||
t.Fatal("chunk 1 on failed_retriable session: expected error")
|
||||
}
|
||||
// re-fill from chunk 0 respawns the pipeline
|
||||
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
|
||||
sendChunk(t, m, user, snap.ID, data, 1, chunkSize)
|
||||
final, err := m.Complete(context.Background(), user, snap.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Complete after refill: %v", err)
|
||||
}
|
||||
if final.State != StateCompleted || attempts != 2 {
|
||||
t.Fatalf("state=%s attempts=%d, want completed/2", final.State, attempts)
|
||||
}
|
||||
if !bytes.Equal(<-got, data) {
|
||||
t.Fatal("driver received different bytes after refill")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionPermanentFailure(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
const chunkSize = 1024
|
||||
data := genData(chunkSize)
|
||||
user := testUser()
|
||||
|
||||
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
defer fs.Close()
|
||||
return fmt.Errorf("denied: %w", errs.PermissionDenied)
|
||||
})
|
||||
|
||||
snap, _, err := m.Init(initReq(user, chunkSize, chunkSize))
|
||||
if err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
waitState(t, m, user, snap.ID, StateFailedPermanent)
|
||||
if _, err := m.Chunk(user, snap.ID, 0, bytes.NewReader(data)); err == nil {
|
||||
t.Fatal("chunk on failed_permanent session: expected error")
|
||||
}
|
||||
if _, err := m.Complete(context.Background(), user, snap.ID); err == nil {
|
||||
t.Fatal("Complete on failed_permanent session: expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRefillCRCMismatch(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(2 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
user := testUser()
|
||||
|
||||
attempts := 0
|
||||
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
defer fs.Close()
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
buf := make([]byte, chunkSize)
|
||||
if _, err := io.ReadFull(fs, buf); err != nil {
|
||||
return err
|
||||
}
|
||||
return errors.New("transient")
|
||||
}
|
||||
_, err := io.ReadAll(fs)
|
||||
return err
|
||||
})
|
||||
|
||||
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
|
||||
if err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
|
||||
waitState(t, m, user, snap.ID, StateFailedRetriable)
|
||||
|
||||
tampered := genData(chunkSize + 5)[:chunkSize] // different content, same length
|
||||
if _, err := m.Chunk(user, snap.ID, 0, bytes.NewReader(tampered)); err == nil {
|
||||
t.Fatal("re-fill with changed content: expected error")
|
||||
}
|
||||
waitState(t, m, user, snap.ID, StateFailedPermanent)
|
||||
}
|
||||
|
||||
func TestCompleteRefusesIncomplete(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(3 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
user := testUser()
|
||||
|
||||
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
defer fs.Close()
|
||||
_, err := io.ReadAll(fs)
|
||||
return err
|
||||
})
|
||||
|
||||
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
|
||||
if err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
|
||||
if _, err := m.Complete(context.Background(), user, snap.ID); err == nil {
|
||||
t.Fatal("Complete with missing chunks: expected error")
|
||||
}
|
||||
// unblock the pipeline goroutine before the test tears down
|
||||
if err := m.Abort(user, snap.ID); err != nil {
|
||||
t.Fatalf("Abort: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAbortAndOwnership(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(4 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
user := testUser()
|
||||
|
||||
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
defer fs.Close()
|
||||
_, err := io.ReadAll(fs)
|
||||
return err
|
||||
})
|
||||
|
||||
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
|
||||
if err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
|
||||
|
||||
stranger := &model.User{ID: 99, Username: "stranger"}
|
||||
if _, err := m.Status(stranger, snap.ID); !errors.Is(err, ErrNotOwner) {
|
||||
t.Fatalf("stranger Status err = %v, want ErrNotOwner", err)
|
||||
}
|
||||
if err := m.Abort(stranger, snap.ID); !errors.Is(err, ErrNotOwner) {
|
||||
t.Fatalf("stranger Abort err = %v, want ErrNotOwner", err)
|
||||
}
|
||||
if err := m.Abort(user, snap.ID); err != nil {
|
||||
t.Fatalf("Abort: %v", err)
|
||||
}
|
||||
if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) {
|
||||
t.Fatalf("Status after Abort err = %v, want ErrSessionNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpiry(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
m.ttl = 30 * time.Millisecond
|
||||
|
||||
const chunkSize = 1024
|
||||
user := testUser()
|
||||
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
defer fs.Close()
|
||||
_, err := io.ReadAll(fs)
|
||||
return err
|
||||
})
|
||||
|
||||
snap, _, err := m.Init(initReq(user, 4*chunkSize, chunkSize))
|
||||
if err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
time.Sleep(60 * time.Millisecond)
|
||||
m.gc()
|
||||
if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) {
|
||||
t.Fatalf("Status after expiry err = %v, want ErrSessionNotFound", err)
|
||||
}
|
||||
// the terminated pipeline must release the ring file
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
entries, _ := os.ReadDir(m.dir())
|
||||
if len(entries) == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
names := make([]string, 0, len(entries))
|
||||
for _, e := range entries {
|
||||
names = append(names, e.Name())
|
||||
}
|
||||
t.Fatalf("ring files not cleaned up after expiry: %v", names)
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitResume(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(4 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
user := testUser()
|
||||
|
||||
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
defer fs.Close()
|
||||
_, err := io.ReadAll(fs)
|
||||
return err
|
||||
})
|
||||
|
||||
hashed := initReq(user, totalSize, chunkSize)
|
||||
hashed.Hashes = map[*utils.HashType]string{utils.MD5: "0123456789abcdef0123456789abcdef"}
|
||||
snap, _, err := m.Init(hashed)
|
||||
if err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
sendChunk(t, m, user, snap.ID, data, 2, chunkSize)
|
||||
|
||||
// same hashes prove the same file: resume with buffered chunks skippable
|
||||
again, resumed, err := m.Init(hashed)
|
||||
if err != nil || !resumed {
|
||||
t.Fatalf("hashed re-Init = (resumed=%v, err=%v), want resumed", resumed, err)
|
||||
}
|
||||
if again.ID != snap.ID {
|
||||
t.Fatalf("resumed session id = %s, want %s", again.ID, snap.ID)
|
||||
}
|
||||
if len(again.Received) != 1 || again.Received[0] != [2]int{2, 2} {
|
||||
t.Fatalf("resumed received = %v, want [[2,2]]", again.Received)
|
||||
}
|
||||
|
||||
// a different size is a different upload
|
||||
other, resumed, err := m.Init(initReq(user, totalSize+1, chunkSize))
|
||||
if err != nil || resumed || other.ID == snap.ID {
|
||||
t.Fatalf("different-size Init = (id=%s, resumed=%v, err=%v)", other.ID, resumed, err)
|
||||
}
|
||||
|
||||
if _, err := m.Find(user, "/local/test.bin", totalSize); err != nil {
|
||||
t.Fatalf("Find: %v", err)
|
||||
}
|
||||
if _, err := m.Find(user, "/local/nope.bin", totalSize); !errors.Is(err, ErrSessionNotFound) {
|
||||
t.Fatalf("Find miss err = %v, want ErrSessionNotFound", err)
|
||||
}
|
||||
|
||||
// without hashes, path+size cannot prove identity against a receiving
|
||||
// session holding buffered data — the old session must be dropped, or a
|
||||
// same-sized different file would be silently mixed into the result
|
||||
bare := initReq(user, totalSize, chunkSize)
|
||||
fresh, resumed, err := m.Init(bare)
|
||||
if err != nil || resumed || fresh.ID == snap.ID {
|
||||
t.Fatalf("bare re-Init = (id=%s, resumed=%v, err=%v), want a fresh session", fresh.ID, resumed, err)
|
||||
}
|
||||
if _, err := m.Status(user, snap.ID); !errors.Is(err, ErrSessionNotFound) {
|
||||
t.Fatalf("superseded session err = %v, want ErrSessionNotFound", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResumeFailedRetriableWithoutHash(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(2 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
user := testUser()
|
||||
|
||||
attempts := 0
|
||||
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
defer fs.Close()
|
||||
attempts++
|
||||
if attempts == 1 {
|
||||
buf := make([]byte, chunkSize)
|
||||
if _, err := io.ReadFull(fs, buf); err != nil {
|
||||
return err
|
||||
}
|
||||
return errors.New("transient")
|
||||
}
|
||||
_, err := io.ReadAll(fs)
|
||||
return err
|
||||
})
|
||||
|
||||
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
|
||||
if err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
|
||||
waitState(t, m, user, snap.ID, StateFailedRetriable)
|
||||
|
||||
// a failed_retriable session keeps no chunk data, so identity proof is not
|
||||
// required to resume: the retry-button flow works without rapid hashing,
|
||||
// and the re-fill CRC check still catches a changed file
|
||||
again, resumed, err := m.Init(initReq(user, totalSize, chunkSize))
|
||||
if err != nil || !resumed || again.ID != snap.ID {
|
||||
t.Fatalf("re-Init on failed_retriable = (id=%s, resumed=%v, err=%v), want resumed same session", again.ID, resumed, err)
|
||||
}
|
||||
sendChunk(t, m, user, snap.ID, data, 0, chunkSize)
|
||||
sendChunk(t, m, user, snap.ID, data, 1, chunkSize)
|
||||
if _, err := m.Complete(context.Background(), user, snap.ID); err != nil {
|
||||
t.Fatalf("Complete after hashless refill: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRapidUploadShortCircuit(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(6 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
user := testUser()
|
||||
|
||||
wantMD5 := "0123456789abcdef0123456789abcdef"
|
||||
wantSHA1 := "da39a3ee5e6b4b0d3255bfef95601890afd80709"
|
||||
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
defer fs.Close()
|
||||
// the hash provided at init must reach the driver through the stream,
|
||||
// that is what lets PutRapid-style drivers skip the transfer entirely
|
||||
if got := fs.GetHash().GetHash(utils.MD5); got != wantMD5 {
|
||||
return fmt.Errorf("md5 not propagated: %q", got)
|
||||
}
|
||||
if got := fs.GetHash().GetHash(utils.SHA1); got != wantSHA1 {
|
||||
return fmt.Errorf("sha1 not propagated: %q", got)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond) // simulated rapid-upload API round trip
|
||||
up(100)
|
||||
return nil // rapid upload hit: succeed without reading the stream
|
||||
})
|
||||
|
||||
req := initReq(user, totalSize, chunkSize)
|
||||
req.Hashes = map[*utils.HashType]string{utils.MD5: wantMD5, utils.SHA1: wantSHA1}
|
||||
snap, _, err := m.Init(req)
|
||||
if err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
|
||||
// spam chunks across the completion moment: some land while receiving,
|
||||
// some race the window close, some arrive after completion — with the
|
||||
// completed-session absorption none of them may surface an error
|
||||
var wg sync.WaitGroup
|
||||
errCh := make(chan error, 64)
|
||||
for w := 0; w < 4; w++ {
|
||||
wg.Add(1)
|
||||
go func(w int) {
|
||||
defer wg.Done()
|
||||
for round := 0; round < 8; round++ {
|
||||
idx := (w*8 + round) % 6
|
||||
_, err := m.Chunk(user, snap.ID, idx, bytes.NewReader(chunkOf(data, idx, chunkSize)))
|
||||
// flow-control signals are part of the protocol, not failures
|
||||
if err != nil && !errors.Is(err, ErrChunkInFlight) && !errors.Is(err, ErrOutOfWindow) {
|
||||
errCh <- fmt.Errorf("worker %d chunk %d: %w", w, idx, err)
|
||||
return
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errCh)
|
||||
for err := range errCh {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
final, err := m.Complete(context.Background(), user, snap.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("Complete: %v", err)
|
||||
}
|
||||
if final.State != StateCompleted || final.StorageProgress != 100 {
|
||||
t.Fatalf("final snapshot = %+v", final)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChunkRacesRapidCompletion pins the exact race the completed-session
|
||||
// absorption exists for: a chunk request grabs the live window, stalls while
|
||||
// receiving its body, the pipeline completes off the hash alone (rapid
|
||||
// upload) and closes the window — the stalled chunk must then succeed
|
||||
// idempotently instead of surfacing "window closed" to a client whose upload
|
||||
// in fact just finished.
|
||||
func TestChunkRacesRapidCompletion(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(2 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
user := testUser()
|
||||
|
||||
proceed := make(chan struct{})
|
||||
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
defer fs.Close()
|
||||
<-proceed // the rapid-upload verdict arrives when the test says so
|
||||
return nil
|
||||
})
|
||||
|
||||
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
|
||||
if err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
|
||||
gate := make(chan struct{})
|
||||
type result struct {
|
||||
snap SessionSnapshot
|
||||
err error
|
||||
}
|
||||
resCh := make(chan result, 1)
|
||||
go func() {
|
||||
s, e := m.Chunk(user, snap.ID, 0,
|
||||
&gatedReader{release: gate, inner: bytes.NewReader(chunkOf(data, 0, chunkSize))})
|
||||
resCh <- result{s, e}
|
||||
}()
|
||||
|
||||
// wait until the chunk writer holds slot 0 in the filling state
|
||||
m.mu.Lock()
|
||||
sess := m.byID[snap.ID]
|
||||
m.mu.Unlock()
|
||||
sess.mu.Lock()
|
||||
win := sess.win
|
||||
sess.mu.Unlock()
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
win.mu.Lock()
|
||||
filling := win.slotState[0] == slotFilling
|
||||
win.mu.Unlock()
|
||||
if filling {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("chunk writer never reached the filling state")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
close(proceed) // rapid upload succeeds, window closes
|
||||
waitState(t, m, user, snap.ID, StateCompleted) // completion recorded
|
||||
close(gate) // stalled chunk finishes against the closed window
|
||||
|
||||
res := <-resCh
|
||||
if res.err != nil {
|
||||
t.Fatalf("in-flight chunk across rapid completion must be absorbed, got: %v", res.err)
|
||||
}
|
||||
if res.snap.State != StateCompleted {
|
||||
t.Fatalf("absorbed chunk snapshot state = %s, want completed", res.snap.State)
|
||||
}
|
||||
}
|
||||
|
||||
// TestChunkDuringCompletionGap covers the moment between op.Put closing the
|
||||
// window (its defer) and the verdict being recorded: a chunk hitting the
|
||||
// closed window inside that gap must wait for the verdict and be absorbed,
|
||||
// not bounce a "window closed" error at a client whose upload just succeeded.
|
||||
func TestChunkDuringCompletionGap(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(2 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
user := testUser()
|
||||
|
||||
windowClosed := make(chan struct{})
|
||||
allowVerdict := make(chan struct{})
|
||||
stubPut(t, func(ctx context.Context, dst string, fs *stream.FileStream, up driver.UpdateProgress) error {
|
||||
_ = fs.Close() // what op.Put's defer does before Put returns
|
||||
close(windowClosed)
|
||||
<-allowVerdict // hold the pipeline return open: this IS the gap
|
||||
return nil
|
||||
})
|
||||
|
||||
snap, _, err := m.Init(initReq(user, totalSize, chunkSize))
|
||||
if err != nil {
|
||||
t.Fatalf("Init: %v", err)
|
||||
}
|
||||
<-windowClosed
|
||||
|
||||
type result struct {
|
||||
snap SessionSnapshot
|
||||
err error
|
||||
}
|
||||
resCh := make(chan result, 1)
|
||||
go func() {
|
||||
s, e := m.Chunk(user, snap.ID, 0, bytes.NewReader(chunkOf(data, 0, chunkSize)))
|
||||
resCh <- result{s, e}
|
||||
}()
|
||||
|
||||
select {
|
||||
case r := <-resCh:
|
||||
t.Fatalf("chunk inside the gap returned early with (%s, %v); it must wait for the verdict", r.snap.State, r.err)
|
||||
case <-time.After(150 * time.Millisecond):
|
||||
// still waiting on the verdict, as designed
|
||||
}
|
||||
|
||||
close(allowVerdict)
|
||||
select {
|
||||
case r := <-resCh:
|
||||
if r.err != nil {
|
||||
t.Fatalf("gap chunk must be absorbed after completion, got: %v", r.err)
|
||||
}
|
||||
if r.snap.State != StateCompleted {
|
||||
t.Fatalf("gap chunk snapshot state = %s, want completed", r.snap.State)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("gap chunk never returned after the verdict")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitRejectsBadSize(t *testing.T) {
|
||||
m := setupSessionTest(t)
|
||||
if _, _, err := m.Init(initReq(testUser(), 0, 1024)); err == nil {
|
||||
t.Fatal("Init with size 0: expected error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,377 @@
|
||||
package multipart
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
)
|
||||
|
||||
const (
|
||||
slotFree uint8 = iota
|
||||
slotFilling
|
||||
slotReady
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrClosed is the sticky error after Close; all pending and future reads/writes fail with it.
|
||||
ErrClosed = errors.New("multipart upload window closed")
|
||||
// ErrChunkInFlight means another request is uploading the same chunk right now.
|
||||
ErrChunkInFlight = errors.New("chunk is being uploaded by another request")
|
||||
// ErrOutOfWindow means the chunk is still too far ahead of the consumption
|
||||
// frontier after waiting WindowWaitTimeout; the client should back off and
|
||||
// resend it later (flow control, not a failure).
|
||||
ErrOutOfWindow = errors.New("chunk is out of the receiving window")
|
||||
)
|
||||
|
||||
// WindowWaitTimeout bounds how long WriteChunk blocks waiting for its slot.
|
||||
// Browsers cannot reliably read responses sent before the request body is
|
||||
// consumed (they report a network error), so under backpressure it is far
|
||||
// better to hold the request until a slot frees — the wait must just stay
|
||||
// well below CDN request deadlines (Cloudflare: ~100s). Tests shrink this.
|
||||
var WindowWaitTimeout = 10 * time.Second
|
||||
|
||||
// Window reassembles concurrently uploaded chunks into a sequential stream.
|
||||
// Chunks land in a ring file of slots*chunkSize bytes (chunk i -> slot i%slots),
|
||||
// and Read serves bytes in order, blocking until the next needed chunk arrives.
|
||||
// A chunk slot is released as soon as the reader crosses its boundary, so the
|
||||
// disk footprint is bounded by slots*chunkSize regardless of the file size.
|
||||
//
|
||||
// WriteChunk is safe for concurrent use; Read must be called from a single
|
||||
// goroutine (the same contract as the FileStreamer it backs).
|
||||
type Window struct {
|
||||
mu sync.Mutex
|
||||
cond *sync.Cond
|
||||
|
||||
f *os.File
|
||||
path string
|
||||
|
||||
chunkSize int64
|
||||
totalSize int64
|
||||
total int
|
||||
slots int
|
||||
|
||||
slotState []uint8
|
||||
slotChunk []int
|
||||
readPos int64
|
||||
|
||||
crcs []uint32
|
||||
crcSet []bool
|
||||
|
||||
err error
|
||||
}
|
||||
|
||||
// Snapshot describes the receiving state, used for status responses and resume.
|
||||
type Snapshot struct {
|
||||
// Frontier is the next chunk index to be consumed (== TotalChunks when the stream is fully consumed).
|
||||
Frontier int
|
||||
// ReadPos is the number of bytes already consumed by the pipeline.
|
||||
ReadPos int64
|
||||
// ReceivedBytes is the number of payload bytes received from the client (consumed + buffered).
|
||||
ReceivedBytes int64
|
||||
// Received holds inclusive ranges of chunk indexes the client does not need to resend.
|
||||
Received [][2]int
|
||||
}
|
||||
|
||||
func NewWindow(dir, id string, chunkSize, totalSize int64, slots int) (*Window, error) {
|
||||
if chunkSize <= 0 || totalSize <= 0 || slots <= 0 {
|
||||
return nil, fmt.Errorf("invalid window params: chunkSize=%d totalSize=%d slots=%d", chunkSize, totalSize, slots)
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
path := filepath.Join(dir, id+".ring")
|
||||
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
total := int((totalSize + chunkSize - 1) / chunkSize)
|
||||
w := &Window{
|
||||
f: f,
|
||||
path: path,
|
||||
chunkSize: chunkSize,
|
||||
totalSize: totalSize,
|
||||
total: total,
|
||||
slots: slots,
|
||||
slotState: make([]uint8, slots),
|
||||
slotChunk: make([]int, slots),
|
||||
crcs: make([]uint32, total),
|
||||
crcSet: make([]bool, total),
|
||||
}
|
||||
for i := range w.slotChunk {
|
||||
w.slotChunk[i] = -1
|
||||
}
|
||||
w.cond = sync.NewCond(&w.mu)
|
||||
return w, nil
|
||||
}
|
||||
|
||||
func (w *Window) TotalChunks() int { return w.total }
|
||||
|
||||
// ChunkLen returns the payload length of chunk idx (the last chunk may be short).
|
||||
func (w *Window) ChunkLen(idx int) int64 {
|
||||
if idx == w.total-1 {
|
||||
return w.totalSize - int64(idx)*w.chunkSize
|
||||
}
|
||||
return w.chunkSize
|
||||
}
|
||||
|
||||
func stopTimer(t *time.Timer) {
|
||||
if t != nil {
|
||||
t.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
// frontier returns the next chunk index to be consumed. Callers must hold mu.
|
||||
func (w *Window) frontier() int {
|
||||
if w.readPos >= w.totalSize {
|
||||
return w.total
|
||||
}
|
||||
return int(w.readPos / w.chunkSize)
|
||||
}
|
||||
|
||||
// WriteChunk reads exactly the chunk payload from r into the ring and returns its CRC32 (IEEE).
|
||||
// Re-sending an already buffered or consumed chunk succeeds immediately without touching data.
|
||||
func (w *Window) WriteChunk(idx int, r io.Reader) (uint32, error) {
|
||||
w.mu.Lock()
|
||||
if w.err != nil {
|
||||
w.mu.Unlock()
|
||||
return 0, w.err
|
||||
}
|
||||
if idx < 0 || idx >= w.total {
|
||||
w.mu.Unlock()
|
||||
return 0, fmt.Errorf("chunk index %d out of range [0,%d)", idx, w.total)
|
||||
}
|
||||
length := w.ChunkLen(idx)
|
||||
slot := idx % w.slots
|
||||
// Admission control with a bounded wait: instead of bouncing a chunk the
|
||||
// moment its slot is busy, park the request until the reader frees the
|
||||
// slot. Rejecting fast would answer before the request body is read, which
|
||||
// browsers surface as a network error — so under backpressure, waiting IS
|
||||
// the flow control. sync.Cond has no timed wait; a timer broadcast wakes
|
||||
// the loop at the deadline.
|
||||
deadline := time.Now().Add(WindowWaitTimeout)
|
||||
var timer *time.Timer
|
||||
for {
|
||||
if w.err != nil {
|
||||
w.mu.Unlock()
|
||||
stopTimer(timer)
|
||||
return 0, w.err
|
||||
}
|
||||
if int64(idx)*w.chunkSize+length <= w.readPos {
|
||||
// already fully consumed
|
||||
crc := w.crcs[idx]
|
||||
w.mu.Unlock()
|
||||
stopTimer(timer)
|
||||
return crc, nil
|
||||
}
|
||||
if w.slotChunk[slot] == idx {
|
||||
if w.slotState[slot] == slotReady {
|
||||
crc := w.crcs[idx]
|
||||
w.mu.Unlock()
|
||||
stopTimer(timer)
|
||||
return crc, nil
|
||||
}
|
||||
if w.slotState[slot] == slotFilling {
|
||||
w.mu.Unlock()
|
||||
stopTimer(timer)
|
||||
return 0, ErrChunkInFlight
|
||||
}
|
||||
}
|
||||
if w.slotState[slot] == slotFree && idx < w.frontier()+w.slots {
|
||||
break // admissible
|
||||
}
|
||||
if !time.Now().Before(deadline) {
|
||||
w.mu.Unlock()
|
||||
stopTimer(timer)
|
||||
return 0, ErrOutOfWindow
|
||||
}
|
||||
if timer == nil {
|
||||
timer = time.AfterFunc(time.Until(deadline), func() {
|
||||
w.mu.Lock()
|
||||
w.cond.Broadcast()
|
||||
w.mu.Unlock()
|
||||
})
|
||||
}
|
||||
w.cond.Wait()
|
||||
}
|
||||
stopTimer(timer)
|
||||
w.slotState[slot] = slotFilling
|
||||
w.slotChunk[slot] = idx
|
||||
f := w.f
|
||||
w.mu.Unlock()
|
||||
|
||||
h := crc32.NewIEEE()
|
||||
n, err := utils.CopyWithBufferN(io.NewOffsetWriter(f, int64(slot)*w.chunkSize), io.TeeReader(r, h), length)
|
||||
if err == nil {
|
||||
// the body must contain exactly one chunk
|
||||
var b [1]byte
|
||||
if m, _ := io.ReadFull(r, b[:]); m > 0 {
|
||||
err = fmt.Errorf("chunk %d larger than expected %d bytes", idx, length)
|
||||
}
|
||||
} else {
|
||||
err = fmt.Errorf("incomplete chunk %d: got %d of %d bytes: %w", idx, n, length, err)
|
||||
}
|
||||
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.slotState[slot] != slotFilling || w.slotChunk[slot] != idx {
|
||||
// the window was closed and reset the slot while we were writing
|
||||
if w.err != nil {
|
||||
return 0, w.err
|
||||
}
|
||||
return 0, ErrClosed
|
||||
}
|
||||
if err == nil && w.err != nil {
|
||||
err = w.err
|
||||
}
|
||||
if err != nil {
|
||||
w.slotState[slot] = slotFree
|
||||
w.slotChunk[slot] = -1
|
||||
return 0, err
|
||||
}
|
||||
w.slotState[slot] = slotReady
|
||||
w.crcs[idx] = h.Sum32()
|
||||
w.crcSet[idx] = true
|
||||
w.cond.Broadcast()
|
||||
return w.crcs[idx], nil
|
||||
}
|
||||
|
||||
// Read serves the reassembled stream in order, blocking until the next chunk
|
||||
// is available, the window is closed, or the stream ends (io.EOF).
|
||||
func (w *Window) Read(p []byte) (int, error) {
|
||||
if len(p) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
w.mu.Lock()
|
||||
for {
|
||||
if w.err != nil {
|
||||
w.mu.Unlock()
|
||||
return 0, w.err
|
||||
}
|
||||
if w.readPos >= w.totalSize {
|
||||
w.mu.Unlock()
|
||||
return 0, io.EOF
|
||||
}
|
||||
cur := int(w.readPos / w.chunkSize)
|
||||
slot := cur % w.slots
|
||||
if w.slotState[slot] == slotReady && w.slotChunk[slot] == cur {
|
||||
chunkStart := int64(cur) * w.chunkSize
|
||||
chunkEnd := chunkStart + w.ChunkLen(cur)
|
||||
n := int64(len(p))
|
||||
if avail := chunkEnd - w.readPos; n > avail {
|
||||
n = avail
|
||||
}
|
||||
off := int64(slot)*w.chunkSize + (w.readPos - chunkStart)
|
||||
f := w.f
|
||||
w.mu.Unlock()
|
||||
|
||||
read, err := f.ReadAt(p[:n], off)
|
||||
|
||||
w.mu.Lock()
|
||||
if read > 0 {
|
||||
w.readPos += int64(read)
|
||||
if w.readPos >= chunkEnd {
|
||||
w.slotState[slot] = slotFree
|
||||
w.slotChunk[slot] = -1
|
||||
w.cond.Broadcast() // writers may be parked waiting for this slot
|
||||
}
|
||||
}
|
||||
sticky := w.err
|
||||
w.mu.Unlock()
|
||||
if read > 0 {
|
||||
return read, nil
|
||||
}
|
||||
if sticky != nil {
|
||||
return 0, sticky
|
||||
}
|
||||
if err == nil {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
w.cond.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
// Close makes all pending and future operations fail with ErrClosed and removes
|
||||
// the ring file. It is invoked by op.Put via FileStream.Closers when the
|
||||
// pipeline ends, and is safe to call multiple times.
|
||||
func (w *Window) Close() error {
|
||||
return w.CloseWithError(ErrClosed)
|
||||
}
|
||||
|
||||
// CloseWithError is Close with a caller-chosen sticky error. The session's
|
||||
// abort path passes an error wrapping context.Canceled so that a driver woken
|
||||
// up from a blocked Read treats the abort exactly like a canceled request
|
||||
// (e.g. the local driver only removes partially written files in that case).
|
||||
func (w *Window) CloseWithError(sticky error) error {
|
||||
w.mu.Lock()
|
||||
if w.err == nil {
|
||||
w.err = sticky
|
||||
}
|
||||
f := w.f
|
||||
w.f = nil
|
||||
for i := range w.slotState {
|
||||
w.slotState[i] = slotFree
|
||||
w.slotChunk[i] = -1
|
||||
}
|
||||
w.cond.Broadcast()
|
||||
w.mu.Unlock()
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
err := f.Close()
|
||||
if rmErr := os.Remove(w.path); rmErr != nil && err == nil {
|
||||
err = rmErr
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Snapshot reports the receiving state for status responses and resume discovery.
|
||||
func (w *Window) Snapshot() Snapshot {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
snap := Snapshot{Frontier: w.frontier(), ReadPos: w.readPos, ReceivedBytes: w.readPos}
|
||||
var ranges [][2]int
|
||||
if snap.Frontier > 0 {
|
||||
ranges = append(ranges, [2]int{0, snap.Frontier - 1})
|
||||
}
|
||||
for idx := snap.Frontier; idx < snap.Frontier+w.slots && idx < w.total; idx++ {
|
||||
slot := idx % w.slots
|
||||
if w.slotState[slot] != slotReady || w.slotChunk[slot] != idx {
|
||||
continue
|
||||
}
|
||||
start := int64(idx) * w.chunkSize
|
||||
var consumed int64
|
||||
if w.readPos > start {
|
||||
consumed = w.readPos - start
|
||||
}
|
||||
snap.ReceivedBytes += w.ChunkLen(idx) - consumed
|
||||
if len(ranges) > 0 && ranges[len(ranges)-1][1] == idx-1 {
|
||||
ranges[len(ranges)-1][1] = idx
|
||||
} else {
|
||||
ranges = append(ranges, [2]int{idx, idx})
|
||||
}
|
||||
}
|
||||
snap.Received = ranges
|
||||
return snap
|
||||
}
|
||||
|
||||
// CRCs returns a copy of the per-chunk CRC32 table and which entries are set.
|
||||
// It remains readable after Close, so the session can compare re-filled chunks
|
||||
// against a previous attempt.
|
||||
func (w *Window) CRCs() ([]uint32, []bool) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
crcs := make([]uint32, len(w.crcs))
|
||||
set := make([]bool, len(w.crcSet))
|
||||
copy(crcs, w.crcs)
|
||||
copy(set, w.crcSet)
|
||||
return crcs, set
|
||||
}
|
||||
@@ -0,0 +1,527 @@
|
||||
package multipart
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func genData(size int64) []byte {
|
||||
data := make([]byte, size)
|
||||
rnd := rand.New(rand.NewSource(size*7919 + 13))
|
||||
rnd.Read(data)
|
||||
return data
|
||||
}
|
||||
|
||||
func newTestWindow(t *testing.T, chunkSize, totalSize int64, slots int) *Window {
|
||||
t.Helper()
|
||||
w, err := NewWindow(t.TempDir(), "test", chunkSize, totalSize, slots)
|
||||
if err != nil {
|
||||
t.Fatalf("NewWindow: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = w.Close() })
|
||||
return w
|
||||
}
|
||||
|
||||
func chunkOf(data []byte, idx int, chunkSize int64) []byte {
|
||||
start := int64(idx) * chunkSize
|
||||
end := start + chunkSize
|
||||
if end > int64(len(data)) {
|
||||
end = int64(len(data))
|
||||
}
|
||||
return data[start:end]
|
||||
}
|
||||
|
||||
func writeChunkOK(t *testing.T, w *Window, data []byte, idx int) uint32 {
|
||||
t.Helper()
|
||||
crc, err := w.WriteChunk(idx, bytes.NewReader(chunkOf(data, idx, w.chunkSize)))
|
||||
if err != nil {
|
||||
t.Fatalf("WriteChunk(%d): %v", idx, err)
|
||||
}
|
||||
if want := crc32.ChecksumIEEE(chunkOf(data, idx, w.chunkSize)); crc != want {
|
||||
t.Fatalf("WriteChunk(%d) crc = %08x, want %08x", idx, crc, want)
|
||||
}
|
||||
return crc
|
||||
}
|
||||
|
||||
// readAllWithin reads the whole stream in a goroutine and fails the test on timeout,
|
||||
// so a reassembly bug cannot hang the suite.
|
||||
func readAllWithin(t *testing.T, w *Window, timeout time.Duration) []byte {
|
||||
t.Helper()
|
||||
type result struct {
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
ch := make(chan result, 1)
|
||||
go func() {
|
||||
data, err := io.ReadAll(w)
|
||||
ch <- result{data, err}
|
||||
}()
|
||||
select {
|
||||
case res := <-ch:
|
||||
if res.err != nil {
|
||||
t.Fatalf("ReadAll: %v", res.err)
|
||||
}
|
||||
return res.data
|
||||
case <-time.After(timeout):
|
||||
t.Fatal("ReadAll timed out")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestSequentialReadWrite(t *testing.T) {
|
||||
const chunkSize = 64 * 1024
|
||||
totalSize := int64(4*chunkSize + 32*1024) // last chunk is short
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 8)
|
||||
|
||||
go func() {
|
||||
for i := 0; i < w.TotalChunks(); i++ {
|
||||
if _, err := w.WriteChunk(i, bytes.NewReader(chunkOf(data, i, chunkSize))); err != nil {
|
||||
t.Errorf("WriteChunk(%d): %v", i, err)
|
||||
return
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}()
|
||||
|
||||
got := readAllWithin(t, w, 10*time.Second)
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Fatalf("reassembled stream differs: got %d bytes, want %d", len(got), len(data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutOfOrderWrites(t *testing.T) {
|
||||
const chunkSize = 16 * 1024
|
||||
totalSize := int64(5*chunkSize - 100)
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 8)
|
||||
|
||||
for _, idx := range []int{3, 0, 4, 2, 1} {
|
||||
writeChunkOK(t, w, data, idx)
|
||||
}
|
||||
got := readAllWithin(t, w, 10*time.Second)
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Fatal("reassembled stream differs after out-of-order writes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrentWriters(t *testing.T) {
|
||||
const chunkSize = 32 * 1024
|
||||
totalSize := int64(8 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 8)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < w.TotalChunks(); i++ {
|
||||
wg.Add(1)
|
||||
go func(idx int) {
|
||||
defer wg.Done()
|
||||
if _, err := w.WriteChunk(idx, bytes.NewReader(chunkOf(data, idx, chunkSize))); err != nil {
|
||||
t.Errorf("WriteChunk(%d): %v", idx, err)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
got := readAllWithin(t, w, 10*time.Second)
|
||||
wg.Wait()
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Fatal("reassembled stream differs after concurrent writes")
|
||||
}
|
||||
}
|
||||
|
||||
func setWaitTimeout(t *testing.T, d time.Duration) {
|
||||
t.Helper()
|
||||
old := WindowWaitTimeout
|
||||
WindowWaitTimeout = d
|
||||
t.Cleanup(func() { WindowWaitTimeout = old })
|
||||
}
|
||||
|
||||
func TestBackpressure(t *testing.T) {
|
||||
setWaitTimeout(t, 50*time.Millisecond) // assert the post-deadline rejection
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(5 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 2)
|
||||
|
||||
if _, err := w.WriteChunk(2, bytes.NewReader(chunkOf(data, 2, chunkSize))); !errors.Is(err, ErrOutOfWindow) {
|
||||
t.Fatalf("chunk 2 with frontier 0: err = %v, want ErrOutOfWindow", err)
|
||||
}
|
||||
writeChunkOK(t, w, data, 0)
|
||||
writeChunkOK(t, w, data, 1)
|
||||
// both slots occupied, frontier still 0
|
||||
if _, err := w.WriteChunk(2, bytes.NewReader(chunkOf(data, 2, chunkSize))); !errors.Is(err, ErrOutOfWindow) {
|
||||
t.Fatalf("chunk 2 with full window: err = %v, want ErrOutOfWindow", err)
|
||||
}
|
||||
// consume chunk 0 -> slot released, frontier advances
|
||||
buf := make([]byte, chunkSize)
|
||||
if _, err := io.ReadFull(w, buf); err != nil {
|
||||
t.Fatalf("ReadFull chunk 0: %v", err)
|
||||
}
|
||||
if !bytes.Equal(buf, chunkOf(data, 0, chunkSize)) {
|
||||
t.Fatal("chunk 0 content differs")
|
||||
}
|
||||
writeChunkOK(t, w, data, 2)
|
||||
// chunk 3 maps to the slot still holding buffered chunk 1
|
||||
if _, err := w.WriteChunk(3, bytes.NewReader(chunkOf(data, 3, chunkSize))); !errors.Is(err, ErrOutOfWindow) {
|
||||
t.Fatalf("chunk 3 with occupied slot: err = %v, want ErrOutOfWindow", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteChunkWaitsForSlot pins the browser-friendly flow control: a chunk
|
||||
// whose slot is occupied parks until the reader frees it instead of bouncing
|
||||
// with an immediate rejection (early responses read as network errors in
|
||||
// browsers).
|
||||
func TestWriteChunkWaitsForSlot(t *testing.T) {
|
||||
setWaitTimeout(t, 5*time.Second)
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(5 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 2)
|
||||
|
||||
writeChunkOK(t, w, data, 0)
|
||||
writeChunkOK(t, w, data, 1)
|
||||
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := w.WriteChunk(2, bytes.NewReader(chunkOf(data, 2, chunkSize)))
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case err := <-done:
|
||||
t.Fatalf("chunk 2 should be parked while the window is full, returned %v", err)
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
// parked, as designed
|
||||
}
|
||||
|
||||
// consuming chunk 0 frees its slot and must wake the parked writer
|
||||
buf := make([]byte, chunkSize)
|
||||
if _, err := io.ReadFull(w, buf); err != nil {
|
||||
t.Fatalf("ReadFull: %v", err)
|
||||
}
|
||||
select {
|
||||
case err := <-done:
|
||||
if err != nil {
|
||||
t.Fatalf("parked chunk after slot freed: %v", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("parked chunk never admitted after its slot freed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdempotentResend(t *testing.T) {
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(2 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 4)
|
||||
|
||||
first := writeChunkOK(t, w, data, 0)
|
||||
again := writeChunkOK(t, w, data, 0) // buffered, not yet consumed
|
||||
if first != again {
|
||||
t.Fatalf("resend crc = %08x, want %08x", again, first)
|
||||
}
|
||||
buf := make([]byte, chunkSize)
|
||||
if _, err := io.ReadFull(w, buf); err != nil {
|
||||
t.Fatalf("ReadFull: %v", err)
|
||||
}
|
||||
consumed := writeChunkOK(t, w, data, 0) // already consumed
|
||||
if consumed != first {
|
||||
t.Fatalf("post-consume resend crc = %08x, want %08x", consumed, first)
|
||||
}
|
||||
}
|
||||
|
||||
// gatedReader blocks the first Read until released, to hold a chunk in the filling state.
|
||||
type gatedReader struct {
|
||||
release <-chan struct{}
|
||||
inner io.Reader
|
||||
}
|
||||
|
||||
func (g *gatedReader) Read(p []byte) (int, error) {
|
||||
<-g.release
|
||||
return g.inner.Read(p)
|
||||
}
|
||||
|
||||
func TestInFlightConflict(t *testing.T) {
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(2 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 4)
|
||||
|
||||
release := make(chan struct{})
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := w.WriteChunk(0, &gatedReader{release: release, inner: bytes.NewReader(chunkOf(data, 0, chunkSize))})
|
||||
done <- err
|
||||
}()
|
||||
|
||||
// wait until the writer marked the slot as filling
|
||||
deadline := time.Now().Add(5 * time.Second)
|
||||
for {
|
||||
w.mu.Lock()
|
||||
filling := w.slotState[0] == slotFilling
|
||||
w.mu.Unlock()
|
||||
if filling {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatal("writer never reached filling state")
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
|
||||
if _, err := w.WriteChunk(0, bytes.NewReader(chunkOf(data, 0, chunkSize))); !errors.Is(err, ErrChunkInFlight) {
|
||||
t.Fatalf("concurrent same-chunk write: err = %v, want ErrChunkInFlight", err)
|
||||
}
|
||||
close(release)
|
||||
if err := <-done; err != nil {
|
||||
t.Fatalf("gated WriteChunk: %v", err)
|
||||
}
|
||||
writeChunkOK(t, w, data, 0) // idempotent after settle
|
||||
}
|
||||
|
||||
func TestShortBodyRecovers(t *testing.T) {
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(2 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 4)
|
||||
|
||||
if _, err := w.WriteChunk(0, bytes.NewReader(chunkOf(data, 0, chunkSize)[:100])); err == nil {
|
||||
t.Fatal("short body: expected error")
|
||||
}
|
||||
writeChunkOK(t, w, data, 0) // slot must have been recycled
|
||||
}
|
||||
|
||||
func TestOversizeBodyRejected(t *testing.T) {
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(2 * chunkSize)
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 4)
|
||||
|
||||
oversize := append(append([]byte{}, chunkOf(data, 0, chunkSize)...), 0xFF)
|
||||
if _, err := w.WriteChunk(0, bytes.NewReader(oversize)); err == nil {
|
||||
t.Fatal("oversize body: expected error")
|
||||
}
|
||||
writeChunkOK(t, w, data, 0)
|
||||
|
||||
// the short last chunk must also reject a full-size body
|
||||
last := w.TotalChunks() - 1
|
||||
if last == 0 {
|
||||
t.Fatal("test needs at least 2 chunks")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLastChunkShortStrict(t *testing.T) {
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(chunkSize + 100)
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 4)
|
||||
|
||||
if _, err := w.WriteChunk(1, bytes.NewReader(genData(chunkSize))); err == nil {
|
||||
t.Fatal("full-size body for short last chunk: expected error")
|
||||
}
|
||||
writeChunkOK(t, w, data, 1)
|
||||
writeChunkOK(t, w, data, 0)
|
||||
got := readAllWithin(t, w, 10*time.Second)
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Fatal("reassembled stream differs")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseUnblocksReader(t *testing.T) {
|
||||
w := newTestWindow(t, 1024, 4096, 4)
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
buf := make([]byte, 16)
|
||||
_, err := w.Read(buf)
|
||||
errCh <- err
|
||||
}()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
_ = w.Close()
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if !errors.Is(err, ErrClosed) {
|
||||
t.Fatalf("blocked Read after Close: err = %v, want ErrClosed", err)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Read still blocked after Close")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseWithErrorPropagatesSticky(t *testing.T) {
|
||||
w := newTestWindow(t, 1024, 4096, 4)
|
||||
cause := errors.New("aborted for a reason")
|
||||
errCh := make(chan error, 1)
|
||||
go func() {
|
||||
buf := make([]byte, 16)
|
||||
_, err := w.Read(buf)
|
||||
errCh <- err
|
||||
}()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
_ = w.CloseWithError(cause)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if !errors.Is(err, cause) {
|
||||
t.Fatalf("blocked Read after CloseWithError: err = %v, want %v", err, cause)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("Read still blocked after CloseWithError")
|
||||
}
|
||||
if _, err := w.WriteChunk(0, bytes.NewReader(make([]byte, 1024))); !errors.Is(err, cause) {
|
||||
t.Fatalf("WriteChunk after CloseWithError: err = %v, want %v", err, cause)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWriteAfterClose(t *testing.T) {
|
||||
w := newTestWindow(t, 1024, 4096, 4)
|
||||
_ = w.Close()
|
||||
if _, err := w.WriteChunk(0, bytes.NewReader(make([]byte, 1024))); !errors.Is(err, ErrClosed) {
|
||||
t.Fatalf("WriteChunk after Close: err = %v, want ErrClosed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloseUnblocksInFlightWriter(t *testing.T) {
|
||||
const chunkSize = 1024
|
||||
data := genData(2 * chunkSize)
|
||||
w := newTestWindow(t, chunkSize, 2*chunkSize, 4)
|
||||
|
||||
release := make(chan struct{})
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
_, err := w.WriteChunk(0, &gatedReader{release: release, inner: bytes.NewReader(chunkOf(data, 0, chunkSize))})
|
||||
done <- err
|
||||
}()
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
_ = w.Close()
|
||||
close(release)
|
||||
if err := <-done; !errors.Is(err, ErrClosed) {
|
||||
t.Fatalf("in-flight WriteChunk across Close: err = %v, want ErrClosed", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEOFExact(t *testing.T) {
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(chunkSize + 5)
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 4)
|
||||
writeChunkOK(t, w, data, 0)
|
||||
writeChunkOK(t, w, data, 1)
|
||||
|
||||
got := readAllWithin(t, w, 10*time.Second)
|
||||
if !bytes.Equal(got, data) {
|
||||
t.Fatal("reassembled stream differs")
|
||||
}
|
||||
buf := make([]byte, 1)
|
||||
if n, err := w.Read(buf); n != 0 || err != io.EOF {
|
||||
t.Fatalf("Read at EOF = (%d, %v), want (0, io.EOF)", n, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManyLapsSmallWindow(t *testing.T) {
|
||||
const chunkSize = 8 * 1024
|
||||
const chunks = 64
|
||||
totalSize := int64(chunks*chunkSize - 777)
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 3)
|
||||
|
||||
var next atomic.Int64
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 3; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for {
|
||||
idx := int(next.Add(1) - 1)
|
||||
if idx >= w.TotalChunks() {
|
||||
return
|
||||
}
|
||||
for {
|
||||
_, err := w.WriteChunk(idx, bytes.NewReader(chunkOf(data, idx, chunkSize)))
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if errors.Is(err, ErrOutOfWindow) || errors.Is(err, ErrChunkInFlight) {
|
||||
time.Sleep(200 * time.Microsecond)
|
||||
continue
|
||||
}
|
||||
t.Errorf("WriteChunk(%d): %v", idx, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
got := readAllWithin(t, w, 30*time.Second)
|
||||
wg.Wait()
|
||||
if wantSum, gotSum := sha256.Sum256(data), sha256.Sum256(got); wantSum != gotSum {
|
||||
t.Fatalf("reassembled stream differs: got %d bytes, want %d", len(got), len(data))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSnapshot(t *testing.T) {
|
||||
const chunkSize = 1024
|
||||
totalSize := int64(5*chunkSize + 512)
|
||||
data := genData(totalSize)
|
||||
w := newTestWindow(t, chunkSize, totalSize, 4)
|
||||
|
||||
writeChunkOK(t, w, data, 0)
|
||||
writeChunkOK(t, w, data, 1)
|
||||
writeChunkOK(t, w, data, 3)
|
||||
|
||||
snap := w.Snapshot()
|
||||
if snap.Frontier != 0 || snap.ReadPos != 0 {
|
||||
t.Fatalf("snapshot frontier/readPos = %d/%d, want 0/0", snap.Frontier, snap.ReadPos)
|
||||
}
|
||||
wantRanges := [][2]int{{0, 1}, {3, 3}}
|
||||
if len(snap.Received) != len(wantRanges) || snap.Received[0] != wantRanges[0] || snap.Received[1] != wantRanges[1] {
|
||||
t.Fatalf("snapshot received = %v, want %v", snap.Received, wantRanges)
|
||||
}
|
||||
if snap.ReceivedBytes != 3*chunkSize {
|
||||
t.Fatalf("snapshot receivedBytes = %d, want %d", snap.ReceivedBytes, 3*chunkSize)
|
||||
}
|
||||
|
||||
buf := make([]byte, chunkSize)
|
||||
if _, err := io.ReadFull(w, buf); err != nil {
|
||||
t.Fatalf("ReadFull: %v", err)
|
||||
}
|
||||
snap = w.Snapshot()
|
||||
if snap.Frontier != 1 || snap.ReadPos != chunkSize {
|
||||
t.Fatalf("snapshot frontier/readPos = %d/%d, want 1/%d", snap.Frontier, snap.ReadPos, chunkSize)
|
||||
}
|
||||
if len(snap.Received) != 2 || snap.Received[0] != [2]int{0, 1} || snap.Received[1] != [2]int{3, 3} {
|
||||
t.Fatalf("snapshot received = %v, want [[0,1],[3,3]]", snap.Received)
|
||||
}
|
||||
if snap.ReceivedBytes != 3*chunkSize {
|
||||
t.Fatalf("snapshot receivedBytes = %d, want %d", snap.ReceivedBytes, 3*chunkSize)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIndexOutOfRange(t *testing.T) {
|
||||
w := newTestWindow(t, 1024, 4096, 4)
|
||||
if _, err := w.WriteChunk(-1, bytes.NewReader(nil)); err == nil {
|
||||
t.Fatal("negative index: expected error")
|
||||
}
|
||||
if _, err := w.WriteChunk(4, bytes.NewReader(nil)); err == nil {
|
||||
t.Fatal("index == total: expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCRCsSurviveClose(t *testing.T) {
|
||||
const chunkSize = 1024
|
||||
data := genData(2 * chunkSize)
|
||||
w := newTestWindow(t, chunkSize, 2*chunkSize, 4)
|
||||
want := writeChunkOK(t, w, data, 0)
|
||||
_ = w.Close()
|
||||
crcs, set := w.CRCs()
|
||||
if !set[0] || crcs[0] != want {
|
||||
t.Fatalf("CRCs after Close = (%08x, %v), want (%08x, true)", crcs[0], set[0], want)
|
||||
}
|
||||
if set[1] {
|
||||
t.Fatal("chunk 1 crc should not be set")
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
_ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/123"
|
||||
_ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/123_open"
|
||||
_ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/aria2"
|
||||
_ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/guangyapan"
|
||||
_ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/http"
|
||||
_ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/pikpak"
|
||||
_ "github.com/OpenListTeam/OpenList/v4/internal/offline_download/qbit"
|
||||
|
||||
@@ -0,0 +1,132 @@
|
||||
package guangyapan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync/atomic"
|
||||
|
||||
guangyapandriver "github.com/OpenListTeam/OpenList/v4/drivers/guangyapan"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/offline_download/tool"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||
)
|
||||
|
||||
type GuangYaPan struct {
|
||||
refreshTaskCache atomic.Bool
|
||||
}
|
||||
|
||||
func (g *GuangYaPan) Name() string {
|
||||
return "GuangYaPan"
|
||||
}
|
||||
|
||||
func (g *GuangYaPan) Items() []model.SettingItem {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *GuangYaPan) Run(task *tool.DownloadTask) error {
|
||||
return errs.NotSupport
|
||||
}
|
||||
|
||||
func (g *GuangYaPan) Init() (string, error) {
|
||||
g.refreshTaskCache.Store(false)
|
||||
return "ok", nil
|
||||
}
|
||||
|
||||
func (g *GuangYaPan) IsReady() bool {
|
||||
tempDir := setting.GetStr(conf.GuangYaPanTempDir)
|
||||
if tempDir == "" {
|
||||
return false
|
||||
}
|
||||
storage, _, err := op.GetStorageAndActualPath(tempDir)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
if _, ok := storage.(*guangyapandriver.GuangYaPan); !ok {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (g *GuangYaPan) AddURL(args *tool.AddUrlArgs) (string, error) {
|
||||
g.refreshTaskCache.Store(true)
|
||||
storage, actualPath, err := op.GetStorageAndActualPath(args.TempDir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
driver, ok := storage.(*guangyapandriver.GuangYaPan)
|
||||
if !ok {
|
||||
return "", errors.New("GuangYaPan offline download only supports GuangYaPan destination storage")
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
if err := op.MakeDir(ctx, storage, actualPath); err != nil {
|
||||
return "", err
|
||||
}
|
||||
parentDir, err := op.GetUnwrap(ctx, storage, actualPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
task, err := driver.OfflineDownload(ctx, args.Url, parentDir, "")
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to add offline download task: %w", err)
|
||||
}
|
||||
return task.TaskID, nil
|
||||
}
|
||||
|
||||
func (g *GuangYaPan) Remove(task *tool.DownloadTask) error {
|
||||
storage, _, err := op.GetStorageAndActualPath(task.TempDir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
driver, ok := storage.(*guangyapandriver.GuangYaPan)
|
||||
if !ok {
|
||||
return errors.New("GuangYaPan offline download only supports GuangYaPan destination storage")
|
||||
}
|
||||
ctx := context.Background()
|
||||
if err := driver.DeleteOfflineTasks(ctx, []string{task.GID}); err != nil {
|
||||
return err
|
||||
}
|
||||
g.DelTaskCache(driver, task.GID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *GuangYaPan) Status(task *tool.DownloadTask) (*tool.Status, error) {
|
||||
storage, _, err := op.GetStorageAndActualPath(task.TempDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
driver, ok := storage.(*guangyapandriver.GuangYaPan)
|
||||
if !ok {
|
||||
return nil, errors.New("GuangYaPan offline download only supports GuangYaPan destination storage")
|
||||
}
|
||||
tasks, err := g.GetTasks(driver, task.GID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
status := &tool.Status{
|
||||
Status: "the task has been deleted",
|
||||
}
|
||||
for _, t := range tasks {
|
||||
if t.TaskID != task.GID {
|
||||
continue
|
||||
}
|
||||
status.Progress = float64(t.Progress)
|
||||
status.TotalBytes = t.TotalSize
|
||||
status.Completed = t.Status == offlineStatusCompleted || t.Status == offlineStatusPartiallyCompleted
|
||||
status.Status = taskStatusText(t)
|
||||
if t.Status == offlineStatusFailed || t.Status == offlineStatusCanceled {
|
||||
status.Err = errors.New(status.Status)
|
||||
}
|
||||
return status, nil
|
||||
}
|
||||
status.Err = errors.New("the task has been deleted")
|
||||
return status, nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
tool.Tools.Add(&GuangYaPan{})
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package guangyapan
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
guangyapandriver "github.com/OpenListTeam/OpenList/v4/drivers/guangyapan"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/singleflight"
|
||||
"github.com/OpenListTeam/go-cache"
|
||||
)
|
||||
|
||||
const (
|
||||
offlineStatusQueued = 0
|
||||
offlineStatusRunning = 1
|
||||
offlineStatusCompleted = 2
|
||||
offlineStatusFailed = 3
|
||||
offlineStatusCanceled = 4
|
||||
offlineStatusPartiallyCompleted = 5
|
||||
)
|
||||
|
||||
var taskCache = cache.NewMemCache(cache.WithShards[[]guangyapandriver.OfflineTask](16))
|
||||
var taskG singleflight.Group[[]guangyapandriver.OfflineTask]
|
||||
|
||||
func (g *GuangYaPan) GetTasks(driver *guangyapandriver.GuangYaPan, taskID string) ([]guangyapandriver.OfflineTask, error) {
|
||||
key := op.Key(driver, "/cloudcollection/v1/list_task/"+taskID)
|
||||
if !g.refreshTaskCache.Load() {
|
||||
if tasks, ok := taskCache.Get(key); ok {
|
||||
return tasks, nil
|
||||
}
|
||||
}
|
||||
g.refreshTaskCache.Store(false)
|
||||
tasks, err, _ := taskG.Do(key, func() ([]guangyapandriver.OfflineTask, error) {
|
||||
ctx := context.Background()
|
||||
tasks, err := driver.OfflineList(ctx, []string{taskID}, nil, "", 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(tasks) > 0 {
|
||||
taskCache.Set(key, tasks, cache.WithEx[[]guangyapandriver.OfflineTask](time.Second*10))
|
||||
} else {
|
||||
taskCache.Del(key)
|
||||
}
|
||||
return tasks, nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return tasks, nil
|
||||
}
|
||||
|
||||
func (g *GuangYaPan) DelTaskCache(driver *guangyapandriver.GuangYaPan, taskID string) {
|
||||
taskCache.Del(op.Key(driver, "/cloudcollection/v1/list_task/"+taskID))
|
||||
}
|
||||
|
||||
func taskStatusText(task guangyapandriver.OfflineTask) string {
|
||||
switch task.Status {
|
||||
case offlineStatusQueued:
|
||||
return "queued"
|
||||
case offlineStatusRunning:
|
||||
if task.Progress > 0 {
|
||||
return fmt.Sprintf("running (%d%%)", task.Progress)
|
||||
}
|
||||
return "running"
|
||||
case offlineStatusCompleted:
|
||||
return "completed"
|
||||
case offlineStatusFailed:
|
||||
return "failed"
|
||||
case offlineStatusCanceled:
|
||||
return "canceled"
|
||||
case offlineStatusPartiallyCompleted:
|
||||
return "partially completed"
|
||||
default:
|
||||
return fmt.Sprintf("unknown status %d", task.Status)
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
_115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open"
|
||||
_123 "github.com/OpenListTeam/OpenList/v4/drivers/123"
|
||||
_123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open"
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/guangyapan"
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/pikpak"
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/thunder"
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser"
|
||||
@@ -162,6 +163,16 @@ func AddURL(ctx context.Context, args *AddURLArgs) (task.TaskExtensionInfo, erro
|
||||
} else {
|
||||
tempDir = filepath.Join(setting.GetStr(conf.ThunderXTempDir), uid)
|
||||
}
|
||||
case "GuangYaPan":
|
||||
if _, ok := storage.(*guangyapan.GuangYaPan); ok {
|
||||
tempDir = args.DstDirPath
|
||||
} else {
|
||||
tempBase := setting.GetStr(conf.GuangYaPanTempDir)
|
||||
if tempBase == "" {
|
||||
return nil, errors.New("GuangYaPan temp dir is not set")
|
||||
}
|
||||
tempDir = filepath.Join(tempBase, uid)
|
||||
}
|
||||
}
|
||||
|
||||
taskCreator, _ := ctx.Value(conf.UserKey).(*model.User) // taskCreator is nil when convert failed
|
||||
|
||||
@@ -98,6 +98,9 @@ outer:
|
||||
if t.tool.Name() == "ThunderX" {
|
||||
return nil
|
||||
}
|
||||
if t.tool.Name() == "GuangYaPan" {
|
||||
return nil
|
||||
}
|
||||
if t.tool.Name() == "115 Cloud" {
|
||||
// hack for 115
|
||||
<-time.After(time.Second * 1)
|
||||
@@ -176,7 +179,7 @@ func (t *DownloadTask) Update() (bool, error) {
|
||||
|
||||
func (t *DownloadTask) Transfer() error {
|
||||
toolName := t.tool.Name()
|
||||
if toolName == "115 Cloud" || toolName == "115 Open" || toolName == "123 Open" || toolName == "123Pan" || toolName == "PikPak" || toolName == "Thunder" || toolName == "ThunderX" || toolName == "ThunderBrowser" {
|
||||
if toolName == "115 Cloud" || toolName == "115 Open" || toolName == "123 Open" || toolName == "123Pan" || toolName == "PikPak" || toolName == "Thunder" || toolName == "ThunderX" || toolName == "ThunderBrowser" || toolName == "GuangYaPan" {
|
||||
// 如果不是直接下载到目标路径,则进行转存
|
||||
if t.TempDir != t.DstDirPath {
|
||||
return transferObj(t.Ctx(), t.TempDir, t.DstDirPath, t.DeletePolicy)
|
||||
|
||||
@@ -2,7 +2,9 @@ package op
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
@@ -83,6 +85,17 @@ var settingItemHooks = map[string]SettingItemHook{
|
||||
conf.SlicesMap[conf.IgnoreDirectLinkParams] = strings.Split(item.Value, ",")
|
||||
return nil
|
||||
},
|
||||
conf.MultipartChunkSize: func(item *model.SettingItem) error {
|
||||
size, err := strconv.Atoi(strings.TrimSpace(item.Value))
|
||||
if err != nil || size < 1 {
|
||||
// deliberately a plain error: SaveSettings formats hook errors
|
||||
// with %+v, which would dump a full stack trace into the UI
|
||||
// notification for stack-carrying errors
|
||||
return fmt.Errorf("multipart chunk size must be a positive integer (MB), got %q", item.Value)
|
||||
}
|
||||
item.Value = strconv.Itoa(size)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
func RegisterSettingItemHook(key string, hook SettingItemHook) {
|
||||
|
||||
@@ -519,7 +519,7 @@ func (r *RangeReadReadAtSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
default:
|
||||
return 0, errors.New("Seek: invalid whence")
|
||||
}
|
||||
if offset < 0 || offset > r.ss.GetSize() {
|
||||
if offset < 0 {
|
||||
return 0, errors.New("Seek: invalid offset")
|
||||
}
|
||||
r.masterOff = offset
|
||||
|
||||
+1
-1
@@ -79,7 +79,7 @@ func (r *DynamicReadAtSeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
return 0, errors.New("Seek: invalid whence")
|
||||
}
|
||||
|
||||
if offset < 0 || offset > r.block.Size() {
|
||||
if offset < 0 {
|
||||
return 0, errors.New("Seek: invalid offset")
|
||||
}
|
||||
r.offset = offset
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
package handles
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/url"
|
||||
stdpath "path"
|
||||
"strconv"
|
||||
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/errs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/fs"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/model"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/multipart"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/op"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/setting"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
"github.com/OpenListTeam/OpenList/v4/server/common"
|
||||
"github.com/gin-gonic/gin"
|
||||
)
|
||||
|
||||
const multipartMinChunkSize = int64(1) << 20 // 1MB
|
||||
|
||||
// multipartChunkSize resolves the effective chunk size. The admin setting is
|
||||
// the ceiling: a client may suggest a smaller chunk via X-Chunk-Size but never
|
||||
// a larger one — the server buffers a window of several chunks per session, so
|
||||
// an unbounded client suggestion would translate directly into server-side
|
||||
// disk usage.
|
||||
func multipartChunkSize(requested int64) int64 {
|
||||
ceiling := int64(setting.GetInt(conf.MultipartChunkSize, 10)) << 20
|
||||
if ceiling < multipartMinChunkSize {
|
||||
ceiling = multipartMinChunkSize
|
||||
}
|
||||
size := ceiling
|
||||
if requested > 0 && requested < ceiling {
|
||||
size = max(requested, multipartMinChunkSize)
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
type MultipartInitResp struct {
|
||||
multipart.SessionSnapshot
|
||||
Resumed bool `json:"resumed"`
|
||||
}
|
||||
|
||||
// MultipartInit creates (or resumes) a multipart upload session and starts its
|
||||
// upload pipeline. Headers mirror FsStream (fsup.go).
|
||||
func MultipartInit(c *gin.Context) {
|
||||
if !setting.GetBool(conf.MultipartEnabled) {
|
||||
common.ErrorStrResp(c, "multipart upload is disabled", 403)
|
||||
return
|
||||
}
|
||||
path := c.GetHeader("File-Path")
|
||||
path, err := url.PathUnescape(path)
|
||||
if err != nil {
|
||||
common.ErrorResp(c, err, 400)
|
||||
return
|
||||
}
|
||||
user := c.Request.Context().Value(conf.UserKey).(*model.User)
|
||||
path, err = user.JoinPath(path)
|
||||
if err != nil {
|
||||
common.ErrorResp(c, err, 403)
|
||||
return
|
||||
}
|
||||
size, err := strconv.ParseInt(c.GetHeader("X-File-Size"), 10, 64)
|
||||
if err != nil {
|
||||
common.ErrorStrResp(c, "multipart upload requires a valid X-File-Size header", 400)
|
||||
return
|
||||
}
|
||||
if size <= 0 {
|
||||
common.ErrorStrResp(c, "multipart upload requires a positive X-File-Size; upload empty files via /fs/put", 400)
|
||||
return
|
||||
}
|
||||
var requestedChunkSize int64
|
||||
if v := c.GetHeader("X-Chunk-Size"); v != "" {
|
||||
requestedChunkSize, err = strconv.ParseInt(v, 10, 64)
|
||||
if err != nil {
|
||||
common.ErrorResp(c, err, 400)
|
||||
return
|
||||
}
|
||||
}
|
||||
overwrite := c.GetHeader("Overwrite") != "false"
|
||||
if !overwrite {
|
||||
if res, _ := fs.Get(c.Request.Context(), path, &fs.GetArgs{NoLog: true}); res != nil {
|
||||
common.ErrorStrResp(c, "file exists", 403)
|
||||
return
|
||||
}
|
||||
}
|
||||
dir, name := stdpath.Split(path)
|
||||
if shouldIgnoreSystemFile(name) {
|
||||
common.ErrorStrResp(c, errs.IgnoredSystemFile.Error(), 403)
|
||||
return
|
||||
}
|
||||
// fail fast on unusable destinations instead of letting the pipeline discover it
|
||||
storage, _, err := op.GetStorageAndActualPath(dir)
|
||||
if err != nil {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
if storage.Config().NoUpload {
|
||||
common.ErrorResp(c, errs.UploadNotSupported, 405)
|
||||
return
|
||||
}
|
||||
h := make(map[*utils.HashType]string)
|
||||
if md5 := c.GetHeader("X-File-Md5"); md5 != "" {
|
||||
h[utils.MD5] = md5
|
||||
}
|
||||
if sha1 := c.GetHeader("X-File-Sha1"); sha1 != "" {
|
||||
h[utils.SHA1] = sha1
|
||||
}
|
||||
if sha256 := c.GetHeader("X-File-Sha256"); sha256 != "" {
|
||||
h[utils.SHA256] = sha256
|
||||
}
|
||||
mimetype := c.GetHeader("Content-Type")
|
||||
if len(mimetype) == 0 {
|
||||
mimetype = utils.GetMimeType(name)
|
||||
}
|
||||
snap, resumed, err := multipart.DefaultManager.Init(multipart.InitReq{
|
||||
User: user,
|
||||
Path: path,
|
||||
Size: size,
|
||||
ChunkSize: multipartChunkSize(requestedChunkSize),
|
||||
Mimetype: mimetype,
|
||||
Modified: getLastModified(c),
|
||||
Hashes: h,
|
||||
})
|
||||
if err != nil {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
common.SuccessResp(c, MultipartInitResp{SessionSnapshot: snap, Resumed: resumed})
|
||||
}
|
||||
|
||||
// MultipartChunk ingests one chunk. Chunks are idempotent and may be sent
|
||||
// concurrently and out of order within the receiving window.
|
||||
// code 429 = window full (flow control, retry after a short delay),
|
||||
// code 409 = the same chunk is already in flight on another connection.
|
||||
func MultipartChunk(c *gin.Context) {
|
||||
user := c.Request.Context().Value(conf.UserKey).(*model.User)
|
||||
id := c.GetHeader("X-Upload-Id")
|
||||
idx, err := strconv.Atoi(c.GetHeader("X-Chunk-Index"))
|
||||
if err != nil {
|
||||
common.ErrorStrResp(c, "invalid X-Chunk-Index header", 400)
|
||||
return
|
||||
}
|
||||
snap, err := multipart.DefaultManager.Chunk(user, id, idx, c.Request.Body)
|
||||
// Answer only after the request body is consumed — on EVERY path. Flow
|
||||
// control (429), absorbed chunks and validation errors would otherwise
|
||||
// respond while the browser is still streaming the body, which it reports
|
||||
// as a network error and which poisons its connection pool. A rejected
|
||||
// chunk gets resent anyway, so draining costs no extra round trip. The
|
||||
// drain is bounded so a malformed request cannot pin the handler.
|
||||
limit := multipartChunkSize(0) + 64*1024
|
||||
if snap.ChunkSize > 0 {
|
||||
limit = snap.ChunkSize + 64*1024
|
||||
}
|
||||
_, _ = utils.CopyWithBuffer(io.Discard, io.LimitReader(c.Request.Body, limit))
|
||||
if err != nil {
|
||||
common.ErrorWithDataResp(c, err, multipartErrCode(err), snap)
|
||||
return
|
||||
}
|
||||
common.SuccessResp(c, snap)
|
||||
}
|
||||
|
||||
// MultipartComplete waits for the pipeline outcome and reports it, mirroring
|
||||
// how /fs/put only responds once the driver upload finished.
|
||||
func MultipartComplete(c *gin.Context) {
|
||||
user := c.Request.Context().Value(conf.UserKey).(*model.User)
|
||||
id := c.GetHeader("X-Upload-Id")
|
||||
snap, err := multipart.DefaultManager.Complete(c.Request.Context(), user, id)
|
||||
if err != nil {
|
||||
common.ErrorWithDataResp(c, err, multipartErrCode(err), snap)
|
||||
return
|
||||
}
|
||||
common.SuccessResp(c, snap)
|
||||
}
|
||||
|
||||
// MultipartStatus looks a session up by upload_id, or by path+size so an
|
||||
// interrupted client can discover a resumable session.
|
||||
func MultipartStatus(c *gin.Context) {
|
||||
user := c.Request.Context().Value(conf.UserKey).(*model.User)
|
||||
if id := c.Query("upload_id"); id != "" {
|
||||
snap, err := multipart.DefaultManager.Status(user, id)
|
||||
if err != nil {
|
||||
common.ErrorResp(c, err, multipartErrCode(err))
|
||||
return
|
||||
}
|
||||
common.SuccessResp(c, snap)
|
||||
return
|
||||
}
|
||||
path, err := user.JoinPath(c.Query("path"))
|
||||
if err != nil {
|
||||
common.ErrorResp(c, err, 403)
|
||||
return
|
||||
}
|
||||
size, err := strconv.ParseInt(c.Query("size"), 10, 64)
|
||||
if err != nil {
|
||||
common.ErrorStrResp(c, "status lookup requires upload_id, or path and size", 400)
|
||||
return
|
||||
}
|
||||
snap, err := multipart.DefaultManager.Find(user, path, size)
|
||||
if err != nil {
|
||||
common.ErrorResp(c, err, multipartErrCode(err))
|
||||
return
|
||||
}
|
||||
common.SuccessResp(c, snap)
|
||||
}
|
||||
|
||||
// MultipartAbort cancels the pipeline and discards the session.
|
||||
func MultipartAbort(c *gin.Context) {
|
||||
user := c.Request.Context().Value(conf.UserKey).(*model.User)
|
||||
if err := multipart.DefaultManager.Abort(user, c.GetHeader("X-Upload-Id")); err != nil {
|
||||
common.ErrorResp(c, err, multipartErrCode(err))
|
||||
return
|
||||
}
|
||||
common.SuccessResp(c)
|
||||
}
|
||||
|
||||
func multipartErrCode(err error) int {
|
||||
switch {
|
||||
case errors.Is(err, multipart.ErrOutOfWindow):
|
||||
return 429
|
||||
case errors.Is(err, multipart.ErrChunkInFlight):
|
||||
return 409
|
||||
case errors.Is(err, multipart.ErrSessionNotFound):
|
||||
return 404
|
||||
case errors.Is(err, multipart.ErrNotOwner):
|
||||
return 403
|
||||
default:
|
||||
return 400
|
||||
}
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
_115_open "github.com/OpenListTeam/OpenList/v4/drivers/115_open"
|
||||
_123 "github.com/OpenListTeam/OpenList/v4/drivers/123"
|
||||
_123_open "github.com/OpenListTeam/OpenList/v4/drivers/123_open"
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/guangyapan"
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/pikpak"
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/thunder"
|
||||
"github.com/OpenListTeam/OpenList/v4/drivers/thunder_browser"
|
||||
@@ -472,6 +473,50 @@ func SetThunderBrowser(c *gin.Context) {
|
||||
common.SuccessResp(c, "ok")
|
||||
}
|
||||
|
||||
type SetGuangYaPanReq struct {
|
||||
TempDir string `json:"temp_dir" form:"temp_dir"`
|
||||
}
|
||||
|
||||
func SetGuangYaPan(c *gin.Context) {
|
||||
var req SetGuangYaPanReq
|
||||
if err := c.ShouldBind(&req); err != nil {
|
||||
common.ErrorResp(c, err, 400)
|
||||
return
|
||||
}
|
||||
if req.TempDir != "" {
|
||||
storage, _, err := op.GetStorageAndActualPath(req.TempDir)
|
||||
if err != nil {
|
||||
common.ErrorStrResp(c, "storage does not exists", 400)
|
||||
return
|
||||
}
|
||||
if storage.Config().CheckStatus && storage.GetStorage().Status != op.WORK {
|
||||
common.ErrorStrResp(c, "storage not init: "+storage.GetStorage().Status, 400)
|
||||
return
|
||||
}
|
||||
if _, ok := storage.(*guangyapan.GuangYaPan); !ok {
|
||||
common.ErrorStrResp(c, "unsupported storage driver for offline download, only GuangYaPan is supported", 400)
|
||||
return
|
||||
}
|
||||
}
|
||||
items := []model.SettingItem{
|
||||
{Key: conf.GuangYaPanTempDir, Value: req.TempDir, Type: conf.TypeString, Group: model.OFFLINE_DOWNLOAD, Flag: model.PRIVATE},
|
||||
}
|
||||
if err := op.SaveSettingItems(items); err != nil {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
_tool, err := tool.Tools.Get("GuangYaPan")
|
||||
if err != nil {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
if _, err := _tool.Init(); err != nil {
|
||||
common.ErrorResp(c, err, 500)
|
||||
return
|
||||
}
|
||||
common.SuccessResp(c, "ok")
|
||||
}
|
||||
|
||||
func OfflineDownloadTools(c *gin.Context) {
|
||||
tools := tool.Tools.Names()
|
||||
common.SuccessResp(c, tools)
|
||||
|
||||
@@ -95,7 +95,7 @@ func FinishAuthnLogin(c *gin.Context) {
|
||||
} else { // client-side discoverable login
|
||||
_, err = authnInstance.FinishDiscoverableLogin(func(_, userHandle []byte) (webauthn.User, error) {
|
||||
// first param `rawID` in this callback function is equal to ID in webauthn.Credential,
|
||||
// but it's unnnecessary to check it.
|
||||
// but it's unnecessary to check it.
|
||||
// userHandle param is equal to (User).WebAuthnID().
|
||||
userID := uint(binary.LittleEndian.Uint64(userHandle))
|
||||
user, err = db.GetUserById(userID)
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"github.com/OpenListTeam/OpenList/v4/cmd/flags"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/conf"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/message"
|
||||
multipartPkg "github.com/OpenListTeam/OpenList/v4/internal/multipart"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/sign"
|
||||
"github.com/OpenListTeam/OpenList/v4/internal/stream"
|
||||
"github.com/OpenListTeam/OpenList/v4/pkg/utils"
|
||||
@@ -167,6 +168,7 @@ func admin(g *gin.RouterGroup) {
|
||||
setting.POST("/set_thunder", handles.SetThunder)
|
||||
setting.POST("/set_thunderx", handles.SetThunderX)
|
||||
setting.POST("/set_thunder_browser", handles.SetThunderBrowser)
|
||||
setting.POST("/set_guangyapan", handles.SetGuangYaPan)
|
||||
|
||||
// retain /admin/task API to ensure compatibility with legacy automation scripts
|
||||
_task(g.Group("/task"))
|
||||
@@ -212,6 +214,13 @@ func _fs(g *gin.RouterGroup) {
|
||||
uploadLimiter := middlewares.UploadRateLimiter(stream.ClientUploadLimit)
|
||||
g.PUT("/put", middlewares.FsUp, uploadLimiter, handles.FsStream)
|
||||
g.PUT("/form", middlewares.FsUp, uploadLimiter, handles.FsForm)
|
||||
multipartPkg.DefaultManager.StartGC() // reclaim ring files orphaned by a previous run
|
||||
multipart := g.Group("/multipart")
|
||||
multipart.POST("/init", middlewares.FsUp, handles.MultipartInit)
|
||||
multipart.PUT("/chunk", uploadLimiter, handles.MultipartChunk)
|
||||
multipart.POST("/complete", handles.MultipartComplete)
|
||||
multipart.GET("/status", handles.MultipartStatus)
|
||||
multipart.POST("/abort", handles.MultipartAbort)
|
||||
g.POST("/link", middlewares.AuthAdmin, handles.Link)
|
||||
// g.POST("/add_aria2", handles.AddOfflineDownload)
|
||||
// g.POST("/add_qbit", handles.AddQbittorrent)
|
||||
|
||||
+1
-1
@@ -8,7 +8,7 @@ import (
|
||||
"github.com/itsHenry35/gofakes3"
|
||||
)
|
||||
|
||||
// pager splits the object list into smulitply pages.
|
||||
// pager splits the object list into multiple pages.
|
||||
func (db *s3Backend) pager(list *gofakes3.ObjectList, page gofakes3.ListBucketPage) (*gofakes3.ObjectList, error) {
|
||||
// sort by alphabet
|
||||
sort.Slice(list.CommonPrefixes, func(i, j int) bool {
|
||||
|
||||
Reference in New Issue
Block a user