fix(archive/zip): user specific encoding for non-EFS zips (#1599)

* fix(archive/zip): user specific encoding for non-EFS zips

* fix(stream): simplify head cache initialization and improve reader retrieval logic

* fix: support multipart zips (.z01)

* chore(deps): update github.com/KirCute/zip to v1.0.1

---------

Co-authored-by: j2rong4cn <j2rong@qq.com>
Co-authored-by: Pikachu Ren <40362270+PIKACHUIM@users.noreply.github.com>

cmd/flags/config.go

@@ -2,6 +2,7 @@ package flags
 
 var (
 	DataDir     string
+	ConfigPath  string
 	Debug       bool
 	NoPrefix    bool
 	Dev         bool

cmd/root.go

@@ -27,7 +27,8 @@ func Execute() {
 }
 
 func init() {
-	RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data folder")
+	RootCmd.PersistentFlags().StringVar(&flags.DataDir, "data", "data", "data directory (relative paths are resolved against the current working directory)")
+	RootCmd.PersistentFlags().StringVar(&flags.ConfigPath, "config", "", "path to config.json (relative to current working directory; defaults to [data directory]/config.json, where [data directory] is set by --data)")
 	RootCmd.PersistentFlags().BoolVar(&flags.Debug, "debug", false, "start with debug mode")
 	RootCmd.PersistentFlags().BoolVar(&flags.NoPrefix, "no-prefix", false, "disable env prefix")
 	RootCmd.PersistentFlags().BoolVar(&flags.Dev, "dev", false, "start with dev mode")

drivers/115/meta.go

@@ -17,7 +17,7 @@ type Addition struct {
 var config = driver.Config{
 	Name:          "115 Cloud",
 	DefaultRoot:   "0",
-	LinkCacheType: 2,
+	LinkCacheMode: driver.LinkCacheUA,
 }
 
 func init() {

drivers/115_open/meta.go

@@ -19,7 +19,7 @@ type Addition struct {
 var config = driver.Config{
 	Name:          "115 Open",
 	DefaultRoot:   "0",
-	LinkCacheType: 2,
+	LinkCacheMode: driver.LinkCacheUA,
 }
 
 func init() {

drivers/123/driver.go

@@ -41,7 +41,9 @@ func (d *Pan123) GetAddition() driver.Additional {
 }
 
 func (d *Pan123) Init(ctx context.Context) error {
-	_, err := d.Request(UserInfo, http.MethodGet, nil, nil)
+	_, err := d.Request(UserInfo, http.MethodGet, func(req *resty.Request) {
+		req.SetHeader("platform", "web")
+	}, nil)
 	return err
 }
 

drivers/123/meta.go

@@ -12,7 +12,8 @@ type Addition struct {
 	//OrderBy        string `json:"order_by" type:"select" options:"file_id,file_name,size,update_at" default:"file_name"`
 	//OrderDirection string `json:"order_direction" type:"select" options:"asc,desc" default:"asc"`
 	AccessToken  string
-	UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"`
+	UploadThread int    `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"`
+	Platform     string `json:"platform" type:"string" default:"web" help:"the platform header value, sent with API requests"`
 }
 
 var config = driver.Config{

drivers/123/util.go

@@ -203,7 +203,7 @@ do:
 		"referer":       "https://www.123pan.com/",
 		"authorization": "Bearer " + d.AccessToken,
 		"user-agent":    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) openlist-client",
-		"platform":      "web",
+		"platform":      d.Platform,
 		"app-version":   "3",
 		//"user-agent":    base.UserAgent,
 	})

drivers/189/driver.go

@@ -200,10 +200,7 @@ func (d *Cloud189) GetDetails(ctx context.Context) (*model.StorageDetails, error
 		return nil, err
 	}
 	return &model.StorageDetails{
-		DiskUsage: model.DiskUsage{
-			TotalSpace: capacityInfo.CloudCapacityInfo.TotalSize,
-			FreeSpace:  capacityInfo.CloudCapacityInfo.FreeSize,
-		},
+		DiskUsage: driver.DiskUsageFromUsedAndTotal(capacityInfo.CloudCapacityInfo.UsedSize, capacityInfo.CloudCapacityInfo.TotalSize),
 	}, nil
 }
 

drivers/189/types.go

@@ -72,13 +72,13 @@ type CapacityResp struct {
 	ResMessage        string `json:"res_message"`
 	Account           string `json:"account"`
 	CloudCapacityInfo struct {
-		FreeSize     uint64 `json:"freeSize"`
+		FreeSize     int64  `json:"freeSize"`
 		MailUsedSize uint64 `json:"mail189UsedSize"`
 		TotalSize    uint64 `json:"totalSize"`
 		UsedSize     uint64 `json:"usedSize"`
 	} `json:"cloudCapacityInfo"`
 	FamilyCapacityInfo struct {
-		FreeSize  uint64 `json:"freeSize"`
+		FreeSize  int64  `json:"freeSize"`
 		TotalSize uint64 `json:"totalSize"`
 		UsedSize  uint64 `json:"usedSize"`
 	} `json:"familyCapacityInfo"`

drivers/189_tv/driver.go

@@ -284,18 +284,15 @@ func (y *Cloud189TV) GetDetails(ctx context.Context) (*model.StorageDetails, err
 	if err != nil {
 		return nil, err
 	}
-	var total, free uint64
+	var total, used uint64
 	if y.isFamily() {
 		total = capacityInfo.FamilyCapacityInfo.TotalSize
-		free = capacityInfo.FamilyCapacityInfo.FreeSize
+		used = capacityInfo.FamilyCapacityInfo.UsedSize
 	} else {
 		total = capacityInfo.CloudCapacityInfo.TotalSize
-		free = capacityInfo.CloudCapacityInfo.FreeSize
+		used = capacityInfo.CloudCapacityInfo.UsedSize
 	}
 	return &model.StorageDetails{
-		DiskUsage: model.DiskUsage{
-			TotalSpace: total,
-			FreeSpace:  free,
-		},
+		DiskUsage: driver.DiskUsageFromUsedAndTotal(used, total),
 	}, nil
 }

drivers/189_tv/types.go

@@ -322,13 +322,13 @@ type CapacityResp struct {
 	ResMessage        string `json:"res_message"`
 	Account           string `json:"account"`
 	CloudCapacityInfo struct {
-		FreeSize     uint64 `json:"freeSize"`
+		FreeSize     int64  `json:"freeSize"`
 		MailUsedSize uint64 `json:"mail189UsedSize"`
 		TotalSize    uint64 `json:"totalSize"`
 		UsedSize     uint64 `json:"usedSize"`
 	} `json:"cloudCapacityInfo"`
 	FamilyCapacityInfo struct {
-		FreeSize  uint64 `json:"freeSize"`
+		FreeSize  int64  `json:"freeSize"`
 		TotalSize uint64 `json:"totalSize"`
 		UsedSize  uint64 `json:"usedSize"`
 	} `json:"familyCapacityInfo"`

drivers/189pc/driver.go

@@ -416,18 +416,15 @@ func (y *Cloud189PC) GetDetails(ctx context.Context) (*model.StorageDetails, err
 	if err != nil {
 		return nil, err
 	}
-	var total, free uint64
+	var total, used uint64
 	if y.isFamily() {
 		total = capacityInfo.FamilyCapacityInfo.TotalSize
-		free = capacityInfo.FamilyCapacityInfo.FreeSize
+		used = capacityInfo.FamilyCapacityInfo.UsedSize
 	} else {
 		total = capacityInfo.CloudCapacityInfo.TotalSize
-		free = capacityInfo.CloudCapacityInfo.FreeSize
+		used = capacityInfo.CloudCapacityInfo.UsedSize
 	}
 	return &model.StorageDetails{
-		DiskUsage: model.DiskUsage{
-			TotalSpace: total,
-			FreeSpace:  free,
-		},
+		DiskUsage: driver.DiskUsageFromUsedAndTotal(used, total),
 	}, nil
 }

drivers/189pc/types.go

@@ -415,13 +415,13 @@ type CapacityResp struct {
 	ResMessage        string `json:"res_message"`
 	Account           string `json:"account"`
 	CloudCapacityInfo struct {
-		FreeSize     uint64 `json:"freeSize"`
+		FreeSize     int64  `json:"freeSize"`
 		MailUsedSize uint64 `json:"mail189UsedSize"`
 		TotalSize    uint64 `json:"totalSize"`
 		UsedSize     uint64 `json:"usedSize"`
 	} `json:"cloudCapacityInfo"`
 	FamilyCapacityInfo struct {
-		FreeSize  uint64 `json:"freeSize"`
+		FreeSize  int64  `json:"freeSize"`
 		TotalSize uint64 `json:"totalSize"`
 		UsedSize  uint64 `json:"usedSize"`
 	} `json:"familyCapacityInfo"`

drivers/alias/driver.go

@@ -524,4 +524,25 @@ func (d *Alias) ArchiveDecompress(ctx context.Context, srcObj, dstDir model.Obj,
 	}
 }
 
+func (d *Alias) ResolveLinkCacheMode(path string) driver.LinkCacheMode {
+	root, sub := d.getRootAndPath(path)
+	dsts, ok := d.pathMap[root]
+	if !ok {
+		return 0
+	}
+	for _, dst := range dsts {
+		storage, actualPath, err := op.GetStorageAndActualPath(stdpath.Join(dst, sub))
+		if err != nil {
+			continue
+		}
+		mode := storage.Config().LinkCacheMode
+		if mode == -1 {
+			return storage.(driver.LinkCacheModeResolver).ResolveLinkCacheMode(actualPath)
+		} else {
+			return mode
+		}
+	}
+	return 0
+}
+
 var _ driver.Driver = (*Alias)(nil)

drivers/alias/meta.go

@@ -26,6 +26,7 @@ var config = driver.Config{
 	NoUpload:         false,
 	DefaultRoot:      "/",
 	ProxyRangeOption: true,
+	LinkCacheMode:    driver.LinkCacheAuto,
 }
 
 func init() {

drivers/aliyundrive_open/driver.go

@@ -299,10 +299,7 @@ func (d *AliyundriveOpen) GetDetails(ctx context.Context) (*model.StorageDetails
 	total := utils.Json.Get(res, "personal_space_info", "total_size").ToUint64()
 	used := utils.Json.Get(res, "personal_space_info", "used_size").ToUint64()
 	return &model.StorageDetails{
-		DiskUsage: model.DiskUsage{
-			TotalSpace: total,
-			FreeSpace:  total - used,
-		},
+		DiskUsage: driver.DiskUsageFromUsedAndTotal(used, total),
 	}, nil
 }
 

drivers/baidu_netdisk/meta.go

@@ -3,6 +3,7 @@ package baidu_netdisk
 import (
 	"github.com/OpenListTeam/OpenList/v4/internal/driver"
 	"github.com/OpenListTeam/OpenList/v4/internal/op"
+	"time"
 )
 
 type Addition struct {
@@ -19,11 +20,21 @@ type Addition struct {
 	RefreshToken          string `json:"refresh_token" required:"true"`
 	UploadThread          string `json:"upload_thread" default:"3" help:"1<=thread<=32"`
 	UploadAPI             string `json:"upload_api" default:"https://d.pcs.baidu.com"`
+	UseDynamicUploadAPI   bool   `json:"use_dynamic_upload_api" default:"true" help:"dynamically get upload api domain, when enabled, the 'Upload API' setting will be used as a fallback if failed to get"`
 	CustomUploadPartSize  int64  `json:"custom_upload_part_size" type:"number" default:"0" help:"0 for auto"`
 	LowBandwithUploadMode bool   `json:"low_bandwith_upload_mode" default:"false"`
 	OnlyListVideoFile     bool   `json:"only_list_video_file" default:"false"`
 }
 
+const (
+	UPLOAD_FALLBACK_API        = "https://d.pcs.baidu.com" // 备用上传地址
+	UPLOAD_URL_EXPIRE_TIME     = time.Minute * 60          // 上传地址有效期(分钟)
+	UPLOAD_TIMEOUT             = time.Minute * 30          // 上传请求超时时间
+	UPLOAD_RETRY_COUNT         = 3
+	UPLOAD_RETRY_WAIT_TIME     = time.Second * 1
+	UPLOAD_RETRY_MAX_WAIT_TIME = time.Second * 5
+)
+
 var config = driver.Config{
 	Name:        "BaiduNetdisk",
 	DefaultRoot: "/",

drivers/baidu_netdisk/types.go

@@ -1,6 +1,7 @@
 package baidu_netdisk
 
 import (
+	"errors"
 	"path"
 	"strconv"
 	"time"
@@ -9,6 +10,10 @@ import (
 	"github.com/OpenListTeam/OpenList/v4/pkg/utils"
 )
 
+var (
+	ErrBaiduEmptyFilesNotAllowed = errors.New("empty files are not allowed by baidu netdisk")
+)
+
 type TokenErrResp struct {
 	ErrorDescription string `json:"error_description"`
 	Error            string `json:"error"`
@@ -188,6 +193,32 @@ type PrecreateResp struct {
 
 	// return_type=2
 	File File `json:"info"`
+
+	UploadURL string `json:"-"` // 保存断点续传对应的上传域名
+}
+
+type UploadServerResp struct {
+	BakServer  []any `json:"bak_server"`
+	BakServers []struct {
+		Server string `json:"server"`
+	} `json:"bak_servers"`
+	ClientIP    string `json:"client_ip"`
+	ErrorCode   int    `json:"error_code"`
+	ErrorMsg    string `json:"error_msg"`
+	Expire      int    `json:"expire"`
+	Host        string `json:"host"`
+	Newno       string `json:"newno"`
+	QuicServer  []any  `json:"quic_server"`
+	QuicServers []struct {
+		Server string `json:"server"`
+	} `json:"quic_servers"`
+	RequestID  int64 `json:"request_id"`
+	Server     []any `json:"server"`
+	ServerTime int   `json:"server_time"`
+	Servers    []struct {
+		Server string `json:"server"`
+	} `json:"servers"`
+	Sl int `json:"sl"`
 }
 
 type QuotaResp struct {

drivers/baidu_netdisk/util.go

@@ -115,7 +115,7 @@ func (d *BaiduNetdisk) request(furl string, method string, callback base.ReqCall
 		errno := utils.Json.Get(res.Body(), "errno").ToInt()
 		if errno != 0 {
 			if utils.SliceContains([]int{111, -6}, errno) {
-				log.Info("refreshing baidu_netdisk token.")
+				log.Info("[baidu_netdisk] refreshing baidu_netdisk token.")
 				err2 := d.refreshToken()
 				if err2 != nil {
 					return retry.Unrecoverable(err2)
@@ -326,10 +326,10 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 {
 	// 非会员固定为 4MB
 	if d.vipType == 0 {
 		if d.CustomUploadPartSize != 0 {
-			log.Warnf("CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize")
+			log.Warnf("[baidu_netdisk] CustomUploadPartSize is not supported for non-vip user, use DefaultSliceSize")
 		}
 		if filesize > MaxSliceNum*DefaultSliceSize {
-			log.Warnf("File size(%d) is too large, may cause upload failure", filesize)
+			log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize)
 		}
 
 		return DefaultSliceSize
@@ -337,17 +337,17 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 {
 
 	if d.CustomUploadPartSize != 0 {
 		if d.CustomUploadPartSize < DefaultSliceSize {
-			log.Warnf("CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize)
+			log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is less than DefaultSliceSize(%d), use DefaultSliceSize", d.CustomUploadPartSize, DefaultSliceSize)
 			return DefaultSliceSize
 		}
 
 		if d.vipType == 1 && d.CustomUploadPartSize > VipSliceSize {
-			log.Warnf("CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize)
+			log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than VipSliceSize(%d), use VipSliceSize", d.CustomUploadPartSize, VipSliceSize)
 			return VipSliceSize
 		}
 
 		if d.vipType == 2 && d.CustomUploadPartSize > SVipSliceSize {
-			log.Warnf("CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize)
+			log.Warnf("[baidu_netdisk] CustomUploadPartSize(%d) is greater than SVipSliceSize(%d), use SVipSliceSize", d.CustomUploadPartSize, SVipSliceSize)
 			return SVipSliceSize
 		}
 
@@ -377,7 +377,7 @@ func (d *BaiduNetdisk) getSliceSize(filesize int64) int64 {
 	}
 
 	if filesize > MaxSliceNum*maxSliceSize {
-		log.Warnf("File size(%d) is too large, may cause upload failure", filesize)
+		log.Warnf("[baidu_netdisk] File size(%d) is too large, may cause upload failure", filesize)
 	}
 
 	return maxSliceSize
@@ -394,6 +394,97 @@ func (d *BaiduNetdisk) quota(ctx context.Context) (model.DiskUsage, error) {
 	return driver.DiskUsageFromUsedAndTotal(resp.Used, resp.Total), nil
 }
 
+// getUploadUrl 从开放平台获取上传域名/地址，并发请求会被合并，结果会在 uploadid 生命周期内复用。
+// 如果获取失败，则返回 Upload API设置项。
+func (d *BaiduNetdisk) getUploadUrl(path, uploadId string) string {
+	if !d.UseDynamicUploadAPI || uploadId == "" {
+		return d.UploadAPI
+	}
+	getCachedUrlFunc := func() (string, bool) {
+		d.uploadUrlMu.RLock()
+		defer d.uploadUrlMu.RUnlock()
+		if entry, ok := d.uploadUrlCache[uploadId]; ok {
+			return entry.url, true
+		}
+		return "", false
+	}
+	// 检查地址缓存
+	if uploadUrl, ok := getCachedUrlFunc(); ok {
+		return uploadUrl
+	}
+
+	uploadUrlGetFunc := func() (string, error) {
+		// 双重检查缓存
+		if uploadUrl, ok := getCachedUrlFunc(); ok {
+			return uploadUrl, nil
+		}
+
+		uploadUrl, err := d.requestForUploadUrl(path, uploadId)
+		if err != nil {
+			return "", err
+		}
+
+		d.uploadUrlMu.Lock()
+		d.uploadUrlCache[uploadId] = uploadURLCacheEntry{
+			url:        uploadUrl,
+			updateTime: time.Now(),
+		}
+		d.uploadUrlMu.Unlock()
+		return uploadUrl, nil
+	}
+
+	uploadUrl, err, _ := d.uploadUrlG.Do(uploadId, uploadUrlGetFunc)
+	if err != nil {
+		fallback := d.UploadAPI
+		log.Warnf("[baidu_netdisk] get upload URL failed (%v), will use fallback URL: %s", err, fallback)
+		return fallback
+	}
+	return uploadUrl
+}
+
+func (d *BaiduNetdisk) clearUploadUrlCache(uploadId string) {
+	if uploadId == "" {
+		return
+	}
+	d.uploadUrlMu.Lock()
+	if _, ok := d.uploadUrlCache[uploadId]; ok {
+		delete(d.uploadUrlCache, uploadId)
+	}
+	d.uploadUrlMu.Unlock()
+}
+
+// requestForUploadUrl 请求获取上传地址。
+// 实测此接口不需要认证，传method和upload_version就行，不过还是按文档规范调用。
+// https://pan.baidu.com/union/doc/Mlvw5hfnr
+func (d *BaiduNetdisk) requestForUploadUrl(path, uploadId string) (string, error) {
+	params := map[string]string{
+		"method":         "locateupload",
+		"appid":          "250528",
+		"path":           path,
+		"uploadid":       uploadId,
+		"upload_version": "2.0",
+	}
+	apiUrl := "https://d.pcs.baidu.com/rest/2.0/pcs/file"
+	var resp UploadServerResp
+	_, err := d.request(apiUrl, http.MethodGet, func(req *resty.Request) {
+		req.SetQueryParams(params)
+	}, &resp)
+	if err != nil {
+		return "", err
+	}
+	// 应该是https开头的一个地址
+	var uploadUrl string
+	if len(resp.Servers) > 0 {
+		uploadUrl = resp.Servers[0].Server
+	} else if len(resp.BakServers) > 0 {
+		uploadUrl = resp.BakServers[0].Server
+	}
+	if uploadUrl == "" {
+		return "", errors.New("upload URL is empty")
+	}
+	return uploadUrl, nil
+}
+
 // func encodeURIComponent(str string) string {
 // 	r := url.QueryEscape(str)
 // 	r = strings.ReplaceAll(r, "+", "%20")