diff --git a/Dockerfile b/Dockerfile index 2037596..4ba3410 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,6 @@ FROM alpine WORKDIR /root/ -# 安装依赖 -RUN apk add --no-cache skopeo - COPY --from=builder /app/hubproxy . COPY --from=builder /app/config.toml . diff --git a/README.md b/README.md index 3e297b0..100516e 100644 --- a/README.md +++ b/README.md @@ -6,14 +6,14 @@ ## ✨ 特性 -- 🐳 **Docker 镜像加速** - 单域名实现 Docker Hub、GHCR、Quay 等多个镜像仓库加速，以及优化拉取速度。 -- 🐳 **离线镜像包** - 支持批量下载离线镜像包。 +- 🐳 **Docker 镜像加速** - 单域名实现 Docker Hub、GHCR、Quay 等多个镜像仓库加速，流式传输优化拉取速度。 +- 🐳 **离线镜像包** - 支持下载离线镜像包，流式传输加防抖设计。 - 📁 **GitHub 文件加速** - 加速 GitHub Release、Raw 文件下载，支持`api.github.com`，脚本嵌套加速等等 - 🤖 **AI 模型库支持** - 支持 Hugging Face 模型下载加速 - 🛡️ **智能限流** - IP 限流保护，防止滥用 - 🚫 **仓库审计** - 强大的自定义黑名单，白名单，同时审计镜像仓库，和GitHub仓库 - 🔍 **镜像搜索** - 在线搜索 Docker 镜像 -- ⚡ **轻量高效** - 基于 Go 语言，单二进制文件运行，资源占用低 +- ⚡ **轻量高效** - 基于 Go 语言，单二进制文件运行，资源占用低，优雅的内存清理机制。 - 🔧 **配置热重载** - 统一配置管理，部分配置项支持热重载，无需重启服务 ## 🚀 快速开始 @@ -87,11 +87,6 @@ example.com { ``` -## 🙏 致谢 - - -- UI 界面参考了[相关开源项目](https://github.com/WJQSERVER-STUDIO/GHProxy-Frontend) - ## ⚠️ 免责声明 - 本程序仅供学习交流使用，请勿用于非法用途 diff --git a/install-service.sh b/install-service.sh index b1d3fcd..37a9112 100644 --- a/install-service.sh +++ b/install-service.sh @@ -62,7 +62,7 @@ else # 检查依赖 missing_deps=() - for cmd in curl jq tar skopeo; do + for cmd in curl jq tar; do if ! command -v $cmd &> /dev/null; then missing_deps+=($cmd) fi @@ -72,14 +72,14 @@ else echo -e "${YELLOW}检测到缺少依赖: ${missing_deps[*]}${NC}" echo -e "${BLUE}正在自动安装依赖...${NC}" - apt update && apt install -y curl jq skopeo + apt update && apt install -y curl jq if [ $? -ne 0 ]; then echo -e "${RED}依赖安装失败${NC}" exit 1 fi # 重新检查依赖 - for cmd in curl jq tar skopeo; do + for cmd in curl jq tar; do if ! command -v $cmd &> /dev/null; then echo -e "${RED}依赖安装后仍缺少: $cmd${NC}" exit 1 diff --git a/src/LICENSE b/src/LICENSE deleted file mode 100644 index 9e37e25..0000000 --- a/src/LICENSE +++ /dev/null @@ -1,107 +0,0 @@ -WJQserver Studio 开源许可证 -版本 1.2 - -版权所有 © WJQserver Studio 2024 - -定义 -许可：指在本许可证内定义的使用、复制、分发与修改的条款与要求。 -授权方：指拥有版权的个人或组织，亦或是拥有版权的个人或组织所指派的实体。 -您：指行使本许可授予的权限的个人或法律实体。 -开源与自由软件 -本项目为开源软件，允许用户在遵循本许可证的前提下访问和使用源代码。 -本项目不等同于自由软件，使用权限受到本许可证条款的限制。 -强调版权所有，所有权利均由 WJQserver Studio 保留。 -许可证条款 -1. 使用权限 -1.1 您被授予在私人环境中自由使用本软件的权限。 - -1.2 您可以在不修改关键声明的前提下进行商用。 - -2. 复制与分发 -2.1 您可以复制和分发本软件的原始版本，前提是必须保留所有版权声明和本许可证。 - -3. 修改权限 -3.1 您可以在非商业用途下修改本软件，前提是继承本许可证并保留原版权声明。 - -3.2 禁止在修改后进行商业用途。 - -4. 专利引用 -4.1 若项目被专利相关引用，必须保留来源声明。 - -4.2 若为商业场景，需按照商用处理。 - -5. 免责声明 -5.1 本软件按“现状”提供，不提供任何明示或暗示的保证，包括但不限于适销性、特定用途适用性及非侵权性。 - -5.2 在任何情况下，授权方均不对因使用或无法使用本软件而产生的任何直接、间接、偶然、特殊、惩罚性或后果性损害负责，即使已被告知可能发生此类损害。 - -5.3 用户需根据当地法律对待本项目，确保遵守所有适用法规。 - -6. 许可证期限 -6.1 本许可证自2024年开始生效，有效期暂为无限。 - -6.2 项目所有方有权修改许可证相关条例而不另行通知。 - -条款修订 -7.1 授权方保留随时修改本许可证条款的权利，以便更好地适应法律和技术的发展。 - -7.2 修订后的条款将在发布时生效，继续使用本软件即表示接受修订后的条款。 - -其他 -8.1 本许可证不影响您作为最终用户的法定权利。 - -8.2 若本许可证的某些条款被认定为不可执行，其余条款仍然有效。 - -WJQserver Studio Open Source License -Version 1.2 - -Copyright © WJQserver Studio 2024 - -Definitions -License: The terms and conditions defined within this license for use, copying, distribution, and modification. -Licensor: The individual or organization holding the copyright, or the entity designated by them. -You: The individual or legal entity exercising the permissions granted by this license. -Open Source vs. Free Software -This project is open source, allowing users to access and use the source code under the terms of this license. -This project is not equivalent to free software; usage rights are restricted by this license. -Copyright is emphasized, with all rights reserved by WJQserver Studio. -License Terms -1. Usage Rights -1.1 You are granted the right to use this software freely in a private environment. - -1.2 You may use it commercially without modifying key statements. - -2. Copying and Distribution -2.1 You may copy and distribute the original version of this software, provided all copyright notices and this license are retained. - -3. Modification Rights -3.1 You may modify this software for non-commercial purposes, provided you inherit this license and retain the original copyright notice. - -3.2 Modifications cannot be used commercially. - -4. Patent References -4.1 If the project is cited in patent-related contexts, the source statement must be retained. - -4.2 For commercial scenarios, it must be treated as a commercial use. - -5. Disclaimer -5.1 This software is provided "as is", without any express or implied warranties, including but not limited to merchantability, fitness for a particular purpose, and non-infringement. - -5.2 In no event shall the licensor be liable for any direct, indirect, incidental, special, punitive, or consequential damages arising out of the use or inability to use this software, even if advised of the possibility of such damages. - -5.3 Users must comply with all applicable laws regarding this project. - -6. License Duration -6.1 This license is effective from 2024, with an indefinite duration. - -6.2 The project owner reserves the right to modify the license terms without prior notice. - -Amendments -7.1 The licensor reserves the right to amend this license at any time to better adapt to legal and technological developments. - -7.2 Revised terms become effective upon publication, and continued use of the software indicates acceptance of the revised terms. - -Miscellaneous -8.1 This license does not affect your statutory rights as an end user. - -8.2 If any provision of this license is held to be unenforceable, the remaining provisions shall remain in effect. \ No newline at end of file diff --git a/src/access_control.go b/src/access_control.go index c589c86..c2a3dee 100644 --- a/src/access_control.go +++ b/src/access_control.go @@ -31,13 +31,10 @@ var GlobalAccessController = &AccessController{} // ParseDockerImage 解析Docker镜像名称 func (ac *AccessController) ParseDockerImage(image string) DockerImageInfo { - // 移除可能的协议前缀 image = strings.TrimPrefix(image, "docker://") - // 分离标签 var tag string if idx := strings.LastIndex(image, ":"); idx != -1 { - // 检查是否是端口号而不是标签（包含斜杠） part := image[idx+1:] if !strings.Contains(part, "/") { tag = part @@ -48,15 +45,11 @@ func (ac *AccessController) ParseDockerImage(image string) DockerImageInfo { tag = "latest" } - // 分离命名空间和仓库名 var namespace, repository string if strings.Contains(image, "/") { - // 处理自定义registry的情况，如 registry.com/user/repo parts := strings.Split(image, "/") if len(parts) >= 2 { - // 检查第一部分是否是域名（包含.） if strings.Contains(parts[0], ".") { - // 跳过registry域名，取用户名和仓库名 if len(parts) >= 3 { namespace = parts[1] repository = parts[2] @@ -65,13 +58,11 @@ func (ac *AccessController) ParseDockerImage(image string) DockerImageInfo { repository = parts[1] } } else { - // 标准格式：user/repo namespace = parts[0] repository = parts[1] } } } else { - // 官方镜像，如 nginx namespace = "library" repository = image } @@ -171,7 +162,6 @@ func (ac *AccessController) matchImageInList(imageInfo DockerImageInfo, list []s } } - // 5. 子仓库匹配（防止 user/repo 匹配到 user/repo-fork） if strings.HasPrefix(fullName, item+"/") { return true } @@ -185,7 +175,6 @@ func (ac *AccessController) checkList(matches, list []string) bool { return false } - // 组合用户名和仓库名，处理.git后缀 username := strings.ToLower(strings.TrimSpace(matches[0])) repoName := strings.ToLower(strings.TrimSpace(strings.TrimSuffix(matches[1], ".git"))) fullRepo := username + "/" + repoName @@ -196,10 +185,7 @@ func (ac *AccessController) checkList(matches, list []string) bool { continue } - // 支持多种匹配模式： - // 1. 精确匹配: "vaxilu/x-ui" - // 2. 用户级匹配: "vaxilu/*" 或 "vaxilu" - // 3. 前缀匹配: "vaxilu/x-ui-*" + // 支持多种匹配模式 if fullRepo == item { return true } @@ -225,15 +211,10 @@ func (ac *AccessController) checkList(matches, list []string) bool { return false } -// 🔥 Reload 热重载访问控制规则 +// Reload 热重载访问控制规则 func (ac *AccessController) Reload() { ac.mu.Lock() defer ac.mu.Unlock() - // 访问控制器本身不缓存配置，每次检查时都会调用GetConfig() - // 所以这里只需要确保锁的原子性，实际的重载在GetConfig()中完成 - // 可以在这里添加一些初始化逻辑，比如预编译正则表达式等 - - // 目前访问控制器设计为无状态的，每次检查都读取最新配置 - // 这样设计的好处是配置更新后无需额外处理，自动生效 + // 访问控制器本身不缓存配置 } \ No newline at end of file diff --git a/src/config.go b/src/config.go index e8aea65..0d20aa9 100644 --- a/src/config.go +++ b/src/config.go @@ -48,10 +48,8 @@ type AppConfig struct { MaxImages int `toml:"maxImages"` // 单次下载最大镜像数量限制 } `toml:"download"` - // 新增：Registry映射配置 Registries map[string]RegistryMapping `toml:"registries"` - // Token缓存配置 TokenCache struct { Enabled bool `toml:"enabled"` // 是否启用token缓存 DefaultTTL string `toml:"defaultTTL"` // 默认缓存时间 @@ -63,6 +61,11 @@ var ( appConfigLock sync.RWMutex isViperEnabled bool viperInstance *viper.Viper + + cachedConfig *AppConfig + configCacheTime time.Time + configCacheTTL = 5 * time.Second + configCacheMutex sync.RWMutex ) // DefaultConfig 返回默认配置 @@ -141,21 +144,44 @@ func DefaultConfig() *AppConfig { // GetConfig 安全地获取配置副本 func GetConfig() *AppConfig { - appConfigLock.RLock() - defer appConfigLock.RUnlock() + configCacheMutex.RLock() + if cachedConfig != nil && time.Since(configCacheTime) < configCacheTTL { + config := cachedConfig + configCacheMutex.RUnlock() + return config + } + configCacheMutex.RUnlock() - if appConfig == nil { - return DefaultConfig() + // 缓存过期，重新生成配置 + configCacheMutex.Lock() + defer configCacheMutex.Unlock() + + // 双重检查，防止重复生成 + if cachedConfig != nil && time.Since(configCacheTime) < configCacheTTL { + return cachedConfig } - // 返回配置的深拷贝 + appConfigLock.RLock() + if appConfig == nil { + appConfigLock.RUnlock() + defaultCfg := DefaultConfig() + cachedConfig = defaultCfg + configCacheTime = time.Now() + return defaultCfg + } + + // 生成新的配置深拷贝 configCopy := *appConfig configCopy.Security.WhiteList = append([]string(nil), appConfig.Security.WhiteList...) configCopy.Security.BlackList = append([]string(nil), appConfig.Security.BlackList...) configCopy.Proxy.WhiteList = append([]string(nil), appConfig.Proxy.WhiteList...) configCopy.Proxy.BlackList = append([]string(nil), appConfig.Proxy.BlackList...) + appConfigLock.RUnlock() - return &configCopy + cachedConfig = &configCopy + configCacheTime = time.Now() + + return cachedConfig } // setConfig 安全地设置配置 @@ -163,6 +189,10 @@ func setConfig(cfg *AppConfig) { appConfigLock.Lock() defer appConfigLock.Unlock() appConfig = cfg + + configCacheMutex.Lock() + cachedConfig = nil + configCacheMutex.Unlock() } // LoadConfig 加载配置文件 @@ -185,19 +215,13 @@ func LoadConfig() error { // 设置配置 setConfig(cfg) - // 🔥 首次加载后启用Viper热重载 if !isViperEnabled { go enableViperHotReload() } - fmt.Printf("配置加载成功: 监听 %s:%d, 文件大小限制 %d MB, 限流 %d请求/%g小时, 离线镜像并发数 %d\n", - cfg.Server.Host, cfg.Server.Port, cfg.Server.FileSize/(1024*1024), - cfg.RateLimit.RequestLimit, cfg.RateLimit.PeriodHours, cfg.Download.MaxImages) - return nil } -// 🔥 启用Viper自动热重载 func enableViperHotReload() { if isViperEnabled { return @@ -218,9 +242,7 @@ func enableViperHotReload() { } isViperEnabled = true - fmt.Println("自动热重载已启用") - // 🚀 启用文件监听 viperInstance.WatchConfig() viperInstance.OnConfigChange(func(e fsnotify.Event) { fmt.Printf("检测到配置文件变化: %s\n", e.Name) @@ -228,7 +250,6 @@ func enableViperHotReload() { }) } -// 🔥 使用Viper进行热重载 func hotReloadWithViper() { start := time.Now() fmt.Println("🔄 自动热重载...") @@ -242,10 +263,8 @@ func hotReloadWithViper() { return } - // 从环境变量覆盖（保持原有功能） overrideFromEnv(cfg) - // 原子性更新配置 setConfig(cfg) // 异步更新受影响的组件 @@ -255,7 +274,6 @@ func hotReloadWithViper() { }() } -// 🔧 更新受配置影响的组件 func updateAffectedComponents() { // 重新初始化限流器 if globalLimiter != nil { @@ -269,7 +287,6 @@ func updateAffectedComponents() { GlobalAccessController.Reload() } - // 🔥 刷新Registry配置映射 fmt.Println("🌐 更新Registry配置映射...") reloadRegistryConfig() @@ -277,7 +294,6 @@ func updateAffectedComponents() { fmt.Println("🔧 组件更新完成") } -// 🔥 重新加载Registry配置 func reloadRegistryConfig() { cfg := GetConfig() enabledCount := 0 @@ -291,8 +307,6 @@ func reloadRegistryConfig() { fmt.Printf("🌐 Registry配置已更新: %d个启用\n", enabledCount) - // Registry配置是动态读取的，每次请求都会调用GetConfig() - // 所以这里只需要简单通知，实际生效是自动的 } // overrideFromEnv 从环境变量覆盖配置 diff --git a/src/docker.go b/src/docker.go index e28e913..4b4ebf1 100644 --- a/src/docker.go +++ b/src/docker.go @@ -78,8 +78,6 @@ func initDockerProxy() { registry: registry, options: options, } - - fmt.Printf("Docker代理已初始化\n") } // ProxyDockerRegistryGin 标准Docker Registry API v2代理 @@ -105,7 +103,6 @@ func handleRegistryRequest(c *gin.Context, path string) { // 移除 /v2/ 前缀 pathWithoutV2 := strings.TrimPrefix(path, "/v2/") - // 🔍 新增：Registry域名检测和路由 if registryDomain, remainingPath := registryDetector.detectRegistryDomain(pathWithoutV2); registryDomain != "" { if registryDetector.isRegistryEnabled(registryDomain) { // 设置目标Registry信息到Context @@ -118,7 +115,6 @@ func handleRegistryRequest(c *gin.Context, path string) { } } - // 原有逻辑完全保持（零改动） imageName, apiType, reference := parseRegistryPath(pathWithoutV2) if imageName == "" || apiType == "" { c.String(http.StatusBadRequest, "Invalid path format") @@ -392,9 +388,7 @@ func (r *ResponseRecorder) Write(data []byte) (int, error) { return r.ResponseWriter.Write(data) } -// proxyDockerAuthOriginal Docker认证代理（原始逻辑，保持不变） func proxyDockerAuthOriginal(c *gin.Context) { - // 检查是否有目标Registry域名（来自Context） var authURL string if targetDomain, exists := c.Get("target_registry_domain"); exists { if mapping, found := registryDetector.getRegistryMapping(targetDomain.(string)); found { @@ -672,17 +666,11 @@ func createUpstreamOptions(mapping RegistryMapping) []remote.Option { remote.WithUserAgent("hubproxy/go-containerregistry"), } - // 根据Registry类型添加特定的认证选项 + // 根据Registry类型添加特定的认证选项（方便后续扩展） switch mapping.AuthType { case "github": - // GitHub Container Registry 通常使用匿名访问 - // 如需要认证，可在此处添加 case "google": - // Google Container Registry 配置 - // 如需要认证，可在此处添加 case "quay": - // Quay.io 配置 - // 如需要认证，可在此处添加 } return options diff --git a/src/go.mod b/src/go.mod index 45c215a..05d2752 100644 --- a/src/go.mod +++ b/src/go.mod @@ -6,10 +6,8 @@ require ( github.com/fsnotify/fsnotify v1.8.0 github.com/gin-gonic/gin v1.10.0 github.com/google/go-containerregistry v0.20.5 - github.com/gorilla/websocket v1.5.1 github.com/pelletier/go-toml/v2 v2.2.3 github.com/spf13/viper v1.20.1 - golang.org/x/sync v0.14.0 golang.org/x/time v0.11.0 ) @@ -55,6 +53,7 @@ require ( golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.32.0 // indirect golang.org/x/net v0.33.0 // indirect + golang.org/x/sync v0.14.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.36.3 // indirect diff --git a/src/go.sum b/src/go.sum index 5a505da..7176baa 100644 --- a/src/go.sum +++ b/src/go.sum @@ -44,8 +44,6 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/go-containerregistry v0.20.5 h1:4RnlYcDs5hoA++CeFjlbZ/U9Yp1EuWr+UhhTyYQjOP0= github.com/google/go-containerregistry v0.20.5/go.mod h1:Q14vdOOzug02bwnhMkZKD4e30pDaD9W65qzXpyzF49E= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= diff --git a/src/imagetar.go b/src/imagetar.go new file mode 100644 index 0000000..64c8fce --- /dev/null +++ b/src/imagetar.go @@ -0,0 +1,932 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "context" + "crypto/md5" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "sort" + "strings" + "sync" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// DebounceEntry 防抖条目 +type DebounceEntry struct { + LastRequest time.Time + UserID string +} + +// DownloadDebouncer 下载防抖器 +type DownloadDebouncer struct { + mu sync.RWMutex + entries map[string]*DebounceEntry + window time.Duration +} + +// NewDownloadDebouncer 创建下载防抖器 +func NewDownloadDebouncer(window time.Duration) *DownloadDebouncer { + return &DownloadDebouncer{ + entries: make(map[string]*DebounceEntry), + window: window, + } +} + +// ShouldAllow 检查是否应该允许请求 +func (d *DownloadDebouncer) ShouldAllow(userID, contentKey string) bool { + d.mu.Lock() + defer d.mu.Unlock() + + key := userID + ":" + contentKey + now := time.Now() + + if entry, exists := d.entries[key]; exists { + if now.Sub(entry.LastRequest) < d.window { + return false // 在防抖窗口内，拒绝请求 + } + } + + // 更新或创建条目 + d.entries[key] = &DebounceEntry{ + LastRequest: now, + UserID: userID, + } + + // 清理过期条目（简单策略：每100次请求清理一次） + if len(d.entries)%100 == 0 { + d.cleanup(now) + } + + return true +} + +// cleanup 清理过期条目 +func (d *DownloadDebouncer) cleanup(now time.Time) { + for key, entry := range d.entries { + if now.Sub(entry.LastRequest) > d.window*2 { + delete(d.entries, key) + } + } +} + +// generateContentFingerprint 生成内容指纹 +func generateContentFingerprint(images []string, platform string) string { + // 对镜像列表排序确保顺序无关 + sortedImages := make([]string, len(images)) + copy(sortedImages, images) + sort.Strings(sortedImages) + + // 组合内容：镜像列表 + 平台信息 + content := strings.Join(sortedImages, "|") + ":" + platform + + // 生成MD5哈希 + hash := md5.Sum([]byte(content)) + return hex.EncodeToString(hash[:]) +} + +// getUserID 获取用户标识 +func getUserID(c *gin.Context) string { + // 优先使用会话Cookie + if sessionID, err := c.Cookie("session_id"); err == nil && sessionID != "" { + return "session:" + sessionID + } + + // 备用方案：IP + User-Agent组合 + ip := c.ClientIP() + userAgent := c.GetHeader("User-Agent") + if userAgent == "" { + userAgent = "unknown" + } + + // 生成简短标识 + combined := ip + ":" + userAgent + hash := md5.Sum([]byte(combined)) + return "ip:" + hex.EncodeToString(hash[:8]) // 只取前8字节 +} + +// 全局防抖器实例 +var ( + singleImageDebouncer *DownloadDebouncer + batchImageDebouncer *DownloadDebouncer +) + +// initDebouncer 初始化防抖器 +func initDebouncer() { + // 单个镜像：5秒防抖窗口 + singleImageDebouncer = NewDownloadDebouncer(5 * time.Second) + // 批量镜像：30秒防抖窗口（影响更大，需要更长保护） + batchImageDebouncer = NewDownloadDebouncer(30 * time.Second) +} + +// ImageStreamer 镜像流式下载器 +type ImageStreamer struct { + concurrency int + remoteOptions []remote.Option +} + +// ImageStreamerConfig 下载器配置 +type ImageStreamerConfig struct { + Concurrency int +} + +// NewImageStreamer 创建镜像下载器 +func NewImageStreamer(config *ImageStreamerConfig) *ImageStreamer { + if config == nil { + config = &ImageStreamerConfig{} + } + + concurrency := config.Concurrency + if concurrency <= 0 { + cfg := GetConfig() + concurrency = cfg.Download.MaxImages + if concurrency <= 0 { + concurrency = 10 + } + } + + remoteOptions := []remote.Option{ + remote.WithAuth(authn.Anonymous), + remote.WithTransport(GetGlobalHTTPClient().Transport), + } + + return &ImageStreamer{ + concurrency: concurrency, + remoteOptions: remoteOptions, + } +} + +// StreamOptions 下载选项 +type StreamOptions struct { + Platform string + Compression bool +} + +// StreamImageToWriter 流式下载镜像到Writer +func (is *ImageStreamer) StreamImageToWriter(ctx context.Context, imageRef string, writer io.Writer, options *StreamOptions) error { + if options == nil { + options = &StreamOptions{} + } + + ref, err := name.ParseReference(imageRef) + if err != nil { + return fmt.Errorf("解析镜像引用失败: %w", err) + } + + log.Printf("开始下载镜像: %s", ref.String()) + + contextOptions := append(is.remoteOptions, remote.WithContext(ctx)) + + desc, err := is.getImageDescriptorWithPlatform(ref, contextOptions, options.Platform) + if err != nil { + return fmt.Errorf("获取镜像描述失败: %w", err) + } + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + return is.streamMultiArchImage(ctx, desc, writer, options, contextOptions, imageRef) + case types.OCIManifestSchema1, types.DockerManifestSchema2: + return is.streamSingleImage(ctx, desc, writer, options, contextOptions, imageRef) + default: + return is.streamSingleImage(ctx, desc, writer, options, contextOptions, imageRef) + } +} + +// getImageDescriptor 获取镜像描述符 +func (is *ImageStreamer) getImageDescriptor(ref name.Reference, options []remote.Option) (*remote.Descriptor, error) { + return is.getImageDescriptorWithPlatform(ref, options, "") +} + +// getImageDescriptorWithPlatform 获取指定平台的镜像描述符 +func (is *ImageStreamer) getImageDescriptorWithPlatform(ref name.Reference, options []remote.Option, platform string) (*remote.Descriptor, error) { + if isCacheEnabled() { + var reference string + if tagged, ok := ref.(name.Tag); ok { + reference = tagged.TagStr() + } else if digested, ok := ref.(name.Digest); ok { + reference = digested.DigestStr() + } + + if reference != "" { + cacheKey := buildManifestCacheKeyWithPlatform(ref.Context().String(), reference, platform) + if cachedItem := globalCache.Get(cacheKey); cachedItem != nil { + desc := &remote.Descriptor{ + Manifest: cachedItem.Data, + } + log.Printf("使用缓存的manifest: %s (平台: %s)", ref.String(), platform) + return desc, nil + } + } + } + + desc, err := remote.Get(ref, options...) + if err != nil { + return nil, err + } + + if isCacheEnabled() { + var reference string + if tagged, ok := ref.(name.Tag); ok { + reference = tagged.TagStr() + } else if digested, ok := ref.(name.Digest); ok { + reference = digested.DigestStr() + } + + if reference != "" { + cacheKey := buildManifestCacheKeyWithPlatform(ref.Context().String(), reference, platform) + ttl := getManifestTTL(reference) + headers := map[string]string{ + "Docker-Content-Digest": desc.Digest.String(), + } + globalCache.Set(cacheKey, desc.Manifest, string(desc.MediaType), headers, ttl) + log.Printf("缓存manifest: %s (平台: %s, TTL: %v)", ref.String(), platform, ttl) + } + } + + return desc, nil +} + +// StreamImageToGin 流式响应到Gin +func (is *ImageStreamer) StreamImageToGin(ctx context.Context, imageRef string, c *gin.Context, options *StreamOptions) error { + if options == nil { + options = &StreamOptions{} + } + + filename := strings.ReplaceAll(imageRef, "/", "_") + ".tar" + c.Header("Content-Type", "application/octet-stream") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + + if options.Compression { + c.Header("Content-Encoding", "gzip") + } + + return is.StreamImageToWriter(ctx, imageRef, c.Writer, options) +} + +// streamMultiArchImage 处理多架构镜像 +func (is *ImageStreamer) streamMultiArchImage(ctx context.Context, desc *remote.Descriptor, writer io.Writer, options *StreamOptions, remoteOptions []remote.Option, imageRef string) error { + img, err := is.selectPlatformImage(desc, options) + if err != nil { + return err + } + + return is.streamImageLayers(ctx, img, writer, options, imageRef) +} + +// streamSingleImage 处理单架构镜像 +func (is *ImageStreamer) streamSingleImage(ctx context.Context, desc *remote.Descriptor, writer io.Writer, options *StreamOptions, remoteOptions []remote.Option, imageRef string) error { + img, err := desc.Image() + if err != nil { + return fmt.Errorf("获取镜像失败: %w", err) + } + + return is.streamImageLayers(ctx, img, writer, options, imageRef) +} + +// streamImageLayers 处理镜像层 +func (is *ImageStreamer) streamImageLayers(ctx context.Context, img v1.Image, writer io.Writer, options *StreamOptions, imageRef string) error { + var finalWriter io.Writer = writer + + if options.Compression { + gzWriter := gzip.NewWriter(writer) + defer gzWriter.Close() + finalWriter = gzWriter + } + + tarWriter := tar.NewWriter(finalWriter) + defer tarWriter.Close() + + configFile, err := img.ConfigFile() + if err != nil { + return fmt.Errorf("获取镜像配置失败: %w", err) + } + + layers, err := img.Layers() + if err != nil { + return fmt.Errorf("获取镜像层失败: %w", err) + } + + log.Printf("镜像包含 %d 层", len(layers)) + + return is.streamDockerFormat(ctx, tarWriter, img, layers, configFile, imageRef) +} + +// streamDockerFormat 生成Docker格式 +func (is *ImageStreamer) streamDockerFormat(ctx context.Context, tarWriter *tar.Writer, img v1.Image, layers []v1.Layer, configFile *v1.ConfigFile, imageRef string) error { + return is.streamDockerFormatWithReturn(ctx, tarWriter, img, layers, configFile, imageRef, nil, nil) +} + +// streamDockerFormatWithReturn 生成Docker格式并返回manifest和repositories信息 +func (is *ImageStreamer) streamDockerFormatWithReturn(ctx context.Context, tarWriter *tar.Writer, img v1.Image, layers []v1.Layer, configFile *v1.ConfigFile, imageRef string, manifestOut *map[string]interface{}, repositoriesOut *map[string]map[string]string) error { + configDigest, err := img.ConfigName() + if err != nil { + return err + } + + configData, err := json.Marshal(configFile) + if err != nil { + return err + } + + configHeader := &tar.Header{ + Name: configDigest.String() + ".json", + Size: int64(len(configData)), + Mode: 0644, + } + + if err := tarWriter.WriteHeader(configHeader); err != nil { + return err + } + if _, err := tarWriter.Write(configData); err != nil { + return err + } + + layerDigests := make([]string, len(layers)) + for i, layer := range layers { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + if err := func() error { + digest, err := layer.Digest() + if err != nil { + return err + } + layerDigests[i] = digest.String() + + layerDir := digest.String() + layerHeader := &tar.Header{ + Name: layerDir + "/", + Typeflag: tar.TypeDir, + Mode: 0755, + } + + if err := tarWriter.WriteHeader(layerHeader); err != nil { + return err + } + + uncompressedSize, err := partial.UncompressedSize(layer) + if err != nil { + return err + } + + layerReader, err := layer.Uncompressed() + if err != nil { + return err + } + defer layerReader.Close() + + layerTarHeader := &tar.Header{ + Name: layerDir + "/layer.tar", + Size: uncompressedSize, + Mode: 0644, + } + + if err := tarWriter.WriteHeader(layerTarHeader); err != nil { + return err + } + + if _, err := io.Copy(tarWriter, layerReader); err != nil { + return err + } + + return nil + }(); err != nil { + return err + } + + log.Printf("已处理层 %d/%d", i+1, len(layers)) + } + + + // 构建单个镜像的manifest信息 + singleManifest := map[string]interface{}{ + "Config": configDigest.String() + ".json", + "RepoTags": []string{imageRef}, + "Layers": func() []string { + var layers []string + for _, digest := range layerDigests { + layers = append(layers, digest+"/layer.tar") + } + return layers + }(), + } + + // 构建repositories信息 + repositories := make(map[string]map[string]string) + parts := strings.Split(imageRef, ":") + if len(parts) == 2 { + repoName := parts[0] + tag := parts[1] + repositories[repoName] = map[string]string{tag: configDigest.String()} + } + + // 如果是批量下载，返回信息而不写入文件 + if manifestOut != nil && repositoriesOut != nil { + *manifestOut = singleManifest + *repositoriesOut = repositories + return nil + } + + // 单镜像下载，直接写入manifest.json + manifest := []map[string]interface{}{singleManifest} + + manifestData, err := json.Marshal(manifest) + if err != nil { + return err + } + + manifestHeader := &tar.Header{ + Name: "manifest.json", + Size: int64(len(manifestData)), + Mode: 0644, + } + + if err := tarWriter.WriteHeader(manifestHeader); err != nil { + return err + } + + if _, err := tarWriter.Write(manifestData); err != nil { + return err + } + + // 写入repositories文件 + repositoriesData, err := json.Marshal(repositories) + if err != nil { + return err + } + + repositoriesHeader := &tar.Header{ + Name: "repositories", + Size: int64(len(repositoriesData)), + Mode: 0644, + } + + if err := tarWriter.WriteHeader(repositoriesHeader); err != nil { + return err + } + + _, err = tarWriter.Write(repositoriesData) + return err +} + +// streamSingleImageForBatch 为批量下载流式处理单个镜像 +func (is *ImageStreamer) streamSingleImageForBatch(ctx context.Context, tarWriter *tar.Writer, imageRef string, options *StreamOptions) (map[string]interface{}, map[string]map[string]string, error) { + ref, err := name.ParseReference(imageRef) + if err != nil { + return nil, nil, fmt.Errorf("解析镜像引用失败: %w", err) + } + + contextOptions := append(is.remoteOptions, remote.WithContext(ctx)) + + desc, err := is.getImageDescriptorWithPlatform(ref, contextOptions, options.Platform) + if err != nil { + return nil, nil, fmt.Errorf("获取镜像描述失败: %w", err) + } + + var manifest map[string]interface{} + var repositories map[string]map[string]string + + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + // 处理多架构镜像，复用单个下载的逻辑 + img, err := is.selectPlatformImage(desc, options) + if err != nil { + return nil, nil, fmt.Errorf("选择平台镜像失败: %w", err) + } + + layers, err := img.Layers() + if err != nil { + return nil, nil, fmt.Errorf("获取镜像层失败: %w", err) + } + + configFile, err := img.ConfigFile() + if err != nil { + return nil, nil, fmt.Errorf("获取镜像配置失败: %w", err) + } + + log.Printf("镜像包含 %d 层", len(layers)) + + err = is.streamDockerFormatWithReturn(ctx, tarWriter, img, layers, configFile, imageRef, &manifest, &repositories) + if err != nil { + return nil, nil, err + } + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := desc.Image() + if err != nil { + return nil, nil, fmt.Errorf("获取镜像失败: %w", err) + } + + layers, err := img.Layers() + if err != nil { + return nil, nil, fmt.Errorf("获取镜像层失败: %w", err) + } + + configFile, err := img.ConfigFile() + if err != nil { + return nil, nil, fmt.Errorf("获取镜像配置失败: %w", err) + } + + log.Printf("镜像包含 %d 层", len(layers)) + + err = is.streamDockerFormatWithReturn(ctx, tarWriter, img, layers, configFile, imageRef, &manifest, &repositories) + if err != nil { + return nil, nil, err + } + default: + img, err := desc.Image() + if err != nil { + return nil, nil, fmt.Errorf("获取镜像失败: %w", err) + } + + layers, err := img.Layers() + if err != nil { + return nil, nil, fmt.Errorf("获取镜像层失败: %w", err) + } + + configFile, err := img.ConfigFile() + if err != nil { + return nil, nil, fmt.Errorf("获取镜像配置失败: %w", err) + } + + log.Printf("镜像包含 %d 层", len(layers)) + + err = is.streamDockerFormatWithReturn(ctx, tarWriter, img, layers, configFile, imageRef, &manifest, &repositories) + if err != nil { + return nil, nil, err + } + } + + return manifest, repositories, nil +} + +// selectPlatformImage 从多架构镜像中选择合适的平台镜像 +func (is *ImageStreamer) selectPlatformImage(desc *remote.Descriptor, options *StreamOptions) (v1.Image, error) { + index, err := desc.ImageIndex() + if err != nil { + return nil, fmt.Errorf("获取镜像索引失败: %w", err) + } + + manifest, err := index.IndexManifest() + if err != nil { + return nil, fmt.Errorf("获取索引清单失败: %w", err) + } + + // 选择合适的平台 + var selectedDesc *v1.Descriptor + for _, m := range manifest.Manifests { + if m.Platform == nil { + continue + } + + if options.Platform != "" { + platformParts := strings.Split(options.Platform, "/") + if len(platformParts) >= 2 { + targetOS := platformParts[0] + targetArch := platformParts[1] + targetVariant := "" + if len(platformParts) >= 3 { + targetVariant = platformParts[2] + } + + if m.Platform.OS == targetOS && + m.Platform.Architecture == targetArch && + m.Platform.Variant == targetVariant { + selectedDesc = &m + break + } + } + } else if m.Platform.OS == "linux" && m.Platform.Architecture == "amd64" { + selectedDesc = &m + break + } + } + + if selectedDesc == nil && len(manifest.Manifests) > 0 { + selectedDesc = &manifest.Manifests[0] + } + + if selectedDesc == nil { + return nil, fmt.Errorf("未找到合适的平台镜像") + } + + img, err := index.Image(selectedDesc.Digest) + if err != nil { + return nil, fmt.Errorf("获取选中镜像失败: %w", err) + } + + return img, nil +} + +var globalImageStreamer *ImageStreamer + +// initImageStreamer 初始化镜像下载器 +func initImageStreamer() { + globalImageStreamer = NewImageStreamer(nil) + // 镜像下载器初始化完成 +} + +// formatPlatformText 格式化平台文本 +func formatPlatformText(platform string) string { + if platform == "" { + return "自动选择" + } + return platform +} + +// initImageTarRoutes 初始化镜像下载路由 +func initImageTarRoutes(router *gin.Engine) { + imageAPI := router.Group("/api/image") + { + imageAPI.GET("/download/:image", RateLimitMiddleware(globalLimiter), handleDirectImageDownload) + imageAPI.GET("/info/:image", RateLimitMiddleware(globalLimiter), handleImageInfo) + imageAPI.POST("/batch", RateLimitMiddleware(globalLimiter), handleSimpleBatchDownload) + } +} + +// handleDirectImageDownload 处理单镜像下载 +func handleDirectImageDownload(c *gin.Context) { + imageParam := c.Param("image") + if imageParam == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "缺少镜像参数"}) + return + } + + imageRef := strings.ReplaceAll(imageParam, "_", "/") + platform := c.Query("platform") + tag := c.DefaultQuery("tag", "") + + if tag != "" && !strings.Contains(imageRef, ":") && !strings.Contains(imageRef, "@") { + imageRef = imageRef + ":" + tag + } else if !strings.Contains(imageRef, ":") && !strings.Contains(imageRef, "@") { + imageRef = imageRef + ":latest" + } + + if _, err := name.ParseReference(imageRef); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "镜像引用格式错误: " + err.Error()}) + return + } + + // 防抖检查 + userID := getUserID(c) + contentKey := generateContentFingerprint([]string{imageRef}, platform) + + if !singleImageDebouncer.ShouldAllow(userID, contentKey) { + c.JSON(http.StatusTooManyRequests, gin.H{ + "error": "请求过于频繁，请稍后再试", + "retry_after": 5, + }) + return + } + + options := &StreamOptions{ + Platform: platform, + Compression: false, + } + + ctx := c.Request.Context() + log.Printf("下载镜像: %s (平台: %s)", imageRef, formatPlatformText(platform)) + + if err := globalImageStreamer.StreamImageToGin(ctx, imageRef, c, options); err != nil { + log.Printf("镜像下载失败: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "镜像下载失败: " + err.Error()}) + return + } +} + +// handleSimpleBatchDownload 处理批量下载 +func handleSimpleBatchDownload(c *gin.Context) { + var req struct { + Images []string `json:"images" binding:"required"` + Platform string `json:"platform"` + } + + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "请求参数错误: " + err.Error()}) + return + } + + if len(req.Images) == 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "镜像列表不能为空"}) + return + } + + for i, imageRef := range req.Images { + if !strings.Contains(imageRef, ":") && !strings.Contains(imageRef, "@") { + req.Images[i] = imageRef + ":latest" + } + } + + cfg := GetConfig() + if len(req.Images) > cfg.Download.MaxImages { + c.JSON(http.StatusBadRequest, gin.H{ + "error": fmt.Sprintf("镜像数量超过限制，最大允许: %d", cfg.Download.MaxImages), + }) + return + } + + // 批量下载防抖检查 + userID := getUserID(c) + contentKey := generateContentFingerprint(req.Images, req.Platform) + + if !batchImageDebouncer.ShouldAllow(userID, contentKey) { + c.JSON(http.StatusTooManyRequests, gin.H{ + "error": "批量下载请求过于频繁，请稍后再试", + "retry_after": 30, + }) + return + } + + options := &StreamOptions{ + Platform: req.Platform, + Compression: false, + } + + ctx := c.Request.Context() + log.Printf("批量下载 %d 个镜像 (平台: %s)", len(req.Images), formatPlatformText(req.Platform)) + + filename := fmt.Sprintf("batch_%d_images.tar", len(req.Images)) + + c.Header("Content-Type", "application/octet-stream") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + + if err := globalImageStreamer.StreamMultipleImages(ctx, req.Images, c.Writer, options); err != nil { + log.Printf("批量镜像下载失败: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "批量镜像下载失败: " + err.Error()}) + return + } +} + +// handleImageInfo 处理镜像信息查询 +func handleImageInfo(c *gin.Context) { + imageParam := c.Param("image") + if imageParam == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "缺少镜像参数"}) + return + } + + imageRef := strings.ReplaceAll(imageParam, "_", "/") + tag := c.DefaultQuery("tag", "latest") + + if !strings.Contains(imageRef, ":") && !strings.Contains(imageRef, "@") { + imageRef = imageRef + ":" + tag + } + + ref, err := name.ParseReference(imageRef) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "镜像引用格式错误: " + err.Error()}) + return + } + + ctx := c.Request.Context() + contextOptions := append(globalImageStreamer.remoteOptions, remote.WithContext(ctx)) + + desc, err := globalImageStreamer.getImageDescriptor(ref, contextOptions) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "获取镜像信息失败: " + err.Error()}) + return + } + + info := gin.H{ + "name": ref.String(), + "mediaType": desc.MediaType, + "digest": desc.Digest.String(), + "size": desc.Size, + } + + if desc.MediaType == types.OCIImageIndex || desc.MediaType == types.DockerManifestList { + index, err := desc.ImageIndex() + if err == nil { + manifest, err := index.IndexManifest() + if err == nil { + var platforms []string + for _, m := range manifest.Manifests { + if m.Platform != nil { + platforms = append(platforms, m.Platform.OS+"/"+m.Platform.Architecture) + } + } + info["platforms"] = platforms + info["multiArch"] = true + } + } + } else { + info["multiArch"] = false + } + + c.JSON(http.StatusOK, gin.H{"success": true, "data": info}) +} + +// StreamMultipleImages 批量下载多个镜像 +func (is *ImageStreamer) StreamMultipleImages(ctx context.Context, imageRefs []string, writer io.Writer, options *StreamOptions) error { + if options == nil { + options = &StreamOptions{} + } + + var finalWriter io.Writer = writer + if options.Compression { + gzWriter := gzip.NewWriter(writer) + defer gzWriter.Close() + finalWriter = gzWriter + } + + tarWriter := tar.NewWriter(finalWriter) + defer tarWriter.Close() + + var allManifests []map[string]interface{} + var allRepositories = make(map[string]map[string]string) + + // 流式处理每个镜像 + for i, imageRef := range imageRefs { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + log.Printf("处理镜像 %d/%d: %s", i+1, len(imageRefs), imageRef) + + // 防止单个镜像处理时间过长 + timeoutCtx, cancel := context.WithTimeout(ctx, 15*time.Minute) + manifest, repositories, err := is.streamSingleImageForBatch(timeoutCtx, tarWriter, imageRef, options) + cancel() + + if err != nil { + log.Printf("下载镜像 %s 失败: %v", imageRef, err) + return fmt.Errorf("下载镜像 %s 失败: %w", imageRef, err) + } + + if manifest == nil { + return fmt.Errorf("镜像 %s manifest数据为空", imageRef) + } + + // 收集manifest信息 + allManifests = append(allManifests, manifest) + + // 合并repositories信息 + for repo, tags := range repositories { + if allRepositories[repo] == nil { + allRepositories[repo] = make(map[string]string) + } + for tag, digest := range tags { + allRepositories[repo][tag] = digest + } + } + } + + // 写入合并的manifest.json + manifestData, err := json.Marshal(allManifests) + if err != nil { + return fmt.Errorf("序列化manifest失败: %w", err) + } + + manifestHeader := &tar.Header{ + Name: "manifest.json", + Size: int64(len(manifestData)), + Mode: 0644, + } + + if err := tarWriter.WriteHeader(manifestHeader); err != nil { + return fmt.Errorf("写入manifest header失败: %w", err) + } + + if _, err := tarWriter.Write(manifestData); err != nil { + return fmt.Errorf("写入manifest数据失败: %w", err) + } + + // 写入合并的repositories文件 + repositoriesData, err := json.Marshal(allRepositories) + if err != nil { + return fmt.Errorf("序列化repositories失败: %w", err) + } + + repositoriesHeader := &tar.Header{ + Name: "repositories", + Size: int64(len(repositoriesData)), + Mode: 0644, + } + + if err := tarWriter.WriteHeader(repositoriesHeader); err != nil { + return fmt.Errorf("写入repositories header失败: %w", err) + } + + if _, err := tarWriter.Write(repositoriesData); err != nil { + return fmt.Errorf("写入repositories数据失败: %w", err) + } + + log.Printf("批量下载完成，共处理 %d 个镜像", len(imageRefs)) + return nil +} \ No newline at end of file diff --git a/src/main.go b/src/main.go index 21d0664..3a186f0 100644 --- a/src/main.go +++ b/src/main.go @@ -3,12 +3,15 @@ package main import ( "embed" "fmt" - "github.com/gin-gonic/gin" "io" + "log" "net/http" "regexp" "strconv" "strings" + "time" + + "github.com/gin-gonic/gin" ) //go:embed public/* @@ -42,6 +45,9 @@ var ( regexp.MustCompile(`^(?:https?://)?(github|opengraph)\.githubassets\.com/([^/]+)/.+?$`), } globalLimiter *IPRateLimiter + + // 服务启动时间 + serviceStartTime = time.Now() ) func main() { @@ -60,13 +66,31 @@ func main() { // 初始化Docker流式代理 initDockerProxy() + // 初始化镜像流式下载器 + initImageStreamer() + + // 初始化防抖器 + initDebouncer() + gin.SetMode(gin.ReleaseMode) router := gin.Default() - // 初始化skopeo路由（静态文件和API路由） - initSkopeoRoutes(router) + // 全局Panic恢复保护 + router.Use(gin.CustomRecovery(func(c *gin.Context, recovered interface{}) { + log.Printf("🚨 Panic recovered: %v", recovered) + c.JSON(http.StatusInternalServerError, gin.H{ + "error": "Internal server error", + "code": "INTERNAL_ERROR", + }) + })) + + // 初始化监控端点 + initHealthRoutes(router) - // 静态文件路由（使用嵌入文件） + // 初始化镜像tar下载路由 + initImageTarRoutes(router) + + // 静态文件路由 router.GET("/", func(c *gin.Context) { serveEmbedFile(c, "public/index.html") }) @@ -74,8 +98,9 @@ func main() { filepath := strings.TrimPrefix(c.Param("filepath"), "/") serveEmbedFile(c, "public/"+filepath) }) - router.GET("/skopeo.html", func(c *gin.Context) { - serveEmbedFile(c, "public/skopeo.html") + + router.GET("/images.html", func(c *gin.Context) { + serveEmbedFile(c, "public/images.html") }) router.GET("/search.html", func(c *gin.Context) { serveEmbedFile(c, "public/search.html") @@ -95,11 +120,14 @@ func main() { router.Any("/v2/*path", RateLimitMiddleware(globalLimiter), ProxyDockerRegistryGin) - // 注册NoRoute处理器，应用限流中间件 + // 注册NoRoute处理器 router.NoRoute(RateLimitMiddleware(globalLimiter), handler) cfg := GetConfig() - fmt.Printf("启动成功，项目地址：https://github.com/sky22333/hubproxy \n") + fmt.Printf("🚀 HubProxy 启动成功\n") + fmt.Printf("📡 监听地址: %s:%d\n", cfg.Server.Host, cfg.Server.Port) + fmt.Printf("⚡ 限流配置: %d请求/%g小时\n", cfg.RateLimit.RequestLimit, cfg.RateLimit.PeriodHours) + fmt.Printf("🔗 项目地址: https://github.com/sky22333/hubproxy\n") err := router.Run(fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port)) if err != nil { @@ -198,57 +226,77 @@ func proxyWithRedirect(c *gin.Context, u string, redirectCount int) { resp.Header.Del("Referrer-Policy") resp.Header.Del("Strict-Transport-Security") - // 对于需要处理的shell文件，使用chunked传输 - isShellFile := strings.HasSuffix(strings.ToLower(u), ".sh") - if isShellFile { - resp.Header.Del("Content-Length") - resp.Header.Set("Transfer-Encoding", "chunked") + // 获取真实域名 + realHost := c.Request.Header.Get("X-Forwarded-Host") + if realHost == "" { + realHost = c.Request.Host + } + // 如果域名中没有协议前缀，添加https:// + if !strings.HasPrefix(realHost, "http://") && !strings.HasPrefix(realHost, "https://") { + realHost = "https://" + realHost } - // 复制其他响应头 - for key, values := range resp.Header { - for _, value := range values { - c.Header(key, value) - } - } - - if location := resp.Header.Get("Location"); location != "" { - if checkURL(location) != nil { - c.Header("Location", "/"+location) - } else { - // 递归处理重定向，增加计数防止无限循环 - proxyWithRedirect(c, location, redirectCount+1) - return - } - } - - c.Status(resp.StatusCode) - - // 处理响应体 - if isShellFile { - // 获取真实域名 - realHost := c.Request.Header.Get("X-Forwarded-Host") - if realHost == "" { - realHost = c.Request.Host - } - // 如果域名中没有协议前缀，添加https:// - if !strings.HasPrefix(realHost, "http://") && !strings.HasPrefix(realHost, "https://") { - realHost = "https://" + realHost - } - // 使用ProcessGitHubURLs处理.sh文件 - processedBody, _, err := ProcessGitHubURLs(resp.Body, resp.Header.Get("Content-Encoding") == "gzip", realHost, true) + if strings.HasSuffix(strings.ToLower(u), ".sh") { + isGzipCompressed := resp.Header.Get("Content-Encoding") == "gzip" + + processedBody, processedSize, err := ProcessSmart(resp.Body, isGzipCompressed, realHost) if err != nil { - c.String(http.StatusInternalServerError, fmt.Sprintf("处理shell文件时发生错误: %v", err)) - return + fmt.Printf("智能处理失败，回退到直接代理: %v\n", err) + processedBody = resp.Body + processedSize = 0 } + + // 智能设置响应头 + if processedSize > 0 { + resp.Header.Del("Content-Length") + resp.Header.Del("Content-Encoding") + resp.Header.Set("Transfer-Encoding", "chunked") + } + + // 复制其他响应头 + for key, values := range resp.Header { + for _, value := range values { + c.Header(key, value) + } + } + + if location := resp.Header.Get("Location"); location != "" { + if checkURL(location) != nil { + c.Header("Location", "/"+location) + } else { + proxyWithRedirect(c, location, redirectCount+1) + return + } + } + + c.Status(resp.StatusCode) + + // 输出处理后的内容 if _, err := io.Copy(c.Writer, processedBody); err != nil { - c.String(http.StatusInternalServerError, fmt.Sprintf("写入响应时发生错误: %v", err)) return } } else { - // 对于非.sh文件，直接复制响应体 + for key, values := range resp.Header { + for _, value := range values { + c.Header(key, value) + } + } + + // 处理重定向 + if location := resp.Header.Get("Location"); location != "" { + if checkURL(location) != nil { + c.Header("Location", "/"+location) + } else { + proxyWithRedirect(c, location, redirectCount+1) + return + } + } + + c.Status(resp.StatusCode) + + // 直接流式转发 if _, err := io.Copy(c.Writer, resp.Body); err != nil { - return + fmt.Printf("直接代理失败: %v\n", err) } } } @@ -262,4 +310,72 @@ func checkURL(u string) []string { return nil } - +// 初始化健康监控路由 +func initHealthRoutes(router *gin.Engine) { + // 健康检查端点 + router.GET("/health", func(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{ + "status": "healthy", + "timestamp": time.Now().Unix(), + "uptime": time.Since(serviceStartTime).Seconds(), + "service": "hubproxy", + }) + }) + + // 就绪检查端点 + router.GET("/ready", func(c *gin.Context) { + checks := make(map[string]string) + allReady := true + + if GetConfig() != nil { + checks["config"] = "ok" + } else { + checks["config"] = "failed" + allReady = false + } + + // 检查全局缓存状态 + if globalCache != nil { + checks["cache"] = "ok" + } else { + checks["cache"] = "failed" + allReady = false + } + + // 检查限流器状态 + if globalLimiter != nil { + checks["ratelimiter"] = "ok" + } else { + checks["ratelimiter"] = "failed" + allReady = false + } + + // 检查镜像下载器状态 + if globalImageStreamer != nil { + checks["imagestreamer"] = "ok" + } else { + checks["imagestreamer"] = "failed" + allReady = false + } + + // 检查HTTP客户端状态 + if GetGlobalHTTPClient() != nil { + checks["httpclient"] = "ok" + } else { + checks["httpclient"] = "failed" + allReady = false + } + + status := http.StatusOK + if !allReady { + status = http.StatusServiceUnavailable + } + + c.JSON(status, gin.H{ + "ready": allReady, + "checks": checks, + "timestamp": time.Now().Unix(), + "uptime": time.Since(serviceStartTime).Seconds(), + }) + }) +} diff --git a/src/proxysh.go b/src/proxysh.go index f262134..f3f7f58 100644 --- a/src/proxysh.go +++ b/src/proxysh.go @@ -1,7 +1,7 @@ package main import ( - "bufio" + "bytes" "compress/gzip" "fmt" "io" @@ -9,184 +9,87 @@ import ( "strings" ) -var ( - // gitHubDomains 定义所有支持的GitHub相关域名 - gitHubDomains = []string{ - "github.com", - "raw.githubusercontent.com", - "raw.github.com", - "gist.githubusercontent.com", - "gist.github.com", - "api.github.com", +// GitHub URL正则表达式 +var githubRegex = regexp.MustCompile(`https?://(?:github\.com|raw\.githubusercontent\.com|raw\.github\.com|gist\.githubusercontent\.com|gist\.github\.com|api\.github\.com)[^\s'"]+`) + +// ProcessSmart Shell脚本智能处理函数 +func ProcessSmart(input io.ReadCloser, isCompressed bool, host string) (io.Reader, int64, error) { + defer input.Close() + + content, err := readShellContent(input, isCompressed) + if err != nil { + return nil, 0, fmt.Errorf("内容读取失败: %v", err) } - // urlPattern 使用gitHubDomains构建正则表达式 - urlPattern = regexp.MustCompile(`https?://(?:` + strings.Join(gitHubDomains, "|") + `)[^\s'"]+`) - - // 是否启用脚本嵌套代理的调试日志 - DebugLog = true -) - -// 打印调试日志的辅助函数 -func debugPrintf(format string, args ...interface{}) { - if DebugLog { - fmt.Printf(format, args...) + if len(content) == 0 { + return strings.NewReader(""), 0, nil } + + if len(content) > 10*1024*1024 { + return strings.NewReader(content), int64(len(content)), nil + } + + if !strings.Contains(content, "github.com") && !strings.Contains(content, "githubusercontent.com") { + return strings.NewReader(content), int64(len(content)), nil + } + + processed := processGitHubURLs(content, host) + + return strings.NewReader(processed), int64(len(processed)), nil } -// ProcessGitHubURLs 处理数据流中的GitHub URL，将其替换为代理URL。 -// 此处思路借鉴了 https://github.com/WJQSERVER-STUDIO/ghproxy/blob/main/proxy/nest.go - -func ProcessGitHubURLs(input io.ReadCloser, isCompressed bool, host string, isShellFile bool) (io.Reader, int64, error) { - debugPrintf("开始处理文件: isCompressed=%v, host=%s, isShellFile=%v\n", isCompressed, host, isShellFile) +func readShellContent(input io.ReadCloser, isCompressed bool) (string, error) { + var reader io.Reader = input - if !isShellFile { - debugPrintf("非shell文件，跳过处理\n") - return input, 0, nil - } - - // 使用更大的缓冲区以提高性能 - pipeReader, pipeWriter := io.Pipe() - var written int64 - - go func() { - var err error - defer func() { - if err != nil { - debugPrintf("处理过程中发生错误: %v\n", err) - _ = pipeWriter.CloseWithError(err) - } else { - _ = pipeWriter.Close() - } - }() - - defer input.Close() - - var reader io.Reader = input - if isCompressed { - debugPrintf("检测到压缩文件，进行解压处理\n") - gzipReader, gzipErr := gzip.NewReader(input) - if gzipErr != nil { - err = gzipErr - return - } - defer gzipReader.Close() - reader = gzipReader - } - - // 使用更大的缓冲区 - bufReader := bufio.NewReaderSize(reader, 32*1024) // 32KB buffer - var writer io.Writer = pipeWriter - - if isCompressed { - gzipWriter := gzip.NewWriter(writer) - defer gzipWriter.Close() - writer = gzipWriter - } - - bufWriter := bufio.NewWriterSize(writer, 32*1024) // 32KB buffer - defer bufWriter.Flush() - - written, err = processContent(bufReader, bufWriter, host) - if err != nil { - debugPrintf("处理内容时发生错误: %v\n", err) - return + // 处理gzip压缩 + if isCompressed { + peek := make([]byte, 2) + n, err := input.Read(peek) + if err != nil && err != io.EOF { + return "", fmt.Errorf("读取数据失败: %v", err) } - debugPrintf("文件处理完成，共处理 %d 字节\n", written) - }() - - return pipeReader, written, nil -} - -// processContent 优化处理文件内容的函数 -func processContent(reader *bufio.Reader, writer *bufio.Writer, host string) (int64, error) { - var written int64 - lineNum := 0 - - for { - lineNum++ - line, err := reader.ReadString('\n') - if err != nil && err != io.EOF { - return written, fmt.Errorf("读取行时发生错误: %w", err) - } - - if line != "" { - // 在处理前先检查是否包含GitHub URL - if strings.Contains(line, "github.com") || - strings.Contains(line, "raw.githubusercontent.com") { - matches := urlPattern.FindAllString(line, -1) - if len(matches) > 0 { - debugPrintf("\n在第 %d 行发现 %d 个GitHub URL:\n", lineNum, len(matches)) - for _, match := range matches { - debugPrintf("原始URL: %s\n", match) - } - } - - modifiedLine := processLine(line, host, lineNum) - n, writeErr := writer.WriteString(modifiedLine) - if writeErr != nil { - return written, fmt.Errorf("写入修改后的行时发生错误: %w", writeErr) - } - written += int64(n) - } else { - // 如果行中没有GitHub URL，直接写入 - n, writeErr := writer.WriteString(line) - if writeErr != nil { - return written, fmt.Errorf("写入原始行时发生错误: %w", writeErr) - } - written += int64(n) + if n >= 2 && peek[0] == 0x1f && peek[1] == 0x8b { + combinedReader := io.MultiReader(bytes.NewReader(peek[:n]), input) + gzReader, err := gzip.NewReader(combinedReader) + if err != nil { + return "", fmt.Errorf("gzip解压失败: %v", err) } + defer gzReader.Close() + reader = gzReader + } else { + reader = io.MultiReader(bytes.NewReader(peek[:n]), input) } - - if err == io.EOF { - break - } - } - - // 确保所有数据都被写入 - if err := writer.Flush(); err != nil { - return written, fmt.Errorf("刷新缓冲区时发生错误: %w", err) } - return written, nil + data, err := io.ReadAll(reader) + if err != nil { + return "", fmt.Errorf("读取内容失败: %v", err) + } + + return string(data), nil } -// processLine 处理单行文本，替换所有匹配的GitHub URL -func processLine(line string, host string, lineNum int) string { - return urlPattern.ReplaceAllStringFunc(line, func(url string) string { - newURL := modifyGitHubURL(url, host) - if newURL != url { - debugPrintf("第 %d 行URL替换:\n 原始: %s\n 替换后: %s\n", lineNum, url, newURL) - } - return newURL +func processGitHubURLs(content, host string) string { + return githubRegex.ReplaceAllStringFunc(content, func(url string) string { + return transformURL(url, host) }) } -// 判断代理域名前缀 -func modifyGitHubURL(url string, host string) string { - for _, domain := range gitHubDomains { - hasHttps := strings.HasPrefix(url, "https://"+domain) - hasHttp := strings.HasPrefix(url, "http://"+domain) - - if hasHttps || hasHttp || strings.HasPrefix(url, domain) { - if !hasHttps && !hasHttp { - url = "https://" + url - } - if hasHttp { - url = "https://" + strings.TrimPrefix(url, "http://") - } - // 移除host开头的协议头（如果有） - host = strings.TrimPrefix(host, "https://") - host = strings.TrimPrefix(host, "http://") - // 返回组合后的URL - return host + "/" + url - } +// transformURL URL转换函数 +func transformURL(url, host string) string { + if strings.Contains(url, host) { + return url } - return url -} - -// IsShellFile 检查文件是否为shell文件（基于文件名） -func IsShellFile(filename string) bool { - return strings.HasSuffix(filename, ".sh") + + if strings.HasPrefix(url, "http://") { + url = "https" + url[4:] + } else if !strings.HasPrefix(url, "https://") && !strings.HasPrefix(url, "//") { + url = "https://" + url + } + cleanHost := strings.TrimPrefix(host, "https://") + cleanHost = strings.TrimPrefix(cleanHost, "http://") + cleanHost = strings.TrimSuffix(cleanHost, "/") + + return cleanHost + "/" + url } \ No newline at end of file diff --git a/src/public/images.html b/src/public/images.html new file mode 100644 index 0000000..b82f010 --- /dev/null +++ b/src/public/images.html @@ -0,0 +1,789 @@ + + + + + + + + + Docker离线镜像下载 + + + + + + +
+
+
+

Docker离线镜像下载

+

即点即下，无需等待打包，完全符合docker load加载标准

+ +
+
+ ⚡ + 即时下载 +
+
+ 🔄 + 流式传输 +
+
+ 💾 + 无需打包 +
+
+ 🏗️ + 多架构支持 +
+
+
+ +
+

单镜像下载

+ +
+ +
+
+ + +
+ +
+ + +
+ 常用平台: linux/amd64, linux/arm64, linux/arm/v7 +
+
+ + +
+
+ +
+

多个镜像批量下载

+ +
+ +
+
+ + +
+ +
+ + +
+ 所有镜像将使用相同的目标架构 +
+
+ + +
+
+
+
+ + + + \ No newline at end of file diff --git a/src/public/index.html b/src/public/index.html index b52b9f8..89b8df1 100644 --- a/src/public/index.html +++ b/src/public/index.html @@ -9,483 +9,742 @@ Github文件加速 - - - hosts - 镜像包下载 - 镜像搜索 -
-

Github文件加速

-
- -
- - -
- - - 

-        

-        

-            

-                
支持release、archive文件，支持git clone、wget、curl等等操作
支持Al模型库Hugging Face


+    
 
-        
-    

+    

+        

+            

+                
GitHub 文件加速

+                

+                    快速下载GitHub上的文件和仓库，解决国内访问GitHub速度慢的问题，支持AI模型库Hugging Face
+                

+            

+
+            

+                

+                    

+                        ⚡ 快速生成加速链接
+                    

+                    

+                        输入GitHub文件或仓库链接，自动转换加速链接，可以直接在Github域名前面加上本站域名使用。
+                    

+                

+
+                

+                    

+                        
+                        
+                    

+                

+
+                

+                    

+                        ✅
+                        加速链接已生成
+                    

+                    

+                    

+                        
+                        
+                    

+                

+            

+
+            

+                

+                    

+                        🐳 Docker 镜像加速
+                    

+                    

+                        支持多种Registry，在镜像名前添加本站域名即可加速下载。
+                    

+                

+                
+                
+            

+        

+