From 80b3f8959ffbb169e6d35e29cdee2236b0521a24 Mon Sep 17 00:00:00 2001 From: user123456 Date: Fri, 13 Jun 2025 08:22:30 +0800 Subject: [PATCH] =?UTF-8?q?=E9=87=8D=E6=9E=84=E7=A6=BB=E7=BA=BF=E9=95=9C?= =?UTF-8?q?=E5=83=8F=E4=B8=8B=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 3 - install-service.sh | 6 +- src/go.mod | 3 +- src/go.sum | 2 - src/imagetar.go | 577 +++++++++++++++ src/main.go | 12 +- src/public/images.html | 813 +++++++++++++++++++++ src/public/index.html | 2 +- src/public/search.html | 2 +- src/public/skopeo.html | 792 --------------------- src/skopeo_service.go | 1518 ---------------------------------------- 11 files changed, 1404 insertions(+), 2326 deletions(-) create mode 100644 src/imagetar.go create mode 100644 src/public/images.html delete mode 100644 src/public/skopeo.html delete mode 100644 src/skopeo_service.go diff --git a/Dockerfile b/Dockerfile index 2037596..4ba3410 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,6 @@ FROM alpine WORKDIR /root/ -# 安装依赖 -RUN apk add --no-cache skopeo - COPY --from=builder /app/hubproxy . COPY --from=builder /app/config.toml . diff --git a/install-service.sh b/install-service.sh index b1d3fcd..37a9112 100644 --- a/install-service.sh +++ b/install-service.sh @@ -62,7 +62,7 @@ else # 检查依赖 missing_deps=() - for cmd in curl jq tar skopeo; do + for cmd in curl jq tar; do if ! command -v $cmd &> /dev/null; then missing_deps+=($cmd) fi @@ -72,14 +72,14 @@ else echo -e "${YELLOW}检测到缺少依赖: ${missing_deps[*]}${NC}" echo -e "${BLUE}正在自动安装依赖...${NC}" - apt update && apt install -y curl jq skopeo + apt update && apt install -y curl jq if [ $? -ne 0 ]; then echo -e "${RED}依赖安装失败${NC}" exit 1 fi # 重新检查依赖 - for cmd in curl jq tar skopeo; do + for cmd in curl jq tar; do if ! command -v $cmd &> /dev/null; then echo -e "${RED}依赖安装后仍缺少: $cmd${NC}" exit 1 diff --git a/src/go.mod b/src/go.mod index 45c215a..05d2752 100644 --- a/src/go.mod +++ b/src/go.mod @@ -6,10 +6,8 @@ require ( github.com/fsnotify/fsnotify v1.8.0 github.com/gin-gonic/gin v1.10.0 github.com/google/go-containerregistry v0.20.5 - github.com/gorilla/websocket v1.5.1 github.com/pelletier/go-toml/v2 v2.2.3 github.com/spf13/viper v1.20.1 - golang.org/x/sync v0.14.0 golang.org/x/time v0.11.0 ) @@ -55,6 +53,7 @@ require ( golang.org/x/arch v0.8.0 // indirect golang.org/x/crypto v0.32.0 // indirect golang.org/x/net v0.33.0 // indirect + golang.org/x/sync v0.14.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.21.0 // indirect google.golang.org/protobuf v1.36.3 // indirect diff --git a/src/go.sum b/src/go.sum index 5a505da..7176baa 100644 --- a/src/go.sum +++ b/src/go.sum @@ -44,8 +44,6 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/go-containerregistry v0.20.5 h1:4RnlYcDs5hoA++CeFjlbZ/U9Yp1EuWr+UhhTyYQjOP0= github.com/google/go-containerregistry v0.20.5/go.mod h1:Q14vdOOzug02bwnhMkZKD4e30pDaD9W65qzXpyzF49E= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= diff --git a/src/imagetar.go b/src/imagetar.go new file mode 100644 index 0000000..ffd04b9 --- /dev/null +++ b/src/imagetar.go @@ -0,0 +1,577 @@ +package main + +import ( + "archive/tar" + "compress/gzip" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// ImageStreamer 镜像流式下载器 +type ImageStreamer struct { + concurrency int + remoteOptions []remote.Option +} + +// ImageStreamerConfig 下载器配置 +type ImageStreamerConfig struct { + Concurrency int +} + +// NewImageStreamer 创建镜像下载器 +func NewImageStreamer(config *ImageStreamerConfig) *ImageStreamer { + if config == nil { + config = &ImageStreamerConfig{} + } + + concurrency := config.Concurrency + if concurrency <= 0 { + cfg := GetConfig() + concurrency = cfg.Download.MaxImages + if concurrency <= 0 { + concurrency = 10 + } + } + + remoteOptions := []remote.Option{ + remote.WithAuth(authn.Anonymous), + remote.WithTransport(GetGlobalHTTPClient().Transport), + } + + return &ImageStreamer{ + concurrency: concurrency, + remoteOptions: remoteOptions, + } +} + +// StreamOptions 下载选项 +type StreamOptions struct { + Platform string + Compression bool +} + +// StreamImageToWriter 流式下载镜像到Writer +func (is *ImageStreamer) StreamImageToWriter(ctx context.Context, imageRef string, writer io.Writer, options *StreamOptions) error { + if options == nil { + options = &StreamOptions{} + } + + ref, err := name.ParseReference(imageRef) + if err != nil { + return fmt.Errorf("解析镜像引用失败: %w", err) + } + + log.Printf("开始下载镜像: %s", ref.String()) + + contextOptions := append(is.remoteOptions, remote.WithContext(ctx)) + + desc, err := is.getImageDescriptor(ref, contextOptions) + if err != nil { + return fmt.Errorf("获取镜像描述失败: %w", err) + } + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + return is.streamMultiArchImage(ctx, desc, writer, options, contextOptions) + case types.OCIManifestSchema1, types.DockerManifestSchema2: + return is.streamSingleImage(ctx, desc, writer, options, contextOptions) + default: + return is.streamSingleImage(ctx, desc, writer, options, contextOptions) + } +} + +// getImageDescriptor 获取镜像描述符 +func (is *ImageStreamer) getImageDescriptor(ref name.Reference, options []remote.Option) (*remote.Descriptor, error) { + if isCacheEnabled() { + var reference string + if tagged, ok := ref.(name.Tag); ok { + reference = tagged.TagStr() + } else if digested, ok := ref.(name.Digest); ok { + reference = digested.DigestStr() + } + + if reference != "" { + cacheKey := buildManifestCacheKey(ref.Context().String(), reference) + if cachedItem := globalCache.Get(cacheKey); cachedItem != nil { + desc := &remote.Descriptor{ + Manifest: cachedItem.Data, + MediaType: types.MediaType(cachedItem.ContentType), + } + log.Printf("使用缓存的manifest: %s", ref.String()) + return desc, nil + } + } + } + + desc, err := remote.Get(ref, options...) + if err != nil { + return nil, err + } + + if isCacheEnabled() { + var reference string + if tagged, ok := ref.(name.Tag); ok { + reference = tagged.TagStr() + } else if digested, ok := ref.(name.Digest); ok { + reference = digested.DigestStr() + } + + if reference != "" { + cacheKey := buildManifestCacheKey(ref.Context().String(), reference) + ttl := getManifestTTL(reference) + headers := map[string]string{ + "Docker-Content-Digest": desc.Digest.String(), + } + globalCache.Set(cacheKey, desc.Manifest, string(desc.MediaType), headers, ttl) + log.Printf("缓存manifest: %s (TTL: %v)", ref.String(), ttl) + } + } + + return desc, nil +} + +// StreamImageToGin 流式响应到Gin +func (is *ImageStreamer) StreamImageToGin(ctx context.Context, imageRef string, c *gin.Context, options *StreamOptions) error { + if options == nil { + options = &StreamOptions{} + } + + filename := strings.ReplaceAll(imageRef, "/", "_") + ".docker" + c.Header("Content-Type", "application/octet-stream") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + + if options.Compression { + c.Header("Content-Encoding", "gzip") + } + + return is.StreamImageToWriter(ctx, imageRef, c.Writer, options) +} + +// streamMultiArchImage 处理多架构镜像 +func (is *ImageStreamer) streamMultiArchImage(ctx context.Context, desc *remote.Descriptor, writer io.Writer, options *StreamOptions, remoteOptions []remote.Option) error { + index, err := desc.ImageIndex() + if err != nil { + return fmt.Errorf("获取镜像索引失败: %w", err) + } + + manifest, err := index.IndexManifest() + if err != nil { + return fmt.Errorf("获取索引清单失败: %w", err) + } + + // 选择合适的平台 + var selectedDesc *v1.Descriptor + for _, m := range manifest.Manifests { + if m.Platform == nil { + continue + } + + if options.Platform != "" { + platformParts := strings.Split(options.Platform, "/") + if len(platformParts) == 2 && + m.Platform.OS == platformParts[0] && + m.Platform.Architecture == platformParts[1] { + selectedDesc = &m + break + } + } else if m.Platform.OS == "linux" && m.Platform.Architecture == "amd64" { + selectedDesc = &m + break + } + } + + if selectedDesc == nil && len(manifest.Manifests) > 0 { + selectedDesc = &manifest.Manifests[0] + } + + if selectedDesc == nil { + return fmt.Errorf("未找到合适的平台镜像") + } + + img, err := index.Image(selectedDesc.Digest) + if err != nil { + return fmt.Errorf("获取选中镜像失败: %w", err) + } + + return is.streamImageLayers(ctx, img, writer, options) +} + +// streamSingleImage 处理单架构镜像 +func (is *ImageStreamer) streamSingleImage(ctx context.Context, desc *remote.Descriptor, writer io.Writer, options *StreamOptions, remoteOptions []remote.Option) error { + img, err := desc.Image() + if err != nil { + return fmt.Errorf("获取镜像失败: %w", err) + } + + return is.streamImageLayers(ctx, img, writer, options) +} + +// streamImageLayers 处理镜像层 +func (is *ImageStreamer) streamImageLayers(ctx context.Context, img v1.Image, writer io.Writer, options *StreamOptions) error { + var finalWriter io.Writer = writer + + if options.Compression { + gzWriter := gzip.NewWriter(writer) + defer gzWriter.Close() + finalWriter = gzWriter + } + + tarWriter := tar.NewWriter(finalWriter) + defer tarWriter.Close() + + configFile, err := img.ConfigFile() + if err != nil { + return fmt.Errorf("获取镜像配置失败: %w", err) + } + + layers, err := img.Layers() + if err != nil { + return fmt.Errorf("获取镜像层失败: %w", err) + } + + log.Printf("镜像包含 %d 层", len(layers)) + + return is.streamDockerFormat(ctx, tarWriter, img, layers, configFile) +} + +// streamDockerFormat 生成Docker格式 +func (is *ImageStreamer) streamDockerFormat(ctx context.Context, tarWriter *tar.Writer, img v1.Image, layers []v1.Layer, configFile *v1.ConfigFile) error { + configDigest, err := img.ConfigName() + if err != nil { + return err + } + + configData, err := json.Marshal(configFile) + if err != nil { + return err + } + + configHeader := &tar.Header{ + Name: configDigest.String() + ".json", + Size: int64(len(configData)), + Mode: 0644, + } + + if err := tarWriter.WriteHeader(configHeader); err != nil { + return err + } + if _, err := tarWriter.Write(configData); err != nil { + return err + } + + layerDigests := make([]string, len(layers)) + for i, layer := range layers { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + digest, err := layer.Digest() + if err != nil { + return err + } + layerDigests[i] = digest.String() + + layerDir := digest.String() + layerHeader := &tar.Header{ + Name: layerDir + "/", + Typeflag: tar.TypeDir, + Mode: 0755, + } + + if err := tarWriter.WriteHeader(layerHeader); err != nil { + return err + } + + layerReader, err := layer.Uncompressed() + if err != nil { + return err + } + defer layerReader.Close() + + size, err := layer.Size() + if err != nil { + return err + } + + layerTarHeader := &tar.Header{ + Name: layerDir + "/layer.tar", + Size: size, + Mode: 0644, + } + + if err := tarWriter.WriteHeader(layerTarHeader); err != nil { + return err + } + + if _, err := io.Copy(tarWriter, layerReader); err != nil { + return err + } + + log.Printf("已处理层 %d/%d", i+1, len(layers)) + } + + + manifest := []map[string]interface{}{{ + "Config": configDigest.String() + ".json", + "RepoTags": []string{"imported:latest"}, + "Layers": func() []string { + var layers []string + for _, digest := range layerDigests { + layers = append(layers, digest+"/layer.tar") + } + return layers + }(), + }} + + manifestData, err := json.Marshal(manifest) + if err != nil { + return err + } + + manifestHeader := &tar.Header{ + Name: "manifest.json", + Size: int64(len(manifestData)), + Mode: 0644, + } + + if err := tarWriter.WriteHeader(manifestHeader); err != nil { + return err + } + + _, err = tarWriter.Write(manifestData) + return err +} + + + +var globalImageStreamer *ImageStreamer + +// initImageStreamer 初始化镜像下载器 +func initImageStreamer() { + globalImageStreamer = NewImageStreamer(nil) + log.Printf("镜像下载器初始化完成，并发数: %d，缓存: %v", + globalImageStreamer.concurrency, isCacheEnabled()) +} + +// formatPlatformText 格式化平台文本 +func formatPlatformText(platform string) string { + if platform == "" { + return "自动选择" + } + return platform +} + +// initImageTarRoutes 初始化镜像下载路由 +func initImageTarRoutes(router *gin.Engine) { + imageAPI := router.Group("/api/image") + { + imageAPI.GET("/download/:image", RateLimitMiddleware(globalLimiter), handleDirectImageDownload) + imageAPI.GET("/info/:image", RateLimitMiddleware(globalLimiter), handleImageInfo) + imageAPI.POST("/batch", RateLimitMiddleware(globalLimiter), handleSimpleBatchDownload) + } +} + +// handleDirectImageDownload 处理单镜像下载 +func handleDirectImageDownload(c *gin.Context) { + imageParam := c.Param("image") + if imageParam == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "缺少镜像参数"}) + return + } + + imageRef := strings.ReplaceAll(imageParam, "_", "/") + platform := c.Query("platform") + tag := c.DefaultQuery("tag") + + if tag != "" && !strings.Contains(imageRef, ":") && !strings.Contains(imageRef, "@") { + imageRef = imageRef + ":" + tag + } else if !strings.Contains(imageRef, ":") && !strings.Contains(imageRef, "@") { + imageRef = imageRef + ":latest" + } + + if _, err := name.ParseReference(imageRef); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "镜像引用格式错误: " + err.Error()}) + return + } + + options := &StreamOptions{ + Platform: platform, + Compression: false, + } + + ctx := c.Request.Context() + log.Printf("下载镜像: %s (平台: %s)", imageRef, formatPlatformText(platform)) + + if err := globalImageStreamer.StreamImageToGin(ctx, imageRef, c, options); err != nil { + log.Printf("镜像下载失败: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "镜像下载失败: " + err.Error()}) + return + } +} + +// handleSimpleBatchDownload 处理批量下载 +func handleSimpleBatchDownload(c *gin.Context) { + var req struct { + Images []string `json:"images" binding:"required"` + Platform string `json:"platform"` + } + + if err := c.ShouldBindJSON(&req); err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "请求参数错误: " + err.Error()}) + return + } + + if len(req.Images) == 0 { + c.JSON(http.StatusBadRequest, gin.H{"error": "镜像列表不能为空"}) + return + } + + cfg := GetConfig() + if len(req.Images) > cfg.Download.MaxImages { + c.JSON(http.StatusBadRequest, gin.H{ + "error": fmt.Sprintf("镜像数量超过限制，最大允许: %d", cfg.Download.MaxImages), + }) + return + } + + options := &StreamOptions{ + Platform: req.Platform, + Compression: true, + } + + ctx := c.Request.Context() + log.Printf("批量下载 %d 个镜像 (平台: %s)", len(req.Images), formatPlatformText(req.Platform)) + + filename := fmt.Sprintf("batch_%d_images.docker.gz", len(req.Images)) + + c.Header("Content-Type", "application/octet-stream") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) + c.Header("Content-Encoding", "gzip") + + if err := globalImageStreamer.StreamMultipleImages(ctx, req.Images, c.Writer, options); err != nil { + log.Printf("批量镜像下载失败: %v", err) + c.JSON(http.StatusInternalServerError, gin.H{"error": "批量镜像下载失败: " + err.Error()}) + return + } +} + +// handleImageInfo 处理镜像信息查询 +func handleImageInfo(c *gin.Context) { + imageParam := c.Param("image") + if imageParam == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "缺少镜像参数"}) + return + } + + imageRef := strings.ReplaceAll(imageParam, "_", "/") + tag := c.DefaultQuery("tag", "latest") + + if !strings.Contains(imageRef, ":") && !strings.Contains(imageRef, "@") { + imageRef = imageRef + ":" + tag + } + + ref, err := name.ParseReference(imageRef) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "镜像引用格式错误: " + err.Error()}) + return + } + + ctx := c.Request.Context() + contextOptions := append(globalImageStreamer.remoteOptions, remote.WithContext(ctx)) + + desc, err := globalImageStreamer.getImageDescriptor(ref, contextOptions) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "获取镜像信息失败: " + err.Error()}) + return + } + + info := gin.H{ + "name": ref.String(), + "mediaType": desc.MediaType, + "digest": desc.Digest.String(), + "size": desc.Size, + } + + if desc.MediaType == types.OCIImageIndex || desc.MediaType == types.DockerManifestList { + index, err := desc.ImageIndex() + if err == nil { + manifest, err := index.IndexManifest() + if err == nil { + var platforms []string + for _, m := range manifest.Manifests { + if m.Platform != nil { + platforms = append(platforms, m.Platform.OS+"/"+m.Platform.Architecture) + } + } + info["platforms"] = platforms + info["multiArch"] = true + } + } + } else { + info["multiArch"] = false + } + + c.JSON(http.StatusOK, gin.H{"success": true, "data": info}) +} + +// StreamMultipleImages 批量下载多个镜像 +func (is *ImageStreamer) StreamMultipleImages(ctx context.Context, imageRefs []string, writer io.Writer, options *StreamOptions) error { + if options == nil { + options = &StreamOptions{} + } + + var finalWriter io.Writer = writer + if options.Compression { + gzWriter := gzip.NewWriter(writer) + defer gzWriter.Close() + finalWriter = gzWriter + } + + tarWriter := tar.NewWriter(finalWriter) + defer tarWriter.Close() + + for i, imageRef := range imageRefs { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + log.Printf("处理镜像 %d/%d: %s", i+1, len(imageRefs), imageRef) + + dirName := fmt.Sprintf("image_%d_%s/", i, strings.ReplaceAll(imageRef, "/", "_")) + + dirHeader := &tar.Header{ + Name: dirName, + Typeflag: tar.TypeDir, + Mode: 0755, + } + + if err := tarWriter.WriteHeader(dirHeader); err != nil { + return fmt.Errorf("创建镜像目录失败: %w", err) + } + + if err := is.StreamImageToWriter(ctx, imageRef, tarWriter, &StreamOptions{ + Platform: options.Platform, + Compression: false, + }); err != nil { + log.Printf("下载镜像 %s 失败: %v", imageRef, err) + continue + } + } + + return nil +} \ No newline at end of file diff --git a/src/main.go b/src/main.go index 2fcc6ac..b5f48bc 100644 --- a/src/main.go +++ b/src/main.go @@ -60,11 +60,14 @@ func main() { // 初始化Docker流式代理 initDockerProxy() + // 初始化镜像流式下载器 + initImageStreamer() + gin.SetMode(gin.ReleaseMode) router := gin.Default() - // 初始化skopeo路由（静态文件和API路由） - initSkopeoRoutes(router) + // 初始化镜像tar下载路由 + initImageTarRoutes(router) // 静态文件路由（使用嵌入文件） router.GET("/", func(c *gin.Context) { @@ -74,8 +77,9 @@ func main() { filepath := strings.TrimPrefix(c.Param("filepath"), "/") serveEmbedFile(c, "public/"+filepath) }) - router.GET("/skopeo.html", func(c *gin.Context) { - serveEmbedFile(c, "public/skopeo.html") + + router.GET("/images.html", func(c *gin.Context) { + serveEmbedFile(c, "public/images.html") }) router.GET("/search.html", func(c *gin.Context) { serveEmbedFile(c, "public/search.html") diff --git a/src/public/images.html b/src/public/images.html new file mode 100644 index 0000000..f4d314f --- /dev/null +++ b/src/public/images.html @@ -0,0 +1,813 @@ + + + + + + + + + Docker离线镜像下载 + + + + + + + + +
+
+ +
+

Docker离线镜像下载

+

即点即下，无需等待打包，使用docker load加载镜像

+ +
+
+ ⚡ + 即时下载 +
+
+ 🔄 + 流式传输 +
+
+ 💾 + 无需打包 +
+
+ 🏗️ + 多架构支持 +
+
+
+ + +
+

单镜像下载

+ +
+ +
+
+ + +
+ +
+ + +
+ 常用平台: linux/amd64, linux/arm64, linux/arm/v7 +
+
+ + +
+
+ + +
+

批量下载

+ +
+ +
+
+ + +
+ +
+ + +
+ 所有镜像将使用相同的目标平台 +
+
+ + +
+
+
+
+ + + + \ No newline at end of file diff --git a/src/public/index.html b/src/public/index.html index 6cbb012..e7323ab 100644 --- a/src/public/index.html +++ b/src/public/index.html @@ -588,7 +588,7 @@