From b4b1c47eae82b494ee84afa3d3ac2d4522013642 Mon Sep 17 00:00:00 2001 From: Vladimir Hodakov Date: Tue, 14 Jan 2025 02:57:33 +0400 Subject: [PATCH] Add experimental registry cleanup --- scripts/registry-cleanup/go.mod | 19 +++++ scripts/registry-cleanup/go.sum | 40 +++++++++ scripts/registry-cleanup/main.go | 142 +++++++++++++++++++++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 scripts/registry-cleanup/go.mod create mode 100644 scripts/registry-cleanup/go.sum create mode 100644 scripts/registry-cleanup/main.go diff --git a/scripts/registry-cleanup/go.mod b/scripts/registry-cleanup/go.mod new file mode 100644 index 0000000..c90de41 --- /dev/null +++ b/scripts/registry-cleanup/go.mod @@ -0,0 +1,19 @@ +module source.hodakov.me/hdkv/docker-phpbb/scripts/registry-cleanup + +go 1.23.4 + +require ( + code.gitea.io/sdk/gitea v0.20.0 + github.com/carlmjohnson/requests v0.24.3 + github.com/davecgh/go-spew v1.1.0 +) + +require ( + github.com/42wim/httpsig v1.2.1 // indirect + github.com/davidmz/go-pageant v1.0.2 // indirect + github.com/go-fed/httpsig v1.1.0 // indirect + github.com/hashicorp/go-version v1.6.0 // indirect + golang.org/x/crypto v0.26.0 // indirect + golang.org/x/net v0.27.0 // indirect + golang.org/x/sys v0.23.0 // indirect +) diff --git a/scripts/registry-cleanup/go.sum b/scripts/registry-cleanup/go.sum new file mode 100644 index 0000000..72a5726 --- /dev/null +++ b/scripts/registry-cleanup/go.sum @@ -0,0 +1,40 @@ +code.gitea.io/sdk/gitea v0.20.0 h1:Zm/QDwwZK1awoM4AxdjeAQbxolzx2rIP8dDfmKu+KoU= +code.gitea.io/sdk/gitea v0.20.0/go.mod h1:faouBHC/zyx5wLgjmRKR62ydyvMzwWf3QnU0bH7Cw6U= +github.com/42wim/httpsig v1.2.1 h1:oLBxptMe9U4ZmSGtkosT8Dlfg31P3VQnAGq6psXv82Y= +github.com/42wim/httpsig v1.2.1/go.mod h1:P/UYo7ytNBFwc+dg35IubuAUIs8zj5zzFIgUCEl55WY= +github.com/carlmjohnson/requests v0.24.3 h1:LYcM/jVIVPkioigMjEAnBACXl2vb42TVqiC8EYNoaXQ= +github.com/carlmjohnson/requests v0.24.3/go.mod h1:duYA/jDnyZ6f3xbcF5PpZ9N8clgopubP2nK5i6MVMhU= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= +github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/go-fed/httpsig v1.1.0 h1:9M+hb0jkEICD8/cAiNqEB66R87tTINszBRTjwjQzWcI= +github.com/go-fed/httpsig v1.1.0/go.mod h1:RCMrTZvN1bJYtofsG4rd5NaO5obxQ5xBkdiS7xsT7bM= +github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= +github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= +golang.org/x/term v0.23.0/go.mod h1:DgV24QBUrK6jhZXl+20l6UWznPlwAHm1Q1mGHtydmSk= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scripts/registry-cleanup/main.go b/scripts/registry-cleanup/main.go new file mode 100644 index 0000000..72fd73e --- /dev/null +++ b/scripts/registry-cleanup/main.go @@ -0,0 +1,142 @@ +// Picked from https://github.com/go-gitea/gitea/issues/21673#issuecomment-2331964935 + +package main + +import ( + "context" + "fmt" + "os" + "slices" + "strings" + + "code.gitea.io/sdk/gitea" + "github.com/carlmjohnson/requests" + "github.com/davecgh/go-spew/spew" +) + +var ( + RegistryURL = os.Getenv("REGISTRY_URL") + GiteaURL = os.Getenv("GITEA_URL") + Token = os.Getenv("REGISTRY_TOKEN") +) + +func main() { + if err := run(); err != nil { + fmt.Println(err) + os.Exit(1) + } + + fmt.Println("OK") +} + +func run() error { + ctx := context.TODO() + + var catalog RegistryCatalog + if err := requests.URL(RegistryURL).Path("/v2/_catalog").BasicAuth("", Token).ToJSON(&catalog).Fetch(ctx); err != nil { + return err + } + + var images []string + var orgs []string + for _, repo := range catalog.Repositories { + srepo := strings.Split(repo, "/") + if len(srepo) == 2 { + if !slices.Contains(orgs, srepo[0]) { + orgs = append(orgs, srepo[0]) + } + + var regRepo RegistryRepository + if err := requests.URL(RegistryURL).Pathf("/v2/%s/tags/list", repo).BasicAuth("", Token).ToJSON(®Repo).Fetch(ctx); err != nil { + return err + } + for _, tag := range regRepo.Tags { + var regMan RegistryRepositoryManifest + if err := requests.URL(RegistryURL).Pathf("/v2/%s/manifests/%s", repo, tag).BasicAuth("", Token).ToJSON(®Man).Fetch(ctx); err != nil { + return err + } + for _, man := range regMan.Manifests { + images = append(images, repo+"/"+man.Digest) + } + } + } + } + + var toPurge, toKeep []string + + client, err := gitea.NewClient(GiteaURL, gitea.SetToken(Token)) + if err != nil { + return err + } + + for _, org := range orgs { + page := 1 + pageSize := -1 + + for { + list, _, err := client.ListPackages(org, gitea.ListPackagesOptions{ListOptions: gitea.ListOptions{Page: page}}) + if err != nil { + return err + } + if pageSize < 0 { + pageSize = len(list) + } + if len(list) < pageSize || len(list) == 0 { + break + } + page++ + + for _, pkg := range list { + if pkg.Type == "container" && strings.HasPrefix(pkg.Version, "sha") { + if slices.Contains(images, org+"/"+pkg.Name+"/"+pkg.Version) { + toKeep = append(toKeep, org+"/"+pkg.Name+"/"+pkg.Version) + } else { + toPurge = append(toPurge, org+"/"+pkg.Name+"/"+pkg.Version) + } + } + } + } + } + slices.Sort(images) + slices.Sort(toKeep) + + spew.Dump(images) + spew.Dump(toKeep) + spew.Dump(toPurge) + + /* + + if !slices.Equal(images, toKeep) { + return fmt.Errorf("images from registry don't match with those to keep") + } + + */ + + for _, p := range toPurge { + px := strings.Split(p, "/") + if len(px) == 3 { + if _, err := client.DeletePackage(px[0], "container", px[1], px[2]); err != nil { + return err + } + } + } + + return nil +} + +type RegistryCatalog struct { + Repositories []string `json:"repositories"` +} + +type RegistryRepository struct { + Name string `json:"name"` + Tags []string `json:"tags"` +} + +type RegistryRepositoryManifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int `json:"size"` + Manifests []RegistryRepositoryManifest `json:"manifests"` +}