hulthe/lookbuilding
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Refactor into package structure

Browse Source
This commit is contained in:
Joakim Hulthe
2020-11-11 01:46:48 +01:00
parent 04fa50d9ae
commit 27993e9fa0
13 changed files with 329 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
main.go → cmd/lookbuilding/main.go
View File

@ -5,30 +5,29 @@ import (
"net/http"
"os"
"github.com/sirupsen/logrus"
l "hulthe.net/lookbuilding/internal/pkg/logging"
"hulthe.net/lookbuilding/internal/pkg/worker"
)
var (
Logger logrus.Logger = *logrus.New()
)
const EnvAddr = "LOOKBUILDING_ADDR"
func main() {
addr, isPresent := os.LookupEnv(ENV_ADDR)
addr, isPresent := os.LookupEnv(EnvAddr)
if !isPresent {
addr = "0.0.0.0:8000"
}
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
TriggerScan()
worker.TriggerScan()
fmt.Fprintf(w, "OK")
})
fs := http.FileServer(http.Dir("static/"))
http.Handle("/static/", http.StripPrefix("/static/", fs))
Logger.Infof(`listening on %s`, addr)
l.Logger.Infof(`listening on %s`, addr)
go Worker()
go worker.Worker()
err := http.ListenAndServe(addr, nil)
if err != nil {

5
constants.go
View File

@ -1,5 +0,0 @@
package main
const (
ENV_ADDR = "LOOKBUILDING_ADDR"
)

5
go.mod
View File

@ -5,16 +5,13 @@ go 1.15
require (
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/coreos/go-semver v0.3.0
github.com/docker/distribution v2.7.1+incompatible
github.com/docker/docker v1.13.1
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7
github.com/heroku/docker-registry-client v0.0.0-20190909225348-afc9e1acc3d5
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.4.2
golang.org/x/net v0.0.0-20201031054903-ff519b6c9102 // indirect
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
)

1
go.sum
View File

@ -8,6 +8,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/docker/distribution v0.0.0-20171011171712-7484e51bf6af h1:ujR+JcSHkOZMctuIgvi+a/VHpTn0nSy0W7eV5p34xjg=
github.com/docker/distribution v0.0.0-20171011171712-7484e51bf6af/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=

56
internal/pkg/docker/helpers.go Normal file
View File

@ -0,0 +1,56 @@
package docker
import (
"fmt"
"strings"
)
// Extract the repository owner (if any), repository and tag (if any) from a docker image name
func (lc LabeledContainer) SplitImageParts() (*string, string, *string) {
name := lc.Container.Image
var repository string
var owner *string
var tag *string
slashIndex := strings.Index(name, "/")
if slashIndex >= 0 {
tmp := name[:slashIndex]
owner = &tmp
name = name[slashIndex+1:]
}
colonIndex := strings.Index(name, ":")
if colonIndex >= 0 {
tmp := name[colonIndex+1:]
tag = &tmp
repository = name[:colonIndex]
} else {
repository = name
}
return owner, repository, tag
}
func (lc LabeledContainer) GetName() string {
if len(lc.Container.Names) >= 0 {
// trim prefixed "/"
return lc.Container.Names[0][1:]
} else {
return lc.Container.ID[:10]
}
}
func CombineImageParts(owner *string, repository string, tag *string) string {
image := repository
if owner != nil {
image = fmt.Sprintf("%s/%s", *owner, image)
}
if tag != nil {
image = fmt.Sprintf("%s:%s", image, *tag)
}
return image
}

90
docker.go → internal/pkg/docker/rpc.go
View File

@ -1,70 +1,19 @@
package main
package docker
import (
"context"
"fmt"
"strings"
l "hulthe.net/lookbuilding/internal/pkg/logging"
"hulthe.net/lookbuilding/internal/pkg/registry"
"hulthe.net/lookbuilding/internal/pkg/versioning"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/network"
"github.com/docker/docker/client"
)
type LabeledContainer struct {
Container types.Container
Mode VersioningMode
}
// Extract the repository owner (if any), repository and tag (if any) from a docker image name
func (lc LabeledContainer) SplitImageParts() (*string, string, *string) {
name := lc.Container.Image
var repository string
var owner *string
var tag *string
slashIndex := strings.Index(name, "/")
if slashIndex >= 0 {
tmp := name[:slashIndex]
owner = &tmp
name = name[slashIndex+1:]
}
colonIndex := strings.Index(name, ":")
if colonIndex >= 0 {
tmp := name[colonIndex+1:]
tag = &tmp
repository = name[:colonIndex]
} else {
repository = name
}
return owner, repository, tag
}
func (lc LabeledContainer) GetName() string {
if len(lc.Container.Names) >= 0 {
// trim prefixed "/"
return lc.Container.Names[0][1:]
} else {
return lc.Container.ID[:10]
}
}
func combineImageParts(owner *string, repository string, tag *string) string {
image := repository
if owner != nil {
image = fmt.Sprintf("%s/%s", *owner, image)
}
if tag != nil {
image = fmt.Sprintf("%s:%s", image, *tag)
}
return image
}
func getLabeledContainers(cli *client.Client) []LabeledContainer {
func GetLabeledContainers(cli *client.Client) []LabeledContainer {
out := make([]LabeledContainer, 0)
containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{})
@ -72,15 +21,15 @@ func getLabeledContainers(cli *client.Client) []LabeledContainer {
panic(err)
}
Logger.Infof("scanning running container labels")
l.Logger.Infof("scanning running container labels")
for _, container := range containers {
Logger.Debugf("checking %s %s", container.ID[:10], container.Image)
l.Logger.Debugf("checking %s %s", container.ID[:10], container.Image)
for k, v := range container.Labels {
Logger.Debugf(` - "%s": "%s"`, k, v)
if k == versioningModeLabel {
mode := ParseVersioningMode(v)
l.Logger.Debugf(` - "%s": "%s"`, k, v)
if k == versioning.ModeLabel {
mode := versioning.ParseMode(v)
if mode == nil {
Logger.Errorf(`Failed to parse "%s" as a versioning mode`, v)
l.Logger.Errorf(`Failed to parse "%s" as a versioning mode`, v)
continue
}
@ -98,13 +47,13 @@ func getLabeledContainers(cli *client.Client) []LabeledContainer {
return out
}
func (lc LabeledContainer) UpdateTo(cli *client.Client, tag Tag) error {
func (lc LabeledContainer) UpdateTo(cli *client.Client, tag registry.Tag) error {
ctx := context.Background()
owner, repository, _ := lc.SplitImageParts()
image := combineImageParts(owner, repository, &tag.Name)
image := CombineImageParts(owner, repository, &tag.Name)
canonicalImage := fmt.Sprintf("docker.io/%s", image)
Logger.Infof(`pulling image "%s"`, canonicalImage)
l.Logger.Infof(`pulling image "%s"`, canonicalImage)
//containers, err := cli.ContainerList(context.Background(), types.ContainerListOptions{})
imageReader, err := cli.ImagePull(ctx, canonicalImage, types.ImagePullOptions{})
@ -141,13 +90,13 @@ func (lc LabeledContainer) UpdateTo(cli *client.Client, tag Tag) error {
hostConfig := oldContainer.HostConfig
hostConfig.VolumesFrom = []string{tmpOldName}
Logger.Infof(`renaming container %s`, lc.Container.ID)
l.Logger.Infof(`renaming container %s`, lc.Container.ID)
err = cli.ContainerRename(ctx, lc.Container.ID, tmpOldName)
if err != nil {
return err
}
Logger.Infof("creating new container")
l.Logger.Infof("creating new container")
new, err := cli.ContainerCreate(ctx, oldContainer.Config, hostConfig, &network.NetworkingConfig{
EndpointsConfig: oldContainer.NetworkSettings.Networks,
}, name)
@ -156,13 +105,13 @@ func (lc LabeledContainer) UpdateTo(cli *client.Client, tag Tag) error {
return err
}
Logger.Infof("starting new container id: %s", new.ID)
l.Logger.Infof("starting new container id: %s", new.ID)
err = cli.ContainerStart(ctx, new.ID, types.ContainerStartOptions{})
if err != nil {
return err
}
Logger.Infof("removing old container")
l.Logger.Infof("removing old container")
err = cli.ContainerRemove(ctx, oldContainer.ID, types.ContainerRemoveOptions{
RemoveVolumes: false,
RemoveLinks: false,
@ -174,3 +123,4 @@ func (lc LabeledContainer) UpdateTo(cli *client.Client, tag Tag) error {
return nil
}

12
internal/pkg/docker/types.go Normal file
View File

@ -0,0 +1,12 @@
package docker
import (
"hulthe.net/lookbuilding/internal/pkg/versioning"
"github.com/docker/docker/api/types"
)
type LabeledContainer struct {
Container types.Container
Mode versioning.Mode
}

7
internal/pkg/logging/logger.go Normal file
View File

@ -0,0 +1,7 @@
package logging
import "github.com/sirupsen/logrus"
var (
Logger logrus.Logger = *logrus.New()
)

120
internal/pkg/registry/cache.go Normal file
View File

@ -0,0 +1,120 @@
package registry
import (
"hulthe.net/lookbuilding/internal/pkg/semver"
"github.com/heroku/docker-registry-client/registry"
"github.com/opencontainers/go-digest"
"github.com/pkg/errors"
)
type tagListReq struct {
repository string
responder chan<- tagListResp
}
type tagListResp struct {
Data []Tag
Error error
}
type digestReq struct {
repository string
tag string
responder chan<- digestResp
}
type digestResp struct {
Data digest.Digest
Error error
}
type repoCache struct {
Tags []Tag
// Map tags to digests
Digests map[string]digest.Digest
}
type cache struct {
TagListReq chan<- tagListReq
DigestReq chan<- digestReq
}
func newCache(registry registry.Registry) cache {
tagListReq := make(chan tagListReq)
digestReq := make(chan digestReq)
store := map[string]repoCache{}
go func() {
for {
select {
case req := <-tagListReq:
repo, isPresent := store[req.repository]
if isPresent {
// Tag list was already in cache, just return it
req.responder <- tagListResp{Data: repo.Tags}
} else {
// tag list was not in cache, we have to fetch it
tagNames, err := registry.Tags(req.repository)
if err != nil {
req.responder <- tagListResp{
Error: errors.Wrapf(err, `failed to list tags for registry repo "%s"`, req.repository),
}
}
// convert names to Tag{}
var tags []Tag
for _, tagName := range tagNames {
tags = append(tags, Tag{
Name: tagName,
SemVer: semver.ParseTagAsSemVer(tagName),
})
}
// store result in cache
store[req.repository] = repoCache{
Tags: tags,
Digests: map[string]digest.Digest{},
}
req.responder <- tagListResp{
Data: tags,
}
}
case req := <-digestReq:
repo, isPresent := store[req.repository]
if !isPresent {
req.responder <- digestResp{Error: errors.Errorf(
`repo "%s" not present in cache, can't fetch digest'`, req.repository,
)}
}
digest, isPresent := repo.Digests[req.tag]
if isPresent {
req.responder <- digestResp{Data: digest}
} else {
digest, err := registry.ManifestDigest(req.repository, req.tag)
if err != nil {
req.responder <- digestResp{Error: errors.Wrapf(
err, `failed to get digest for repo=%s tag=%s`, req.repository, req.tag,
)}
}
repo.Digests[req.tag] = digest
req.responder <- digestResp{Data: digest}
}
}
}
}()
return cache {
tagListReq,
digestReq,
}
}

60
internal/pkg/registry/client.go Normal file
View File

@ -0,0 +1,60 @@
package registry
import (
"fmt"
l "hulthe.net/lookbuilding/internal/pkg/logging"
"hulthe.net/lookbuilding/internal/pkg/semver"
"github.com/heroku/docker-registry-client/registry"
"github.com/opencontainers/go-digest"
)
type Tag struct {
Name string
SemVer *semver.Tag
repository string
cache cache
}
type Client struct {
cache cache
}
func (tag Tag) GetDigest() (digest.Digest, error) {
responseCh := make(chan digestResp)
tag.cache.DigestReq <- digestReq{ tag.repository, tag.Name, responseCh }
resp := <-responseCh
return resp.Data, resp.Error
}
func (client Client) GetRepoTags(maybeOwner *string, repository string) ([]Tag, error) {
if maybeOwner != nil {
repository = fmt.Sprintf("%s/%s", *maybeOwner, repository)
}
responseCh := make(chan tagListResp)
client.cache.TagListReq <- tagListReq { repository, responseCh }
resp := <-responseCh
return resp.Data, resp.Error
}
func AnonymousClient() (*Client, error) {
url := "https://registry-1.docker.io/"
username := "" // anonymous
password := "" // anonymous
registry, err := registry.New(url, username, password)
if err != nil {
return nil, err
}
registry.Logf = l.Logger.Infof
client := Client{
cache: newCache(*registry),
}
return &client, nil
}

33
versioning_mode.go → internal/pkg/versioning/mode.go
View File

@ -1,18 +1,19 @@
package main
package versioning
import (
"fmt"
"hulthe.net/lookbuilding/internal/pkg/semver"
"sort"
l "hulthe.net/lookbuilding/internal/pkg/logging"
"hulthe.net/lookbuilding/internal/pkg/registry"
"hulthe.net/lookbuilding/internal/pkg/semver"
)
const (
versioningModeLabel = "lookbuilding.mode"
)
const ModeLabel = "lookbuilding.mode"
type VersioningMode interface {
type Mode interface {
Label() string
ShouldUpdate(currentTag string, availableTags []Tag) *Tag
ShouldUpdate(currentTag string, availableTags []registry.Tag) *registry.Tag
}
type SameTag struct{}
@ -21,7 +22,7 @@ type SemVerMinor struct{}
type SemVerPatch struct{}
var (
AllModes = [...]VersioningMode{
AllModes = [...]Mode{
SameTag{},
SemVerMajor{},
SemVerMinor{},
@ -30,18 +31,18 @@ var (
)
func (SameTag) Label() string { return "same_tag" }
func (SameTag) ShouldUpdate(currentTag string, availableTags []Tag) *Tag {
fmt.Println("Not implemented: 'same_tag' versioning mode")
func (SameTag) ShouldUpdate(currentTag string, availableTags []registry.Tag) *registry.Tag {
l.Logger.Errorf("Not implemented: 'same_tag' versioning mode")
return nil // TODO: implement me
}
func semVerShouldUpdate(currentTag string, availableTags []Tag, isValid func(current, available semver.Tag) bool) *Tag {
func semVerShouldUpdate(currentTag string, availableTags []registry.Tag, isValid func(current, available semver.Tag) bool) *registry.Tag {
currentSemVer := semver.ParseTagAsSemVer(currentTag)
if currentSemVer == nil {
return nil
}
semverTags := make([]Tag, 0)
semverTags := make([]registry.Tag, 0)
for _, tag := range availableTags {
if tag.SemVer != nil && isValid(*currentSemVer, *tag.SemVer) {
@ -63,7 +64,7 @@ func semVerShouldUpdate(currentTag string, availableTags []Tag, isValid func(cur
}
func (SemVerMajor) Label() string { return "semver_major" }
func (SemVerMajor) ShouldUpdate(currentTag string, availableTags []Tag) *Tag {
func (SemVerMajor) ShouldUpdate(currentTag string, availableTags []registry.Tag) *registry.Tag {
return semVerShouldUpdate(currentTag, availableTags, func(current, available semver.Tag) bool {
// The new version should be greater
return current.Version.LessThan(available.Version)
@ -71,7 +72,7 @@ func (SemVerMajor) ShouldUpdate(currentTag string, availableTags []Tag) *Tag {
}
func (SemVerMinor) Label() string { return "semver_minor" }
func (SemVerMinor) ShouldUpdate(currentTag string, availableTags []Tag) *Tag {
func (SemVerMinor) ShouldUpdate(currentTag string, availableTags []registry.Tag) *registry.Tag {
return semVerShouldUpdate(currentTag, availableTags, func(current, available semver.Tag) bool {
// The new version should be greater, but still the same major number
return current.Version.LessThan(available.Version) &&
@ -80,7 +81,7 @@ func (SemVerMinor) ShouldUpdate(currentTag string, availableTags []Tag) *Tag {
}
func (SemVerPatch) Label() string { return "semver_patch" }
func (SemVerPatch) ShouldUpdate(currentTag string, availableTags []Tag) *Tag {
func (SemVerPatch) ShouldUpdate(currentTag string, availableTags []registry.Tag) *registry.Tag {
return semVerShouldUpdate(currentTag, availableTags, func(current, available semver.Tag) bool {
// The new version should be greater, but still the same major & minor number
return current.Version.LessThan(available.Version) &&
@ -89,7 +90,7 @@ func (SemVerPatch) ShouldUpdate(currentTag string, availableTags []Tag) *Tag {
})
}
func ParseVersioningMode(input string) *VersioningMode {
func ParseMode(input string) *Mode {
for _, mode := range AllModes {
if mode.Label() == input {
return &mode

49
worker.go → internal/pkg/worker/worker.go
View File

@ -1,8 +1,12 @@
package main
package worker
import (
"github.com/docker/docker/client"
"hulthe.net/lookbuilding/internal/pkg/docker"
l "hulthe.net/lookbuilding/internal/pkg/logging"
"hulthe.net/lookbuilding/internal/pkg/registry"
"hulthe.net/lookbuilding/internal/pkg/semver"
"github.com/docker/docker/client"
)
var (
@ -14,14 +18,14 @@ func TriggerScan() {
}
func Worker() {
Logger.Debugf("background worker starting")
l.Logger.Debugf("background worker starting")
responseCh := make(chan struct{})
workerRunning := false
triggerWaiting := false
for true {
for {
select {
case _ = <-triggerCh:
if workerRunning {
@ -48,55 +52,58 @@ func Worker() {
}
func checkAndDoUpdate() {
Logger.Infof("starting scan")
l.Logger.Infof("starting scan")
cli, err := client.NewEnvClient()
if err != nil {
panic(err)
}
hub, err := anonymousClient()
hub, err := registry.AnonymousClient()
if err != nil {
panic(err)
}
labeledContainers := getLabeledContainers(cli)
labeledContainers := docker.GetLabeledContainers(cli)
Logger.Infof("found %d valid containers", len(labeledContainers))
l.Logger.Infof("found %d valid containers", len(labeledContainers))
for _, lc := range labeledContainers {
owner, repository, tag := lc.SplitImageParts()
name := lc.GetName()
imageName := combineImageParts(owner, repository, nil)
imageName := docker.CombineImageParts(owner, repository, nil)
if tag == nil {
Logger.Errorf(`no tag specified for container "%s", ignoring`, name)
l.Logger.Errorf(`no tag specified for container "%s", ignoring`, name)
continue
}
Logger.Infof(`container "%s" image="%s" mode=%s tag="%s"`, name, imageName, lc.Mode.Label(), *tag)
l.Logger.Infof(`container "%s" image="%s" mode=%s tag="%s"`, name, imageName, lc.Mode.Label(), *tag)
repoTags, err := getDockerRepoTags(hub, owner, repository)
repoTags, err := hub.GetRepoTags(owner, repository)
if err != nil {
panic(err)
}
Logger.Infof(`tags in registry for "%s": %d`, name, len(repoTags))
l.Logger.Infof(`tags in registry for "%s": %d`, name, len(repoTags))
for _, tag := range repoTags {
svt := semver.ParseTagAsSemVer(tag.Name)
Logger.Infof(`tag_name="%s" semver=%t digest=%s`, tag.Name, svt != nil, tag.Digest)
l.Logger.Infof(`tag_name="%s" semver=%t`, tag.Name, svt != nil)
}
shouldUpdateTo := lc.Mode.ShouldUpdate(*tag, repoTags)
if shouldUpdateTo != nil {
Logger.Infof(`updating %s from %s to: %s`, name, *tag, shouldUpdateTo.Name)
err = lc.UpdateTo(cli, *shouldUpdateTo)
if err != nil {
panic(err)
}
l.Logger.Infof(`updating %s from %s to: %s`, name, *tag, shouldUpdateTo.Name)
go func() {
err = lc.UpdateTo(cli, *shouldUpdateTo)
if err != nil {
l.Logger.Error(err)
}
}()
} else {
Logger.Infof("no update available for container %s", name)
l.Logger.Infof("no update available for container %s", name)
}
}
Logger.Infof("all done")
l.Logger.Infof("all done")
}

55
registry.go
View File

@ -1,55 +0,0 @@
package main
import (
"fmt"
"github.com/heroku/docker-registry-client/registry"
digest "github.com/opencontainers/go-digest"
"hulthe.net/lookbuilding/internal/pkg/semver"
)
type Tag struct {
Name string
SemVer *semver.Tag
Digest digest.Digest
}
func anonymousClient() (*registry.Registry, error) {
url := "https://registry-1.docker.io/"
username := "" // anonymous
password := "" // anonymous
registry, err := registry.New(url, username, password)
if err != nil {
return nil, err
}
registry.Logf = Logger.Infof
return registry, nil
}
func getDockerRepoTags(hub *registry.Registry, maybeOwner *string, repository string) ([]Tag, error) {
if maybeOwner != nil {
repository = fmt.Sprintf("%s/%s", *maybeOwner, repository)
}
tags, err := hub.Tags(repository)
if err != nil {
return nil, err
}
var out []Tag
for _, tag := range tags {
digest, err := hub.ManifestDigest(repository, tag)
if err != nil {
return nil, err
}
svt := semver.ParseTagAsSemVer(tag)
out = append(out, Tag{tag, svt, digest})
}
return out, nil
}