idun/main.go

package main

import (
	"bufio"
	"errors"
	"fmt"
	"github.com/rs/zerolog/log"
	"github.com/urfave/cli/v2"
	"gopkg.in/yaml.v3"
	buckets2 "idun/pkg/buckets"
	"idun/pkg/storage"
	"os"
	"strings"
	"time"
)

type config struct {
	Jobs []struct {
		Name    string            `yaml:"name"`
		Driver  string            `yaml:"driver"`
		Config  map[string]string `yaml:"config"`
		Buckets buckets2.Config   `yaml:"buckets"`
	} `yaml:"jobs"`
	Timezone string `yaml:"timezone"`
}

func main() {

	app := &cli.App{
		Commands: []*cli.Command{
			{
				Name:  "plan",
				Usage: "Show all Plans of Buckets",
				Flags: []cli.Flag{
					&cli.StringFlag{
						Name:  "job",
						Value: "",
						Usage: "filter for job name",
					},
					&cli.StringFlag{
						Name:  "config",
						Value: "config.yml",
						Usage: "path to config",
					},
				},
				Action: func(cCtx *cli.Context) error {
					configPath := cCtx.String("path")
					if configPath == "" {
						configPath = "config.yml"
					}

					config, err := readConfig(configPath)
					if err != nil {
						log.Fatal().Err(err).Msg("cant get config")
						return err
					}

					loc, _ := time.LoadLocation(config.Timezone)

					for _, job := range config.Jobs {
						if cCtx.String("job") != "" && cCtx.String("job") != job.Name {
							continue
						}
						buckets, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc)
						if err != nil {
							log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket")
							return err
						}
						fmt.Println("Plan for " + job.Name)
						for _, b := range buckets {
							fmt.Println(b.ToString())
						}
					}
					return nil
				},
			},
			{
				Name:  "dry-run",
				Usage: "Show which file idun would delete",
				Flags: []cli.Flag{
					&cli.StringFlag{
						Name:  "job",
						Value: "",
						Usage: "filter for job name",
					},
					&cli.StringFlag{
						Name:  "config",
						Value: "config.yml",
						Usage: "path to config",
					},
				},
				Action: func(cCtx *cli.Context) error {
					configPath := cCtx.String("path")
					if configPath == "" {
						configPath = "config.yml"
					}

					config, err := readConfig(configPath)
					if err != nil {
						log.Fatal().Err(err).Msg("cant get config")
						return err
					}

					loc, _ := time.LoadLocation(config.Timezone)

					for _, job := range config.Jobs {
						if cCtx.String("job") != "" && cCtx.String("job") != job.Name {
							continue
						}
						log.Debug().Str("name", job.Name).Msg("Run Job")
						bucketList, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc)
						if err != nil {
							log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket")
							return err
						}

						jobStorage, err := getFileSystem(job.Driver, job.Config)
						if err != nil {
							log.Fatal().Str("driver", job.Driver).Err(err).Msg("cant get driver")
							return err
						}

						files, err := jobStorage.ListFiles()
						if err != nil {
							log.Fatal().Err(err).Msg("cant get files")
							return err
						}

						log.Debug().Int("len", len(files)).Msg("got files from jobStorage")

						err = buckets2.InsertFilesInBuckets(&bucketList, files)
						if err != nil {
							log.Fatal().Err(err).Msg("cant insert files to buckets")
						}

						var allFilesToDeleted []storage.File

						for _, b := range bucketList {
							filesToDelete, err := b.GetFilesToDelete()
							if err != nil {
								log.Fatal().Err(err).Msg("cant get files to delete")
								return err
							}
							allFilesToDeleted = append(allFilesToDeleted, filesToDelete...)
						}

						for _, f := range files {
							name, _ := f.GetName()
							toDelete := contains(allFilesToDeleted, f)
							if toDelete {
								fmt.Println("Delete ", name)
							} else {
								fmt.Println("Keep ", name)
							}
						}

					}
					return nil
				},
			},
			{
				Name:  "execute",
				Usage: "Show which file idun would delete",
				Flags: []cli.Flag{
					&cli.StringFlag{
						Name:  "job",
						Value: "",
						Usage: "filter for job name",
					},
					&cli.StringFlag{
						Name:  "config",
						Value: "config.yml",
						Usage: "path to config",
					},
				},
				Action: func(cCtx *cli.Context) error {
					configPath := cCtx.String("path")
					if configPath == "" {
						configPath = "config.yml"
					}

					config, err := readConfig(configPath)
					if err != nil {
						log.Fatal().Err(err).Msg("cant get config")
						return err
					}

					loc, _ := time.LoadLocation(config.Timezone)

					for _, job := range config.Jobs {
						if cCtx.String("job") != "" && cCtx.String("job") != job.Name {
							continue
						}
						log.Debug().Str("name", job.Name).Msg("Run Job")
						bucketList, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc)
						if err != nil {
							log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket")
							return err
						}

						jobStorage, err := getFileSystem(job.Driver, job.Config)
						if err != nil {
							log.Fatal().Str("driver", job.Driver).Err(err).Msg("cant get driver")
							return err
						}

						files, err := jobStorage.ListFiles()
						if err != nil {
							log.Fatal().Err(err).Msg("cant get files")
							return err
						}

						log.Debug().Int("len", len(files)).Msg("got files from jobStorage")

						err = buckets2.InsertFilesInBuckets(&bucketList, files)
						if err != nil {
							log.Fatal().Err(err).Msg("cant insert files to buckets")
						}

						var allFilesToDeleted []storage.File

						for _, b := range bucketList {
							filesToDelete, err := b.GetFilesToDelete()
							if err != nil {
								log.Fatal().Err(err).Msg("cant get files to delete")
								return err
							}
							allFilesToDeleted = append(allFilesToDeleted, filesToDelete...)
						}

						for _, f := range files {
							name, _ := f.GetName()
							toDelete := contains(allFilesToDeleted, f)
							if toDelete {
								fmt.Println("Delete ", name)
							} else {
								fmt.Println("Keep ", name)
							}
						}

						fmt.Println("Write \"yes\" to execute")
						reader := bufio.NewReader(os.Stdin)
						text, _ := reader.ReadString('\n')
						// convert CRLF to LF
						text = strings.Replace(text, "\n", "", -1)

						if text == "yes" {
							err = jobStorage.Delete(allFilesToDeleted)
							if err != nil {
								log.Fatal().Err(err).Msg("cant delete files from storage")
								return err
							}
						}

					}
					return nil
				},
			},
		},
	}

	if err := app.Run(os.Args); err != nil {
		log.Fatal().Err(err).Msg("cant finish")
	}
}

func contains(files []storage.File, file storage.File) bool {
	for _, f := range files {
		if f == file {
			return true
		}
	}
	return false
}

func getFileSystem(driver string, config map[string]string) (storage.Storage, error) {
	switch driver {
	case "sftp":
		sftp := storage.SFTP{Config: config}
		return sftp, nil
	case "s3":
		s3 := storage.S3FileSystem{Config: config}
		return s3, nil

	}

	return nil, errors.New("driver not found")
}

func readConfig(path string) (config, error) {
	file, err := os.ReadFile(path)
	if err != nil {
		return config{}, err
	}

	c := config{}
	err = yaml.Unmarshal(file, &c)
	if err != nil {
		return config{}, err
	}

	return c, nil
}