idun/main.go

319 lines
7.5 KiB
Go
Raw Normal View History

2023-10-12 08:20:53 +00:00
package main
import (
2023-10-12 09:29:49 +00:00
"bufio"
2023-10-12 08:20:53 +00:00
"errors"
"fmt"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"gopkg.in/yaml.v3"
buckets2 "idun/pkg/buckets"
"idun/pkg/storage"
"os"
2023-10-12 09:29:49 +00:00
"strings"
2023-10-12 08:20:53 +00:00
"time"
)
type config struct {
Jobs []struct {
Name string `yaml:"name"`
Driver string `yaml:"driver"`
Config map[string]string `yaml:"config"`
Buckets buckets2.Config `yaml:"buckets"`
} `yaml:"jobs"`
2023-10-12 09:29:49 +00:00
Timezone string `yaml:"timezone"`
2023-10-12 08:20:53 +00:00
}
func main() {
app := &cli.App{
Commands: []*cli.Command{
{
Name: "plan",
Usage: "Show all Plans of Buckets",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "job",
Value: "",
Usage: "filter for job name",
},
&cli.StringFlag{
Name: "config",
Value: "config.yml",
Usage: "path to config",
},
},
Action: func(cCtx *cli.Context) error {
configPath := cCtx.String("path")
if configPath == "" {
configPath = "config.yml"
}
config, err := readConfig(configPath)
if err != nil {
log.Fatal().Err(err).Msg("cant get config")
return err
}
2023-10-12 09:29:49 +00:00
loc, _ := time.LoadLocation(config.Timezone)
2023-10-12 08:20:53 +00:00
for _, job := range config.Jobs {
if cCtx.String("job") != "" && cCtx.String("job") != job.Name {
continue
}
2023-10-12 09:29:49 +00:00
buckets, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc)
2023-10-12 08:20:53 +00:00
if err != nil {
log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket")
return err
}
fmt.Println("Plan for " + job.Name)
for _, b := range buckets {
fmt.Println(b.ToString())
}
}
return nil
},
},
{
Name: "dry-run",
Usage: "Show which file idun would delete",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "job",
Value: "",
Usage: "filter for job name",
},
&cli.StringFlag{
Name: "config",
Value: "config.yml",
Usage: "path to config",
},
},
Action: func(cCtx *cli.Context) error {
configPath := cCtx.String("path")
if configPath == "" {
configPath = "config.yml"
}
config, err := readConfig(configPath)
if err != nil {
log.Fatal().Err(err).Msg("cant get config")
return err
}
2023-10-12 09:29:49 +00:00
loc, _ := time.LoadLocation(config.Timezone)
2023-10-12 08:20:53 +00:00
for _, job := range config.Jobs {
if cCtx.String("job") != "" && cCtx.String("job") != job.Name {
continue
}
log.Debug().Str("name", job.Name).Msg("Run Job")
2023-10-12 09:29:49 +00:00
bucketList, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc)
2023-10-12 08:20:53 +00:00
if err != nil {
log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket")
return err
}
jobStorage, err := getFileSystem(job.Driver, job.Config)
if err != nil {
log.Fatal().Str("driver", job.Driver).Err(err).Msg("cant get driver")
return err
}
files, err := jobStorage.ListFiles()
if err != nil {
log.Fatal().Err(err).Msg("cant get files")
return err
}
log.Debug().Int("len", len(files)).Msg("got files from jobStorage")
err = buckets2.InsertFilesInBuckets(&bucketList, files)
if err != nil {
log.Fatal().Err(err).Msg("cant insert files to buckets")
}
var allFilesToDeleted []storage.File
for _, b := range bucketList {
filesToDelete, err := b.GetFilesToDelete()
if err != nil {
log.Fatal().Err(err).Msg("cant get files to delete")
return err
}
allFilesToDeleted = append(allFilesToDeleted, filesToDelete...)
}
for _, f := range files {
name, _ := f.GetName()
toDelete := contains(allFilesToDeleted, f)
if toDelete {
fmt.Println("Delete ", name)
} else {
fmt.Println("Keep ", name)
}
}
}
return nil
},
},
2023-10-12 09:29:49 +00:00
{
Name: "execute",
Usage: "Show which file idun would delete",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "job",
Value: "",
Usage: "filter for job name",
},
&cli.StringFlag{
Name: "config",
Value: "config.yml",
Usage: "path to config",
},
2023-10-14 13:24:49 +00:00
&cli.BoolFlag{
Name: "cron",
Value: false,
Usage: "Run as cron, dont ask for yes",
},
2023-10-12 09:29:49 +00:00
},
Action: func(cCtx *cli.Context) error {
configPath := cCtx.String("path")
if configPath == "" {
configPath = "config.yml"
}
config, err := readConfig(configPath)
if err != nil {
log.Fatal().Err(err).Msg("cant get config")
return err
}
loc, _ := time.LoadLocation(config.Timezone)
for _, job := range config.Jobs {
if cCtx.String("job") != "" && cCtx.String("job") != job.Name {
continue
}
log.Debug().Str("name", job.Name).Msg("Run Job")
bucketList, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc)
if err != nil {
log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket")
return err
}
jobStorage, err := getFileSystem(job.Driver, job.Config)
if err != nil {
log.Fatal().Str("driver", job.Driver).Err(err).Msg("cant get driver")
return err
}
files, err := jobStorage.ListFiles()
if err != nil {
log.Fatal().Err(err).Msg("cant get files")
return err
}
log.Debug().Int("len", len(files)).Msg("got files from jobStorage")
err = buckets2.InsertFilesInBuckets(&bucketList, files)
if err != nil {
log.Fatal().Err(err).Msg("cant insert files to buckets")
}
var allFilesToDeleted []storage.File
for _, b := range bucketList {
filesToDelete, err := b.GetFilesToDelete()
if err != nil {
log.Fatal().Err(err).Msg("cant get files to delete")
return err
}
allFilesToDeleted = append(allFilesToDeleted, filesToDelete...)
}
for _, f := range files {
name, _ := f.GetName()
toDelete := contains(allFilesToDeleted, f)
if toDelete {
fmt.Println("Delete ", name)
} else {
fmt.Println("Keep ", name)
}
}
2023-10-14 13:24:49 +00:00
cron := cCtx.Bool("cron")
if !cron {
fmt.Println("Write \"yes\" to execute")
reader := bufio.NewReader(os.Stdin)
text, _ := reader.ReadString('\n')
// convert CRLF to LF
text = strings.Replace(text, "\n", "", -1)
if text == "yes" {
err = jobStorage.Delete(allFilesToDeleted)
if err != nil {
log.Fatal().Err(err).Msg("cant delete files from storage")
return err
}
}
} else {
2023-10-12 09:29:49 +00:00
err = jobStorage.Delete(allFilesToDeleted)
if err != nil {
log.Fatal().Err(err).Msg("cant delete files from storage")
return err
}
}
}
return nil
},
},
2023-10-12 08:20:53 +00:00
},
}
if err := app.Run(os.Args); err != nil {
log.Fatal().Err(err).Msg("cant finish")
}
}
func contains(files []storage.File, file storage.File) bool {
for _, f := range files {
if f == file {
return true
}
}
return false
}
func getFileSystem(driver string, config map[string]string) (storage.Storage, error) {
switch driver {
case "sftp":
sftp := storage.SFTP{Config: config}
return sftp, nil
2023-10-14 13:07:58 +00:00
case "s3":
s3 := storage.S3FileSystem{Config: config}
return s3, nil
2023-10-12 08:20:53 +00:00
}
return nil, errors.New("driver not found")
}
func readConfig(path string) (config, error) {
file, err := os.ReadFile(path)
if err != nil {
return config{}, err
}
c := config{}
err = yaml.Unmarshal(file, &c)
if err != nil {
return config{}, err
}
return c, nil
}