package main import ( "bufio" "errors" "fmt" "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" "gopkg.in/yaml.v3" buckets2 "idun/pkg/buckets" "idun/pkg/storage" "os" "strings" "time" ) type config struct { Jobs []struct { Name string `yaml:"name"` Driver string `yaml:"driver"` Config map[string]string `yaml:"config"` Buckets buckets2.Config `yaml:"buckets"` } `yaml:"jobs"` Timezone string `yaml:"timezone"` } func main() { app := &cli.App{ Commands: []*cli.Command{ { Name: "plan", Usage: "Show all Plans of Buckets", Flags: []cli.Flag{ &cli.StringFlag{ Name: "job", Value: "", Usage: "filter for job name", }, &cli.StringFlag{ Name: "config", Value: "config.yml", Usage: "path to config", }, }, Action: func(cCtx *cli.Context) error { configPath := cCtx.String("path") if configPath == "" { configPath = "config.yml" } config, err := readConfig(configPath) if err != nil { log.Fatal().Err(err).Msg("cant get config") return err } loc, _ := time.LoadLocation(config.Timezone) for _, job := range config.Jobs { if cCtx.String("job") != "" && cCtx.String("job") != job.Name { continue } buckets, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc) if err != nil { log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket") return err } fmt.Println("Plan for " + job.Name) for _, b := range buckets { fmt.Println(b.ToString()) } } return nil }, }, { Name: "dry-run", Usage: "Show which file idun would delete", Flags: []cli.Flag{ &cli.StringFlag{ Name: "job", Value: "", Usage: "filter for job name", }, &cli.StringFlag{ Name: "config", Value: "config.yml", Usage: "path to config", }, }, Action: func(cCtx *cli.Context) error { configPath := cCtx.String("path") if configPath == "" { configPath = "config.yml" } config, err := readConfig(configPath) if err != nil { log.Fatal().Err(err).Msg("cant get config") return err } loc, _ := time.LoadLocation(config.Timezone) for _, job := range config.Jobs { if cCtx.String("job") != "" && cCtx.String("job") != job.Name { continue } log.Debug().Str("name", job.Name).Msg("Run Job") bucketList, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc) if err != nil { log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket") return err } jobStorage, err := getFileSystem(job.Driver, job.Config) if err != nil { log.Fatal().Str("driver", job.Driver).Err(err).Msg("cant get driver") return err } files, err := jobStorage.ListFiles() if err != nil { log.Fatal().Err(err).Msg("cant get files") return err } log.Debug().Int("len", len(files)).Msg("got files from jobStorage") err = buckets2.InsertFilesInBuckets(&bucketList, files) if err != nil { log.Fatal().Err(err).Msg("cant insert files to buckets") } var allFilesToDeleted []storage.File for _, b := range bucketList { filesToDelete, err := b.GetFilesToDelete() if err != nil { log.Fatal().Err(err).Msg("cant get files to delete") return err } allFilesToDeleted = append(allFilesToDeleted, filesToDelete...) } for _, f := range files { name, _ := f.GetName() toDelete := contains(allFilesToDeleted, f) if toDelete { fmt.Println("Delete ", name) } else { fmt.Println("Keep ", name) } } } return nil }, }, { Name: "execute", Usage: "Show which file idun would delete", Flags: []cli.Flag{ &cli.StringFlag{ Name: "job", Value: "", Usage: "filter for job name", }, &cli.StringFlag{ Name: "config", Value: "config.yml", Usage: "path to config", }, &cli.BoolFlag{ Name: "cron", Value: false, Usage: "Run as cron, dont ask for yes", }, }, Action: func(cCtx *cli.Context) error { configPath := cCtx.String("path") if configPath == "" { configPath = "config.yml" } config, err := readConfig(configPath) if err != nil { log.Fatal().Err(err).Msg("cant get config") return err } loc, _ := time.LoadLocation(config.Timezone) for _, job := range config.Jobs { if cCtx.String("job") != "" && cCtx.String("job") != job.Name { continue } log.Debug().Str("name", job.Name).Msg("Run Job") bucketList, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc) if err != nil { log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket") return err } jobStorage, err := getFileSystem(job.Driver, job.Config) if err != nil { log.Fatal().Str("driver", job.Driver).Err(err).Msg("cant get driver") return err } files, err := jobStorage.ListFiles() if err != nil { log.Fatal().Err(err).Msg("cant get files") return err } log.Debug().Int("len", len(files)).Msg("got files from jobStorage") err = buckets2.InsertFilesInBuckets(&bucketList, files) if err != nil { log.Fatal().Err(err).Msg("cant insert files to buckets") } var allFilesToDeleted []storage.File for _, b := range bucketList { filesToDelete, err := b.GetFilesToDelete() if err != nil { log.Fatal().Err(err).Msg("cant get files to delete") return err } allFilesToDeleted = append(allFilesToDeleted, filesToDelete...) } for _, f := range files { name, _ := f.GetName() toDelete := contains(allFilesToDeleted, f) if toDelete { fmt.Println("Delete ", name) } else { fmt.Println("Keep ", name) } } cron := cCtx.Bool("cron") if !cron { fmt.Println("Write \"yes\" to execute") reader := bufio.NewReader(os.Stdin) text, _ := reader.ReadString('\n') // convert CRLF to LF text = strings.Replace(text, "\n", "", -1) if text == "yes" { err = jobStorage.Delete(allFilesToDeleted) if err != nil { log.Fatal().Err(err).Msg("cant delete files from storage") return err } } } else { err = jobStorage.Delete(allFilesToDeleted) if err != nil { log.Fatal().Err(err).Msg("cant delete files from storage") return err } } } return nil }, }, }, } if err := app.Run(os.Args); err != nil { log.Fatal().Err(err).Msg("cant finish") } } func contains(files []storage.File, file storage.File) bool { for _, f := range files { if f == file { return true } } return false } func getFileSystem(driver string, config map[string]string) (storage.Storage, error) { switch driver { case "sftp": sftp := storage.SFTP{Config: config} return sftp, nil case "s3": s3 := storage.S3FileSystem{Config: config} return s3, nil case "LocaleFileSystem": lfs := storage.LocaleFileSystem{Config: config} return lfs, nil } return nil, errors.New("driver not found") } func readConfig(path string) (config, error) { file, err := os.ReadFile(path) if err != nil { return config{}, err } c := config{} err = yaml.Unmarshal(file, &c) if err != nil { return config{}, err } return c, nil }