From dd504ec09748abfe4f2b32b3e5124dbd780e94fb Mon Sep 17 00:00:00 2001 From: kekskurse Date: Thu, 12 Oct 2023 11:29:49 +0200 Subject: [PATCH] exec --- Readme.md | 13 ++++- main.go | 110 ++++++++++++++++++++++++++++++++++++-- pkg/buckets/calculator.go | 2 +- pkg/storage/sftp.go | 48 ++++++++++++++++- 4 files changed, 165 insertions(+), 8 deletions(-) diff --git a/Readme.md b/Readme.md index 875f0ce..e7062f0 100644 --- a/Readme.md +++ b/Readme.md @@ -15,7 +15,7 @@ You can configure that for the first X Hour of a Backup livetime. The number of Will keep one Backup per Hour. One Hour is from Min 0 to Min 59, it will keep the olders Backup. If you make Backups at XX:00, XX:15, XX:30 and XX:45 it will keep the Backup at XX:45. ### Daily Backup (config per Day) -Will keep one Backup per Day. One Day is from 00:00:00 UTC to 23:59:59 UTC, it also will keep the olders Backup, tha backup closed to 23:59:59 will stay, all newer Backup at that day will be removed. +Will keep one Backup per Day. One Day is from 00:00:00 UTC to 23:59:59, it also will keep the olders Backup, tha backup closed to 23:59:59 will stay, all newer Backup at that day will be removed. ### Weekly Backup (config per Month)buc @@ -43,10 +43,19 @@ Show all fieles idun dry-run -job abc2 ``` +The job argument is optional and can be used to show just the plan of on job based on the "name" in the config. + ### Execute +Execute the deletion, will ask before deleting the files -### Crongo run +``` +idun execute -job abc2 +``` + +The job argument is optional and can be used to show just the plan of on job based on the "name" in the config. + +### Cron run ## FAQ ### Why minimum Hour/Day/Month diff --git a/main.go b/main.go index 0b8a453..931d917 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "errors" "fmt" "github.com/rs/zerolog/log" @@ -9,6 +10,7 @@ import ( buckets2 "idun/pkg/buckets" "idun/pkg/storage" "os" + "strings" "time" ) @@ -19,10 +21,10 @@ type config struct { Config map[string]string `yaml:"config"` Buckets buckets2.Config `yaml:"buckets"` } `yaml:"jobs"` + Timezone string `yaml:"timezone"` } func main() { - loc, _ := time.LoadLocation("Europe/Berlin") app := &cli.App{ Commands: []*cli.Command{ @@ -53,11 +55,13 @@ func main() { return err } + loc, _ := time.LoadLocation(config.Timezone) + for _, job := range config.Jobs { if cCtx.String("job") != "" && cCtx.String("job") != job.Name { continue } - buckets, err := buckets2.GenerateBuckets(time.Now().AddDate(0, 0, 5), job.Buckets, loc) + buckets, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc) if err != nil { log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket") return err @@ -97,12 +101,14 @@ func main() { return err } + loc, _ := time.LoadLocation(config.Timezone) + for _, job := range config.Jobs { if cCtx.String("job") != "" && cCtx.String("job") != job.Name { continue } log.Debug().Str("name", job.Name).Msg("Run Job") - bucketList, err := buckets2.GenerateBuckets(time.Now().AddDate(0, 0, 5), job.Buckets, loc) + bucketList, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc) if err != nil { log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket") return err @@ -152,6 +158,104 @@ func main() { return nil }, }, + { + Name: "execute", + Usage: "Show which file idun would delete", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "job", + Value: "", + Usage: "filter for job name", + }, + &cli.StringFlag{ + Name: "config", + Value: "config.yml", + Usage: "path to config", + }, + }, + Action: func(cCtx *cli.Context) error { + configPath := cCtx.String("path") + if configPath == "" { + configPath = "config.yml" + } + + config, err := readConfig(configPath) + if err != nil { + log.Fatal().Err(err).Msg("cant get config") + return err + } + + loc, _ := time.LoadLocation(config.Timezone) + + for _, job := range config.Jobs { + if cCtx.String("job") != "" && cCtx.String("job") != job.Name { + continue + } + log.Debug().Str("name", job.Name).Msg("Run Job") + bucketList, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc) + if err != nil { + log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket") + return err + } + + jobStorage, err := getFileSystem(job.Driver, job.Config) + if err != nil { + log.Fatal().Str("driver", job.Driver).Err(err).Msg("cant get driver") + return err + } + + files, err := jobStorage.ListFiles() + if err != nil { + log.Fatal().Err(err).Msg("cant get files") + return err + } + + log.Debug().Int("len", len(files)).Msg("got files from jobStorage") + + err = buckets2.InsertFilesInBuckets(&bucketList, files) + if err != nil { + log.Fatal().Err(err).Msg("cant insert files to buckets") + } + + var allFilesToDeleted []storage.File + + for _, b := range bucketList { + filesToDelete, err := b.GetFilesToDelete() + if err != nil { + log.Fatal().Err(err).Msg("cant get files to delete") + return err + } + allFilesToDeleted = append(allFilesToDeleted, filesToDelete...) + } + + for _, f := range files { + name, _ := f.GetName() + toDelete := contains(allFilesToDeleted, f) + if toDelete { + fmt.Println("Delete ", name) + } else { + fmt.Println("Keep ", name) + } + } + + fmt.Println("Write \"yes\" to execute") + reader := bufio.NewReader(os.Stdin) + text, _ := reader.ReadString('\n') + // convert CRLF to LF + text = strings.Replace(text, "\n", "", -1) + + if text == "yes" { + err = jobStorage.Delete(allFilesToDeleted) + if err != nil { + log.Fatal().Err(err).Msg("cant delete files from storage") + return err + } + } + + } + return nil + }, + }, }, } diff --git a/pkg/buckets/calculator.go b/pkg/buckets/calculator.go index 8c5aaf5..53e6a37 100644 --- a/pkg/buckets/calculator.go +++ b/pkg/buckets/calculator.go @@ -9,7 +9,7 @@ import ( type Config struct { Unlimit int `yaml:"unlimit"` Hourly int `yaml:"hourly"` - Daily int `yaml:"dayly"` + Daily int `yaml:"daily"` Weekly int `yaml:"weekly"` Monthly int `yaml:"monthly"` } diff --git a/pkg/storage/sftp.go b/pkg/storage/sftp.go index d9fb5f0..7a129b0 100644 --- a/pkg/storage/sftp.go +++ b/pkg/storage/sftp.go @@ -1,7 +1,6 @@ package storage import ( - "errors" "fmt" "github.com/pkg/sftp" "golang.org/x/crypto/ssh" @@ -68,5 +67,50 @@ func (f SFTP) ListFiles() ([]File, error) { } func (f SFTP) Delete(files []File) error { - return errors.New("Not implemented") + var auths []ssh.AuthMethod + auths = append(auths, ssh.Password(f.Config["password"])) + + config := ssh.ClientConfig{ + User: f.Config["username"], + Auth: auths, + // Auth: []ssh.AuthMethod{ + // ssh.KeyboardInteractive(SshInteractive), + // }, + + // Uncomment to ignore host key check + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + //HostKeyCallback: ssh.FixedHostKey(hostKey), + // HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { + // return nil + // }, + Timeout: 30 * time.Second, + } + + conn, err := ssh.Dial("tcp", f.Config["addr"], &config) + if err != nil { + return err + } + + defer conn.Close() + + sc, err := sftp.NewClient(conn) + if err != nil { + return err + } + + defer sc.Close() + + for _, f := range files { + name, err := f.GetName() + if err != nil { + return err + } + err = sc.Remove(name) + if err != nil { + return err + } + } + + return nil + }