This commit is contained in:
kekskurse 2023-10-12 11:29:49 +02:00
parent 8ad61755e6
commit dd504ec097
4 changed files with 165 additions and 8 deletions

View file

@ -15,7 +15,7 @@ You can configure that for the first X Hour of a Backup livetime. The number of
Will keep one Backup per Hour. One Hour is from Min 0 to Min 59, it will keep the olders Backup. If you make Backups at XX:00, XX:15, XX:30 and XX:45 it will keep the Backup at XX:45. Will keep one Backup per Hour. One Hour is from Min 0 to Min 59, it will keep the olders Backup. If you make Backups at XX:00, XX:15, XX:30 and XX:45 it will keep the Backup at XX:45.
### Daily Backup (config per Day) ### Daily Backup (config per Day)
Will keep one Backup per Day. One Day is from 00:00:00 UTC to 23:59:59 UTC, it also will keep the olders Backup, tha backup closed to 23:59:59 will stay, all newer Backup at that day will be removed. Will keep one Backup per Day. One Day is from 00:00:00 UTC to 23:59:59, it also will keep the olders Backup, tha backup closed to 23:59:59 will stay, all newer Backup at that day will be removed.
### Weekly Backup (config per Month)buc ### Weekly Backup (config per Month)buc
@ -43,10 +43,19 @@ Show all fieles
idun dry-run -job abc2 idun dry-run -job abc2
``` ```
The job argument is optional and can be used to show just the plan of on job based on the "name" in the config.
### Execute ### Execute
Execute the deletion, will ask before deleting the files
### Crongo run ```
idun execute -job abc2
```
The job argument is optional and can be used to show just the plan of on job based on the "name" in the config.
### Cron run
## FAQ ## FAQ
### Why minimum Hour/Day/Month ### Why minimum Hour/Day/Month

110
main.go
View file

@ -1,6 +1,7 @@
package main package main
import ( import (
"bufio"
"errors" "errors"
"fmt" "fmt"
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
@ -9,6 +10,7 @@ import (
buckets2 "idun/pkg/buckets" buckets2 "idun/pkg/buckets"
"idun/pkg/storage" "idun/pkg/storage"
"os" "os"
"strings"
"time" "time"
) )
@ -19,10 +21,10 @@ type config struct {
Config map[string]string `yaml:"config"` Config map[string]string `yaml:"config"`
Buckets buckets2.Config `yaml:"buckets"` Buckets buckets2.Config `yaml:"buckets"`
} `yaml:"jobs"` } `yaml:"jobs"`
Timezone string `yaml:"timezone"`
} }
func main() { func main() {
loc, _ := time.LoadLocation("Europe/Berlin")
app := &cli.App{ app := &cli.App{
Commands: []*cli.Command{ Commands: []*cli.Command{
@ -53,11 +55,13 @@ func main() {
return err return err
} }
loc, _ := time.LoadLocation(config.Timezone)
for _, job := range config.Jobs { for _, job := range config.Jobs {
if cCtx.String("job") != "" && cCtx.String("job") != job.Name { if cCtx.String("job") != "" && cCtx.String("job") != job.Name {
continue continue
} }
buckets, err := buckets2.GenerateBuckets(time.Now().AddDate(0, 0, 5), job.Buckets, loc) buckets, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc)
if err != nil { if err != nil {
log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket") log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket")
return err return err
@ -97,12 +101,14 @@ func main() {
return err return err
} }
loc, _ := time.LoadLocation(config.Timezone)
for _, job := range config.Jobs { for _, job := range config.Jobs {
if cCtx.String("job") != "" && cCtx.String("job") != job.Name { if cCtx.String("job") != "" && cCtx.String("job") != job.Name {
continue continue
} }
log.Debug().Str("name", job.Name).Msg("Run Job") log.Debug().Str("name", job.Name).Msg("Run Job")
bucketList, err := buckets2.GenerateBuckets(time.Now().AddDate(0, 0, 5), job.Buckets, loc) bucketList, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc)
if err != nil { if err != nil {
log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket") log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket")
return err return err
@ -152,6 +158,104 @@ func main() {
return nil return nil
}, },
}, },
{
Name: "execute",
Usage: "Show which file idun would delete",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "job",
Value: "",
Usage: "filter for job name",
},
&cli.StringFlag{
Name: "config",
Value: "config.yml",
Usage: "path to config",
},
},
Action: func(cCtx *cli.Context) error {
configPath := cCtx.String("path")
if configPath == "" {
configPath = "config.yml"
}
config, err := readConfig(configPath)
if err != nil {
log.Fatal().Err(err).Msg("cant get config")
return err
}
loc, _ := time.LoadLocation(config.Timezone)
for _, job := range config.Jobs {
if cCtx.String("job") != "" && cCtx.String("job") != job.Name {
continue
}
log.Debug().Str("name", job.Name).Msg("Run Job")
bucketList, err := buckets2.GenerateBuckets(time.Now().In(loc), job.Buckets, loc)
if err != nil {
log.Fatal().Err(err).Interface("job", job).Msg("cant create plan for bucket")
return err
}
jobStorage, err := getFileSystem(job.Driver, job.Config)
if err != nil {
log.Fatal().Str("driver", job.Driver).Err(err).Msg("cant get driver")
return err
}
files, err := jobStorage.ListFiles()
if err != nil {
log.Fatal().Err(err).Msg("cant get files")
return err
}
log.Debug().Int("len", len(files)).Msg("got files from jobStorage")
err = buckets2.InsertFilesInBuckets(&bucketList, files)
if err != nil {
log.Fatal().Err(err).Msg("cant insert files to buckets")
}
var allFilesToDeleted []storage.File
for _, b := range bucketList {
filesToDelete, err := b.GetFilesToDelete()
if err != nil {
log.Fatal().Err(err).Msg("cant get files to delete")
return err
}
allFilesToDeleted = append(allFilesToDeleted, filesToDelete...)
}
for _, f := range files {
name, _ := f.GetName()
toDelete := contains(allFilesToDeleted, f)
if toDelete {
fmt.Println("Delete ", name)
} else {
fmt.Println("Keep ", name)
}
}
fmt.Println("Write \"yes\" to execute")
reader := bufio.NewReader(os.Stdin)
text, _ := reader.ReadString('\n')
// convert CRLF to LF
text = strings.Replace(text, "\n", "", -1)
if text == "yes" {
err = jobStorage.Delete(allFilesToDeleted)
if err != nil {
log.Fatal().Err(err).Msg("cant delete files from storage")
return err
}
}
}
return nil
},
},
}, },
} }

View file

@ -9,7 +9,7 @@ import (
type Config struct { type Config struct {
Unlimit int `yaml:"unlimit"` Unlimit int `yaml:"unlimit"`
Hourly int `yaml:"hourly"` Hourly int `yaml:"hourly"`
Daily int `yaml:"dayly"` Daily int `yaml:"daily"`
Weekly int `yaml:"weekly"` Weekly int `yaml:"weekly"`
Monthly int `yaml:"monthly"` Monthly int `yaml:"monthly"`
} }

View file

@ -1,7 +1,6 @@
package storage package storage
import ( import (
"errors"
"fmt" "fmt"
"github.com/pkg/sftp" "github.com/pkg/sftp"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
@ -68,5 +67,50 @@ func (f SFTP) ListFiles() ([]File, error) {
} }
func (f SFTP) Delete(files []File) error { func (f SFTP) Delete(files []File) error {
return errors.New("Not implemented") var auths []ssh.AuthMethod
auths = append(auths, ssh.Password(f.Config["password"]))
config := ssh.ClientConfig{
User: f.Config["username"],
Auth: auths,
// Auth: []ssh.AuthMethod{
// ssh.KeyboardInteractive(SshInteractive),
// },
// Uncomment to ignore host key check
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
//HostKeyCallback: ssh.FixedHostKey(hostKey),
// HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error {
// return nil
// },
Timeout: 30 * time.Second,
}
conn, err := ssh.Dial("tcp", f.Config["addr"], &config)
if err != nil {
return err
}
defer conn.Close()
sc, err := sftp.NewClient(conn)
if err != nil {
return err
}
defer sc.Close()
for _, f := range files {
name, err := f.GetName()
if err != nil {
return err
}
err = sc.Remove(name)
if err != nil {
return err
}
}
return nil
} }