powerful but friendly backup program that runs in your tray, powered by restic
devins.page/restray
go
restic
system-tray
fix: various improvements to scheduling and logging
mainly improving consistency between the daemon/cli and gui
also improved some variable and function names to be cleaner / fit better with what their actual purpose is
+268
-104
+109
-33
cmd/restray/cli.go
+109
-33
cmd/restray/cli.go
···
1
1
package main
2
2
3
3
import (
4
+
"bufio"
4
5
"context"
5
6
"errors"
6
7
"fmt"
8
+
"io"
7
9
"log"
8
10
"os"
9
11
"os/exec"
10
12
"os/signal"
11
13
"runtime"
12
14
"strings"
15
+
"sync"
13
16
"sync/atomic"
14
17
"syscall"
15
18
···
109
112
return nil
110
113
}
111
114
115
+
func cliRunResticRetryLock(prof Profile, args []string) error {
116
+
err := cliRunRestic(prof, args...)
117
+
if err != nil && exitCode(err) == 11 && retryLockEnabled(prof) {
118
+
log.Printf("[%s] locked, running unlock and retrying", prof.displayName())
119
+
_ = cliRunRestic(prof, "unlock")
120
+
err = cliRunRestic(prof, args...)
121
+
}
122
+
return err
123
+
}
124
+
125
+
func cliRunCmdLogged(prof Profile, cmd *exec.Cmd) error {
126
+
stdoutPipe, _ := cmd.StdoutPipe()
127
+
stderrPipe, _ := cmd.StderrPipe()
128
+
if err := cmd.Start(); err != nil {
129
+
return cli.Exit("", exitCode(err))
130
+
}
131
+
var wg sync.WaitGroup
132
+
for _, pipe := range []io.Reader{stdoutPipe, stderrPipe} {
133
+
wg.Add(1)
134
+
go func(pipe io.Reader) {
135
+
defer wg.Done()
136
+
scanner := bufio.NewScanner(pipe)
137
+
for scanner.Scan() {
138
+
logPrefixedLine(prof.displayName(), scanner.Text())
139
+
}
140
+
}(pipe)
141
+
}
142
+
err := cmd.Wait()
143
+
wg.Wait()
144
+
if err != nil {
145
+
return cli.Exit("", exitCode(err))
146
+
}
147
+
return nil
148
+
}
149
+
150
+
func cliRunResticLogged(prof Profile, args ...string) error {
151
+
p, _ := findRestic()
152
+
if p == "" {
153
+
return cli.Exit("error: restic not found", 1)
154
+
}
155
+
return cliRunCmdLogged(prof, resticCmd(prof, args...))
156
+
}
157
+
158
+
func cliRunResticRetryLockLogged(prof Profile, args []string) error {
159
+
err := cliRunResticLogged(prof, args...)
160
+
if err != nil && exitCode(err) == 11 && retryLockEnabled(prof) {
161
+
log.Printf("[%s] locked, running unlock and retrying", prof.displayName())
162
+
_ = cliRunResticLogged(prof, "unlock")
163
+
err = cliRunResticLogged(prof, args...)
164
+
}
165
+
return err
166
+
}
167
+
112
168
func cliRunHook(hook string, prof Profile, extraEnv ...string) error {
113
169
if hook == "" {
114
170
return cli.Exit("error: no hook configured", 1)
···
122
178
return nil
123
179
}
124
180
181
+
func cliRunHookLogged(hook string, prof Profile, extraEnv ...string) error {
182
+
if hook == "" {
183
+
return cli.Exit("error: no hook configured", 1)
184
+
}
185
+
return cliRunCmdLogged(prof, hookCmd(hook, prof, extraEnv...))
186
+
}
187
+
125
188
func cliBackup(prof Profile, scheduled bool) error {
126
-
return cliRunRestic(prof, backupArgs(prof, scheduled)...)
189
+
return cliRunResticRetryLock(prof, backupArgs(prof, scheduled))
127
190
}
128
191
129
192
func cliPrune(prof Profile) error {
130
-
return cliRunRestic(prof, forgetArgs(prof)...)
193
+
return cliRunResticRetryLock(prof, addRetryLockArgs(prof, forgetArgs(prof)))
131
194
}
132
195
133
196
func cliCheck(prof Profile) error {
134
-
return cliRunRestic(prof, checkArgs(prof)...)
197
+
return cliRunResticRetryLock(prof, addRetryLockArgs(prof, checkArgs(prof)))
135
198
}
136
199
137
200
func cliUnlock(prof Profile) error {
···
204
267
}()
205
268
206
269
hookEnv := []string{
207
-
"RESTRAY_OPERATIONS=" + strings.Join(scheduledOps(prof), ","),
270
+
"RESTRAY_OPERATIONS=" + strings.Join(scheduledOpNames(prof), ","),
208
271
"RESTRAY_SCHEDULED=true",
209
272
}
210
273
211
274
if prof.PreHook != "" {
212
275
log.Printf("[%s] running pre-hook", prof.displayName())
213
-
if err := cliRunHook(prof.PreHook, prof, hookEnv...); err != nil {
276
+
if err := cliRunHookLogged(prof.PreHook, prof, hookEnv...); err != nil {
214
277
return cli.Exit("error: pre-hook failed", 1)
215
278
}
216
279
}
217
280
218
-
failed := false
219
-
220
-
if !interrupted.Load() && prof.Schedule.BackupEnabled() {
221
-
log.Printf("[%s] running backup", prof.displayName())
222
-
if err := cliBackup(prof, true); err != nil {
223
-
failed = true
224
-
}
225
-
}
226
-
227
-
if !interrupted.Load() && prof.Schedule.Prune && len(prof.Prune.Args) > 0 {
228
-
log.Printf("[%s] running prune", prof.displayName())
229
-
if err := cliPrune(prof); err != nil {
230
-
failed = true
231
-
}
232
-
}
233
-
234
-
if !interrupted.Load() && prof.Schedule.Check {
235
-
log.Printf("[%s] running check", prof.displayName())
236
-
if err := cliCheck(prof); err != nil {
237
-
failed = true
238
-
}
239
-
}
281
+
failed, failMsg := runScheduledOperations(
282
+
prof,
283
+
interrupted.Load,
284
+
func() (bool, string) {
285
+
log.Printf("[%s] running backup", prof.displayName())
286
+
if err := cliRunResticRetryLockLogged(prof, backupArgs(prof, true)); err != nil {
287
+
return false, "backup failed"
288
+
}
289
+
return true, ""
290
+
},
291
+
func() (bool, string) {
292
+
log.Printf("[%s] running prune", prof.displayName())
293
+
if err := cliRunResticRetryLockLogged(prof, addRetryLockArgs(prof, forgetArgs(prof))); err != nil {
294
+
return false, "prune failed"
295
+
}
296
+
return true, ""
297
+
},
298
+
func() (bool, string) {
299
+
log.Printf("[%s] running check", prof.displayName())
300
+
if err := cliRunResticRetryLockLogged(prof, addRetryLockArgs(prof, checkArgs(prof))); err != nil {
301
+
return false, "check failed"
302
+
}
303
+
return true, ""
304
+
},
305
+
)
240
306
241
307
if prof.PostHook != "" {
242
308
signal.Reset(os.Interrupt, syscall.SIGTERM)
243
309
log.Printf("[%s] running post-hook", prof.displayName())
244
310
postEnv := hookEnv
245
311
if failed || interrupted.Load() {
246
-
errMsg := "operation failed"
312
+
errMsg := failMsg
313
+
if errMsg == "" {
314
+
errMsg = "operation failed"
315
+
}
247
316
if interrupted.Load() {
248
317
errMsg = "interrupted by user"
249
318
}
250
319
postEnv = append(postEnv, "RESTRAY_ERROR="+errMsg)
251
320
}
252
-
if err := cliRunHook(prof.PostHook, prof, postEnv...); err != nil {
321
+
if err := cliRunHookLogged(prof.PostHook, prof, postEnv...); err != nil {
253
322
return cli.Exit("error: post-hook failed", 1)
254
323
}
255
324
}
···
267
336
continue
268
337
}
269
338
name := prof.displayName()
339
+
if errMsg := prof.scheduleError(); errMsg != "" {
340
+
log.Printf("[%s] skipping schedule: %s", name, errMsg)
341
+
continue
342
+
}
270
343
_, err := sched.AddFunc(prof.Schedule.Cron, func() {
271
344
current := loadConfig()
272
345
prof, err := resolveProfileAt(current, idx, name)
···
288
361
log.Printf("[%s] scheduled run panicked: %v", prof.displayName(), r)
289
362
}
290
363
}()
291
-
log.Printf("[%s] running scheduled job", prof.displayName())
292
-
cliSchedule(prof)
364
+
log.Printf("[%s] starting scheduled run", prof.displayName())
365
+
if err := cliSchedule(prof); err != nil {
366
+
log.Printf("[%s] scheduled run failed: %v", prof.displayName(), err)
367
+
}
293
368
})
294
369
if err != nil {
295
-
return nil, fmt.Errorf("invalid cron expression %q for profile %q: %w", prof.Schedule.Cron, name, err)
370
+
log.Printf("[%s] skipping schedule: invalid cron expression %q: %v", name, prof.Schedule.Cron, err)
371
+
continue
296
372
}
297
373
}
298
374
if len(sched.Entries()) == 0 {
299
-
return nil, errors.New("no profiles have a schedule configured")
375
+
return nil, errors.New("no profiles have a valid schedule configured")
300
376
}
301
377
return sched, nil
302
378
}
+11
-1
cmd/restray/config.go
+11
-1
cmd/restray/config.go
···
117
117
return env, nil
118
118
}
119
119
120
-
func (prof Profile) configError() string {
120
+
func (prof Profile) scheduleError() string {
121
121
vars, err := parseEnvFile(prof.EnvFile)
122
122
if err != nil {
123
123
return "Env file not found"
···
127
127
}
128
128
if vars["RESTIC_PASSWORD"] == "" && vars["RESTIC_PASSWORD_FILE"] == "" && vars["RESTIC_PASSWORD_COMMAND"] == "" {
129
129
return "No password set in env file"
130
+
}
131
+
if prof.Schedule.BackupEnabled() && len(prof.Backup.Paths) == 0 {
132
+
return "No paths configured"
133
+
}
134
+
return ""
135
+
}
136
+
137
+
func (prof Profile) configError() string {
138
+
if err := prof.scheduleError(); err != "" {
139
+
return err
130
140
}
131
141
if len(prof.Backup.Paths) == 0 {
132
142
return "No paths configured"
+113
-40
cmd/restray/operations.go
+113
-40
cmd/restray/operations.go
···
19
19
"github.com/gen2brain/beeep"
20
20
)
21
21
22
-
type stderrLogger struct {
22
+
type lastLineLogger struct {
23
23
mu sync.Mutex
24
24
last string
25
25
}
26
26
27
-
func (s *stderrLogger) lastLine() string {
27
+
func (s *lastLineLogger) lastLine() string {
28
28
s.mu.Lock()
29
29
defer s.mu.Unlock()
30
30
return s.last
31
31
}
32
32
33
-
func streamStderr(pipe io.Reader, mStatus prefixedMenuItem) *stderrLogger {
34
-
sl := &stderrLogger{}
33
+
func logPrefixedLine(name, line string) {
34
+
log.Printf("[%s] %s", name, line)
35
+
}
36
+
37
+
func logPrefixedOutput(name, out string) {
38
+
for _, line := range strings.Split(strings.ReplaceAll(out, "\r\n", "\n"), "\n") {
39
+
if line != "" {
40
+
logPrefixedLine(name, line)
41
+
}
42
+
}
43
+
}
44
+
45
+
func streamLogLines(pipe io.Reader, name string, onLine func(string)) *lastLineLogger {
46
+
sl := &lastLineLogger{}
35
47
go func() {
36
48
scanner := bufio.NewScanner(pipe)
37
49
for scanner.Scan() {
38
50
line := scanner.Text()
39
-
log.Print(line)
51
+
logPrefixedLine(name, line)
40
52
sl.mu.Lock()
41
53
sl.last = line
42
54
sl.mu.Unlock()
43
-
if trimmed := strings.TrimSpace(line); trimmed != "" {
44
-
mStatus.SetTitle(trimmed)
55
+
if onLine != nil {
56
+
onLine(line)
45
57
}
46
58
}
47
59
}()
48
60
return sl
49
61
}
50
62
63
+
func streamStderr(pipe io.Reader, prof Profile, mStatus prefixedMenuItem) *lastLineLogger {
64
+
return streamLogLines(pipe, prof.displayName(), func(line string) {
65
+
if trimmed := strings.TrimSpace(line); trimmed != "" {
66
+
mStatus.SetTitle(trimmed)
67
+
}
68
+
})
69
+
}
70
+
51
71
func runResticOnce(idx profileIndex, prof Profile, mStatus prefixedMenuItem, args ...string) (string, error) {
52
72
cmd := resticCmd(prof, args...)
53
73
var stdout strings.Builder
···
60
80
return err.Error(), err
61
81
}
62
82
63
-
sl := streamStderr(stderrPipe, mStatus)
83
+
sl := streamStderr(stderrPipe, prof, mStatus)
64
84
err := cmd.Wait()
65
85
if out := stdout.String(); out != "" {
66
-
log.Print(out)
86
+
logPrefixedOutput(prof.displayName(), out)
67
87
}
68
88
if err == nil {
69
89
log.Printf("[%s] done: restic %s", prof.displayName(), args[0])
···
111
131
return false, -1
112
132
}
113
133
114
-
sl := streamStderr(stderrPipe, mStatus)
134
+
sl := streamStderr(stderrPipe, prof, mStatus)
115
135
116
136
scanner := bufio.NewScanner(stdoutPipe)
117
137
for scanner.Scan() {
···
126
146
FilesDone uint64 `json:"files_done"`
127
147
}
128
148
if json.Unmarshal([]byte(line), &msg) != nil || msg.Type == "" {
129
-
log.Print(line)
149
+
logPrefixedLine(prof.displayName(), line)
130
150
continue
131
151
}
132
152
if msg.Type == "status" {
···
141
161
mStatus.SetTitle(fmt.Sprintf("Scanning: %d files", msg.FilesDone))
142
162
}
143
163
} else {
144
-
log.Print(line)
164
+
logPrefixedLine(prof.displayName(), line)
145
165
}
146
166
}
147
167
···
234
254
cmd := hookCmd(hook, prof, extraEnv...)
235
255
log.Printf("[%s] running hook: %s", prof.displayName(), hook)
236
256
out, err := cmd.CombinedOutput()
257
+
if len(out) > 0 {
258
+
logPrefixedOutput(prof.displayName(), string(out))
259
+
}
237
260
if err != nil {
238
-
log.Printf("[%s] hook %q failed: %v: %s", prof.displayName(), hook, err, out)
261
+
log.Printf("[%s] hook %q failed: %v", prof.displayName(), hook, err)
239
262
}
240
263
return err
241
264
}
242
265
243
-
func scheduledOps(prof Profile) []string {
266
+
func scheduledOpNames(prof Profile) []string {
244
267
var ops []string
245
268
if prof.Schedule.BackupEnabled() {
246
269
ops = append(ops, "backup")
···
254
277
return ops
255
278
}
256
279
280
+
func runScheduledOperations(
281
+
prof Profile,
282
+
interrupted func() bool,
283
+
runBackup func() (bool, string),
284
+
runPrune func() (bool, string),
285
+
runCheck func() (bool, string),
286
+
) (bool, string) {
287
+
isInterrupted := func() bool {
288
+
return interrupted != nil && interrupted()
289
+
}
290
+
291
+
failed := false
292
+
failMsg := ""
293
+
skipRemaining := false
294
+
295
+
if !isInterrupted() && prof.Schedule.BackupEnabled() {
296
+
if ok, msg := runBackup(); !ok {
297
+
failed = true
298
+
failMsg = msg
299
+
skipRemaining = true
300
+
}
301
+
}
302
+
if !isInterrupted() && !skipRemaining && prof.Schedule.Prune && len(prof.Prune.Args) > 0 {
303
+
if ok, msg := runPrune(); !ok {
304
+
failed = true
305
+
if failMsg == "" {
306
+
failMsg = msg
307
+
}
308
+
}
309
+
}
310
+
if !isInterrupted() && !skipRemaining && prof.Schedule.Check {
311
+
if ok, msg := runCheck(); !ok {
312
+
failed = true
313
+
if failMsg == "" {
314
+
failMsg = msg
315
+
}
316
+
}
317
+
}
318
+
319
+
return failed, failMsg
320
+
}
321
+
257
322
func runScheduled(idx profileIndex, mStatus prefixedMenuItem, prof Profile, onDone func()) {
258
323
if p, _ := findRestic(); p == "" {
259
324
return
···
266
331
defer releaseProfile(idx)
267
332
268
333
hookEnv := []string{
269
-
"RESTRAY_OPERATIONS=" + strings.Join(scheduledOps(prof), ","),
334
+
"RESTRAY_OPERATIONS=" + strings.Join(scheduledOpNames(prof), ","),
270
335
"RESTRAY_SCHEDULED=true",
271
336
}
272
337
···
279
344
}
280
345
}
281
346
282
-
failed := false
283
-
skipRemaining := false
284
-
if prof.Schedule.BackupEnabled() {
285
-
mStatus.SetTitle("Backing up...")
286
-
if ok, _ := doBackup(idx, mStatus, prof, true); !ok {
287
-
failed = true
288
-
skipRemaining = true
289
-
}
290
-
}
291
-
if !skipRemaining && prof.Schedule.Prune && len(prof.Prune.Args) > 0 {
292
-
mStatus.SetTitle("Pruning repository...")
293
-
if msg, err := runRestic(idx, prof, mStatus, forgetArgs(prof)...); err != nil {
294
-
setProfileFailed(idx, msg)
295
-
failed = true
296
-
}
297
-
}
298
-
if !skipRemaining && prof.Schedule.Check {
299
-
mStatus.SetTitle("Checking repository...")
300
-
if msg, err := runRestic(idx, prof, mStatus, checkArgs(prof)...); err != nil {
301
-
setProfileFailed(idx, msg)
302
-
failed = true
303
-
}
304
-
}
347
+
failed, _ := runScheduledOperations(
348
+
prof,
349
+
nil,
350
+
func() (bool, string) {
351
+
mStatus.SetTitle("Backing up...")
352
+
if ok, _ := doBackup(idx, mStatus, prof, true); !ok {
353
+
return false, getProfileFailStatus(idx)
354
+
}
355
+
return true, ""
356
+
},
357
+
func() (bool, string) {
358
+
mStatus.SetTitle("Pruning repository...")
359
+
if msg, err := runRestic(idx, prof, mStatus, forgetArgs(prof)...); err != nil {
360
+
setProfileFailed(idx, msg)
361
+
return false, msg
362
+
}
363
+
return true, ""
364
+
},
365
+
func() (bool, string) {
366
+
mStatus.SetTitle("Checking repository...")
367
+
if msg, err := runRestic(idx, prof, mStatus, checkArgs(prof)...); err != nil {
368
+
setProfileFailed(idx, msg)
369
+
return false, msg
370
+
}
371
+
return true, ""
372
+
},
373
+
)
305
374
306
375
if failed {
307
376
notifyError("Schedule", getProfileFailStatus(idx))
···
322
391
}
323
392
}
324
393
325
-
func backupArgs(prof Profile, scheduled bool, extra ...string) []string {
326
-
args := append([]string{"backup"}, extra...)
394
+
func addRetryLockArgs(prof Profile, args []string) []string {
327
395
if retryLockEnabled(prof) {
328
396
args = append(args, "--retry-lock", prof.RetryLock)
329
397
}
398
+
return args
399
+
}
400
+
401
+
func backupArgs(prof Profile, scheduled bool, extra ...string) []string {
402
+
args := addRetryLockArgs(prof, append([]string{"backup"}, extra...))
330
403
args = append(args, prof.Backup.Args...)
331
404
if scheduled {
332
405
args = append(args, prof.Backup.ArgsScheduled...)
+35
-30
cmd/restray/tray.go
+35
-30
cmd/restray/tray.go
···
565
565
setProfileTitle(cfg, activeIdx)
566
566
567
567
sched.Start()
568
+
setIconAnimated("busy")
568
569
569
570
var wg sync.WaitGroup
570
571
for i, prof := range cfg.Profiles {
···
584
585
defer wg.Done()
585
586
ps := probeProfileState(cfg, i, prof)
586
587
587
-
if ps.errMsg == "" && prof.Schedule.Cron != "" {
588
-
name := prof.displayName()
589
-
eid, err := sched.AddFunc(prof.Schedule.Cron, func() {
590
-
current := loadConfig()
591
-
prof, err := resolveProfileAt(current, i, name)
588
+
if prof.Schedule.Cron != "" {
589
+
if errMsg := prof.scheduleError(); errMsg != "" {
590
+
log.Printf("[%s] skipping schedule: %s", prof.displayName(), errMsg)
591
+
} else {
592
+
name := prof.displayName()
593
+
eid, err := sched.AddFunc(prof.Schedule.Cron, func() {
594
+
current := loadConfig()
595
+
prof, err := resolveProfileAt(current, i, name)
596
+
if err != nil {
597
+
log.Printf("[%s] skipping scheduled run: %v", name, err)
598
+
return
599
+
}
600
+
if skipForBattery(prof) {
601
+
log.Printf("[%s] skipping scheduled run: on battery power", prof.displayName())
602
+
return
603
+
}
604
+
go func() {
605
+
defer func() {
606
+
if r := recover(); r != nil {
607
+
log.Printf("[%s] scheduled run panicked: %v", prof.displayName(), r)
608
+
}
609
+
}()
610
+
runScheduled(i, statusItem(i), prof, func() { finishProfile(i) })
611
+
}()
612
+
})
592
613
if err != nil {
593
-
log.Printf("[%s] skipping scheduled backup: %v", name, err)
594
-
return
614
+
log.Printf("[%s] invalid cron expression %q: %v", prof.displayName(), prof.Schedule.Cron, err)
615
+
ps.errMsg = "Invalid cron: " + prof.Schedule.Cron
616
+
} else {
617
+
profileMu.Lock()
618
+
profileEntryIDs[i] = append(profileEntryIDs[i], eid)
619
+
profileMu.Unlock()
595
620
}
596
-
if skipForBattery(prof) {
597
-
log.Printf("[%s] skipping scheduled backup: on battery power", prof.displayName())
598
-
return
599
-
}
600
-
go func() {
601
-
defer func() {
602
-
if r := recover(); r != nil {
603
-
log.Printf("[%s] scheduled run panicked: %v", prof.displayName(), r)
604
-
}
605
-
}()
606
-
runScheduled(i, statusItem(i), prof, func() { finishProfile(i) })
607
-
}()
608
-
})
609
-
if err != nil {
610
-
log.Printf("[%s] invalid cron expression %q: %v", prof.displayName(), prof.Schedule.Cron, err)
611
-
ps.errMsg = "Invalid cron: " + prof.Schedule.Cron
612
-
} else {
613
-
profileMu.Lock()
614
-
profileEntryIDs[i] = append(profileEntryIDs[i], eid)
615
-
profileMu.Unlock()
616
621
}
617
622
}
618
623
···
711
716
repoOp := func(status string, args ...string) {
712
717
idx := getActiveProfile()
713
718
prof := selectedProfile()
714
-
if idx >= maxProfiles || isProfileBusy(idx) {
719
+
if idx >= maxProfiles {
715
720
return
716
721
}
717
722
ms := statusItem(idx)
···
738
743
case <-mSchedule.ClickedCh:
739
744
idx := getActiveProfile()
740
745
prof := selectedProfile()
741
-
if idx < maxProfiles && !isProfileBusy(idx) {
746
+
if idx < maxProfiles {
742
747
go runScheduled(idx, statusItem(idx), prof, func() { finishProfile(idx) })
743
748
}
744
749
case <-mBackup.ClickedCh:
745
750
idx := getActiveProfile()
746
751
prof := selectedProfile()
747
-
if idx < maxProfiles && !isProfileBusy(idx) {
752
+
if idx < maxProfiles {
748
753
go runBackup(idx, statusItem(idx), prof, func() { finishProfile(idx) })
749
754
}
750
755
case <-mCancel.ClickedCh: