powerful but friendly backup program that runs in your tray, powered by restic
devins.page/restray
go
restic
system-tray
fix: more scheduler issues on macos
+108
-133
+85
-89
restic.go
+85
-89
restic.go
···
323
323
324
324
func setProfileBusy(idx int, active bool) {
325
325
state.mu.Lock()
326
+
applyBusy(idx, active)
327
+
}
328
+
329
+
func acquireProfile(idx int) bool {
330
+
state.mu.Lock()
331
+
if _, busy := state.busyProfiles[idx]; busy {
332
+
state.mu.Unlock()
333
+
return false
334
+
}
335
+
applyBusy(idx, true)
336
+
return true
337
+
}
338
+
339
+
func applyBusy(idx int, active bool) {
326
340
wasBusy := len(state.busyProfiles) > 0
327
341
if active {
328
342
state.busyProfiles[idx] = nil
···
390
404
return -1
391
405
}
392
406
407
+
func killedBySystem(err error) bool {
408
+
return err != nil && exitCode(err) == -1 && strings.Contains(err.Error(), "killed")
409
+
}
410
+
393
411
func classifyError(err error, lastStderr string) (string, int) {
394
412
code := exitCode(err)
395
-
if code == -1 && strings.Contains(err.Error(), "killed") {
413
+
if killedBySystem(err) {
396
414
return "Process terminated by system", code
397
415
}
398
416
if code == 12 {
···
461
479
retryUnlock(idx, prof, mStatus)
462
480
msg, err = runResticOnce(idx, prof, mStatus, args...)
463
481
}
482
+
if killedBySystem(err) {
483
+
log.Printf("[%s] %s killed by system, retrying once", prof.displayName(), args[0])
484
+
msg, err = runResticOnce(idx, prof, mStatus, args...)
485
+
}
464
486
return msg, err
465
487
}
466
488
···
537
559
return true, 0
538
560
}
539
561
540
-
func doBackup(idx int, mStatus prefixedMenuItem, prof Profile, scheduled bool) bool {
562
+
func doBackup(idx int, mStatus prefixedMenuItem, prof Profile, scheduled bool) (bool, int) {
541
563
args := []string{"backup", "--json"}
542
564
if retryLockEnabled(prof) {
543
565
args = append(args, "--retry-lock", prof.RetryLock)
···
555
577
setProfileFailed(idx, "")
556
578
ok, code = doBackupOnce(idx, mStatus, prof, args)
557
579
}
558
-
if !scheduled {
559
-
switch {
560
-
case !ok:
561
-
notifyError("Backup", getProfileFailStatus(idx))
562
-
case code == 3:
563
-
notifyError("Backup", "Completed with warnings: some files could not be read")
564
-
default:
565
-
notifySuccess("Backup")
566
-
}
580
+
if !ok && code == -1 {
581
+
log.Printf("[%s] backup killed by system, retrying once", prof.displayName())
582
+
mStatus.SetTitle("Backing up...")
583
+
setProfileFailed(idx, "")
584
+
ok, code = doBackupOnce(idx, mStatus, prof, args)
567
585
}
568
-
return ok
586
+
return ok, code
569
587
}
570
588
571
-
func runBackup(idx int, mStatus prefixedMenuItem, prof Profile, onDone func()) {
572
-
if p, _ := findRestic(); p == "" {
573
-
return
574
-
}
575
-
if isProfileBusy(idx) {
589
+
func runWithHooks(idx int, mStatus prefixedMenuItem, prof Profile, onDone func(),
590
+
label, opsEnv string, scheduled bool, op func() (bool, string)) {
591
+
if !acquireProfile(idx) {
576
592
return
577
593
}
578
-
579
594
setProfileFailed(idx, "")
580
-
setProfileBusy(idx, true)
581
595
defer onDone()
582
596
defer setProfileBusy(idx, false)
583
597
584
-
hookEnv := []string{"RESTRAY_OPERATIONS=backup", "RESTRAY_SCHEDULED=false"}
598
+
scheduledEnv := "false"
599
+
if scheduled {
600
+
scheduledEnv = "true"
601
+
}
602
+
hookEnv := []string{"RESTRAY_OPERATIONS=" + opsEnv, "RESTRAY_SCHEDULED=" + scheduledEnv}
585
603
586
604
if prof.PreHook != "" {
587
605
mStatus.SetTitle("Running pre-hook...")
588
606
if err := runHook(prof.PreHook, prof, hookEnv...); err != nil {
589
607
setProfileFailed(idx, "Pre-hook failed")
590
-
notifyError("Backup", "Pre-hook failed")
608
+
notifyError(label, "Pre-hook failed")
591
609
return
592
610
}
593
611
}
594
612
595
-
mStatus.SetTitle("Backing up...")
596
-
ok := doBackup(idx, mStatus, prof, false)
613
+
ok, successMsg := op()
614
+
615
+
if ok {
616
+
if successMsg != "" {
617
+
notifyError(label, successMsg)
618
+
} else {
619
+
notifySuccess(label)
620
+
}
621
+
} else {
622
+
notifyError(label, getProfileFailStatus(idx))
623
+
}
597
624
598
625
if prof.PostHook != "" {
599
626
mStatus.SetTitle("Running post-hook...")
···
605
632
}
606
633
if hookErr != nil {
607
634
setProfileFailed(idx, "Post-hook failed")
608
-
notifyError("Backup", "Post-hook failed")
635
+
notifyError(label, "Post-hook failed")
609
636
}
610
637
}
611
638
}
612
639
640
+
func runBackup(idx int, mStatus prefixedMenuItem, prof Profile, onDone func()) {
641
+
if p, _ := findRestic(); p == "" {
642
+
return
643
+
}
644
+
runWithHooks(idx, mStatus, prof, onDone, "Backup", "backup", false, func() (bool, string) {
645
+
mStatus.SetTitle("Backing up...")
646
+
ok, code := doBackup(idx, mStatus, prof, false)
647
+
if ok && code == 3 {
648
+
return true, "Completed with warnings: some files could not be read"
649
+
}
650
+
return ok, ""
651
+
})
652
+
}
653
+
613
654
func runHook(hook string, prof Profile, extraEnv ...string) error {
614
655
if hook == "" {
615
656
return nil
···
651
692
if p, _ := findRestic(); p == "" {
652
693
return
653
694
}
654
-
if isProfileBusy(idx) {
655
-
return
656
-
}
657
-
658
-
setProfileFailed(idx, "")
659
-
setProfileBusy(idx, true)
660
-
defer onDone()
661
-
defer setProfileBusy(idx, false)
662
695
663
696
var ops []string
664
697
if prof.Schedule.BackupEnabled() {
···
671
704
ops = append(ops, "check")
672
705
}
673
706
674
-
hookEnv := []string{"RESTRAY_SCHEDULED=true", "RESTRAY_OPERATIONS=" + strings.Join(ops, ",")}
675
-
676
-
if prof.PreHook != "" {
677
-
mStatus.SetTitle("Running pre-hook...")
678
-
if err := runHook(prof.PreHook, prof, hookEnv...); err != nil {
679
-
setProfileFailed(idx, "Pre-hook failed")
680
-
notifyError("Schedule", "Pre-hook failed")
681
-
return
707
+
runWithHooks(idx, mStatus, prof, onDone, "Schedule", strings.Join(ops, ","), true, func() (bool, string) {
708
+
if prof.Schedule.BackupEnabled() {
709
+
mStatus.SetTitle("Backing up...")
710
+
if ok, _ := doBackup(idx, mStatus, prof, true); !ok {
711
+
return false, ""
712
+
}
682
713
}
683
-
}
684
-
685
-
failed := false
686
-
687
-
if prof.Schedule.BackupEnabled() {
688
-
mStatus.SetTitle("Backing up...")
689
-
if !doBackup(idx, mStatus, prof, true) {
690
-
notifyError("Schedule", getProfileFailStatus(idx))
691
-
if prof.PostHook != "" {
692
-
mStatus.SetTitle("Running post-hook...")
693
-
if runHook(prof.PostHook, prof, append(hookEnv, "RESTRAY_STATUS=failure", "RESTRAY_ERROR="+getProfileFailStatus(idx))...) != nil {
694
-
notifyError("Schedule", "Post-hook failed")
695
-
}
714
+
failed := false
715
+
if prof.Schedule.Prune && len(prof.Prune.Args) > 0 {
716
+
mStatus.SetTitle("Pruning repository...")
717
+
if msg, err := runRestic(idx, prof, mStatus, forgetArgs(prof)...); err != nil {
718
+
setProfileFailed(idx, msg)
719
+
failed = true
696
720
}
697
-
return
698
721
}
699
-
}
700
-
701
-
if prof.Schedule.Prune && len(prof.Prune.Args) > 0 {
702
-
mStatus.SetTitle("Pruning repository...")
703
-
if msg, err := runRestic(idx, prof, mStatus, forgetArgs(prof)...); err != nil {
704
-
setProfileFailed(idx, msg)
705
-
failed = true
706
-
}
707
-
}
708
-
if prof.Schedule.Check {
709
-
mStatus.SetTitle("Checking repository...")
710
-
if msg, err := runRestic(idx, prof, mStatus, checkArgs(prof)...); err != nil {
711
-
setProfileFailed(idx, msg)
712
-
failed = true
722
+
if prof.Schedule.Check {
723
+
mStatus.SetTitle("Checking repository...")
724
+
if msg, err := runRestic(idx, prof, mStatus, checkArgs(prof)...); err != nil {
725
+
setProfileFailed(idx, msg)
726
+
failed = true
727
+
}
713
728
}
714
-
}
715
-
716
-
if failed {
717
-
notifyError("Schedule", getProfileFailStatus(idx))
718
-
} else {
719
-
notifySuccess("Schedule")
720
-
}
721
-
722
-
if prof.PostHook != "" {
723
-
mStatus.SetTitle("Running post-hook...")
724
-
var hookErr error
725
-
if failed {
726
-
hookErr = runHook(prof.PostHook, prof, append(hookEnv, "RESTRAY_STATUS=failure", "RESTRAY_ERROR="+getProfileFailStatus(idx))...)
727
-
} else {
728
-
hookErr = runHook(prof.PostHook, prof, append(hookEnv, "RESTRAY_STATUS=success")...)
729
-
}
730
-
if hookErr != nil {
731
-
setProfileFailed(idx, "Post-hook failed")
732
-
notifyError("Schedule", "Post-hook failed")
733
-
}
734
-
}
729
+
return !failed, ""
730
+
})
735
731
}
736
732
737
733
func forgetArgs(prof Profile) []string {
+23
-44
tray.go
+23
-44
tray.go
···
223
223
if idx < len(cfg.Profile) {
224
224
return cfg.Profile[idx]
225
225
}
226
-
return cfg.Profile[0]
226
+
if len(cfg.Profile) > 0 {
227
+
return cfg.Profile[0]
228
+
}
229
+
return Profile{}
227
230
}
228
231
229
232
statusItem := func(idx int) prefixedMenuItem {
···
463
466
log.Printf("[%s] skipping scheduled backup: on battery power", current.Profile[idx].displayName())
464
467
return
465
468
}
466
-
runScheduled(idx, statusItem(idx), current.Profile[idx], applyConfig)
469
+
go func() {
470
+
defer func() {
471
+
if r := recover(); r != nil {
472
+
log.Printf("[%s] scheduled run panicked: %v", current.Profile[idx].displayName(), r)
473
+
}
474
+
}()
475
+
runScheduled(idx, statusItem(idx), current.Profile[idx], applyConfig)
476
+
}()
467
477
})
468
478
if err != nil {
469
479
log.Printf("[%s] invalid cron expression %q: %v", prof.displayName(), prof.Schedule.Cron, err)
···
526
536
527
537
go func() {
528
538
prev := onBatteryPower()
529
-
lastTick := time.Now()
539
+
lastWall := time.Now().Round(0)
530
540
for range time.NewTicker(60 * time.Second).C {
531
-
now := time.Now()
532
-
elapsed := now.Sub(lastTick)
533
-
lastTick = now
541
+
now := time.Now().Round(0)
542
+
drift := now.Sub(lastWall) - 60*time.Second
543
+
lastWall = now
534
544
535
-
if elapsed > 2*time.Minute || elapsed < 0 {
536
-
log.Print("clock jump detected, restarting scheduler")
545
+
if drift.Abs() > 30*time.Second && !isAnyBusy() {
546
+
log.Print("clock jump detected, rebuilding scheduler")
537
547
applyConfig()
538
548
continue
539
549
}
···
557
567
}
558
568
prof := selectedProfile()
559
569
ms := statusItem(idx)
560
-
go func() {
561
-
setProfileBusy(idx, true)
562
-
defer applyConfig()
563
-
defer setProfileBusy(idx, false)
564
-
565
-
hookEnv := []string{"RESTRAY_OPERATIONS=" + args[0], "RESTRAY_SCHEDULED=false"}
566
-
567
-
if prof.PreHook != "" {
568
-
ms.SetTitle("Running pre-hook...")
569
-
if err := runHook(prof.PreHook, prof, hookEnv...); err != nil {
570
-
setProfileFailed(idx, "Pre-hook failed")
571
-
notifyError(args[0], "Pre-hook failed")
572
-
return
573
-
}
574
-
}
575
-
570
+
go runWithHooks(idx, ms, prof, applyConfig, args[0], args[0], false, func() (bool, string) {
576
571
ms.SetTitle(status)
577
-
msg, err := runRestic(idx, prof, ms, args...)
578
-
if err != nil {
572
+
if msg, err := runRestic(idx, prof, ms, args...); err != nil {
579
573
setProfileFailed(idx, msg)
580
-
notifyError(args[0], msg)
581
-
} else {
582
-
setProfileFailed(idx, "")
583
-
notifySuccess(args[0])
574
+
return false, ""
584
575
}
585
-
if prof.PostHook != "" {
586
-
ms.SetTitle("Running post-hook...")
587
-
var hookErr error
588
-
if err != nil {
589
-
hookErr = runHook(prof.PostHook, prof, append(hookEnv, "RESTRAY_STATUS=failure", "RESTRAY_ERROR="+msg)...)
590
-
} else {
591
-
hookErr = runHook(prof.PostHook, prof, append(hookEnv, "RESTRAY_STATUS=success")...)
592
-
}
593
-
if hookErr != nil {
594
-
setProfileFailed(idx, "Post-hook failed")
595
-
notifyError(args[0], "Post-hook failed")
596
-
}
597
-
}
598
-
}()
576
+
return true, ""
577
+
})
599
578
}
600
579
601
580
go func() {