Monorepo for Tangled
spindle/microvm: surface OOM errors better if we can
Signed-off-by: dawn <dawn@tangled.org>
+41
-6
+11
spindle/engines/microvm/cgroup.go
+11
spindle/engines/microvm/cgroup.go
···
143
143
return h.manager.Delete()
144
144
}
145
145
146
+
func (h *CgroupHandle) OOMKilled() bool {
147
+
if h == nil || h.manager == nil {
148
+
return false
149
+
}
150
+
metrics, err := h.manager.Stat()
151
+
if err != nil || metrics == nil || metrics.MemoryEvents == nil {
152
+
return false
153
+
}
154
+
return metrics.MemoryEvents.OomKill > 0
155
+
}
156
+
146
157
func resolveCgroupParent(parent string) (string, string, error) {
147
158
mountpoint, err := cgroup2Mountpoint()
148
159
if err != nil {
+22
-6
spindle/engines/microvm/engine.go
+22
-6
spindle/engines/microvm/engine.go
···
342
342
return nil
343
343
}
344
344
l := e.l.With("workflow", wid, "step", step.Name())
345
+
345
346
if vmExited != nil && vmExited.Load() {
347
+
reason := "microVM exited unexpectedly"
348
+
oom := state.VM != nil && state.VM.OOMKilled()
349
+
if oom {
350
+
reason = "microVM killed by OOM (cgroup memory limit exceeded)"
351
+
}
346
352
if detail := vmCrashLog(state.VM); detail != "" {
347
-
fmt.Fprintf(stderr, "microVM exited unexpectedly:\n%s\n", detail)
348
-
l.Debug("microVM exited unexpectedly during step", "detail", detail)
353
+
fmt.Fprintf(stderr, "%s:\n%s\n", reason, detail)
354
+
l.Debug(reason, "oom", oom, "detail", detail)
349
355
} else {
350
-
fmt.Fprintln(stderr, "microVM exited unexpectedly")
351
-
l.Debug("microVM exited unexpectedly during step")
356
+
fmt.Fprintln(stderr, reason)
357
+
l.Debug(reason, "oom", oom)
352
358
}
353
-
return errors.New("microVM exited unexpectedly; see workflow logs for serial/qemu output")
359
+
return errors.New(reason + "; see workflow logs for serial output")
354
360
}
361
+
355
362
if errors.Is(err, errGuestTimedOut) || ctx.Err() != nil {
356
363
l.Debug("step timed out", "guestReported", errors.Is(err, errGuestTimedOut))
357
364
return engine.ErrTimedOut
358
365
}
359
-
l.Debug("step failed", "error", err)
366
+
367
+
// the agent connection dropped while qemu stayed up (eg. the guest kernel
368
+
// OOM-killed the agent or a guest panic), so surface serial logs, those
369
+
// will be more helpful.
370
+
if detail := vmCrashLog(state.VM); detail != "" {
371
+
fmt.Fprintf(stderr, "step failed (%v):\n%s\n", err, detail)
372
+
l.Debug("step failed", "error", err, "detail", detail)
373
+
} else {
374
+
l.Debug("step failed", "error", err)
375
+
}
360
376
return err
361
377
}
362
378
+7
spindle/engines/microvm/qemu.go
+7
spindle/engines/microvm/qemu.go
···
400
400
return h.workDir
401
401
}
402
402
403
+
func (h *QEMUVMHandle) OOMKilled() bool {
404
+
if h == nil {
405
+
return false
406
+
}
407
+
return h.cgroup.OOMKilled()
408
+
}
409
+
403
410
func (h *QEMUVMHandle) waitForQMP(ctx context.Context, timeout time.Duration) error {
404
411
qmpCtx, cancel := context.WithTimeout(ctx, timeout)
405
412
defer cancel()