Compare commits

..

2 Commits

Author SHA1 Message Date
MangoPig 891e8b83ed Fix: harden bootstrap worker flow 2026-06-26 23:30:50 +01:00
MangoPig ae1f347549 Merge branch 'Refactor/Code-Quality' 2026-06-26 18:33:12 +01:00
4 changed files with 244 additions and 30 deletions
+7 -1
View File
@@ -57,7 +57,13 @@ func (runner *Runner) Run(ctx context.Context) error {
return nil return nil
} }
return err runner.logger.Error("worker claim failed", "error", err)
if err := waitForNextPoll(ctx, runner.pollInterval); err != nil {
return nil
}
continue
} }
if job == nil { if job == nil {
+48
View File
@@ -8,6 +8,7 @@ import (
"strings" "strings"
"sync" "sync"
"testing" "testing"
"time"
"moku-backend/internal/jobs" "moku-backend/internal/jobs"
) )
@@ -109,10 +110,50 @@ func TestRunnerMarksFailedWhenHandlerMissing(t *testing.T) {
} }
} }
func TestRunnerRetriesClaimErrors(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
store := &fakeJobStore{
claimErrors: []error{errors.New("relation \"background_jobs\" does not exist")},
job: &jobs.Job{
ID: "job-4",
Kind: jobs.KindBootstrapStructureMaterialize,
},
cancel: cancel,
}
runner := NewRunner(store, slog.New(slog.NewTextHandler(io.Discard, nil)), time.Millisecond)
handlerCalled := false
runner.Register(jobs.KindBootstrapStructureMaterialize, func(ctx context.Context, job jobs.Job) error {
handlerCalled = true
return nil
})
if err := runner.Run(ctx); err != nil {
t.Fatalf("runner returned error: %v", err)
}
if !handlerCalled {
t.Fatal("expected handler to be called after claim retry")
}
if store.claimAttempts < 2 {
t.Fatalf("expected at least two claim attempts, got %d", store.claimAttempts)
}
if len(store.succeeded) != 1 || store.succeeded[0] != "job-4" {
t.Fatalf("expected job to be marked succeeded once, got %#v", store.succeeded)
}
if len(store.failed) != 0 {
t.Fatalf("expected no failed jobs, got %#v", store.failed)
}
}
type fakeJobStore struct { type fakeJobStore struct {
mu sync.Mutex mu sync.Mutex
job *jobs.Job job *jobs.Job
claimed bool claimed bool
claimErrors []error
claimAttempts int
succeeded []string succeeded []string
failed []fakeFailure failed []fakeFailure
cancel context.CancelFunc cancel context.CancelFunc
@@ -126,6 +167,13 @@ type fakeFailure struct {
func (store *fakeJobStore) ClaimNext(ctx context.Context) (*jobs.Job, error) { func (store *fakeJobStore) ClaimNext(ctx context.Context) (*jobs.Job, error) {
store.mu.Lock() store.mu.Lock()
defer store.mu.Unlock() defer store.mu.Unlock()
store.claimAttempts++
if len(store.claimErrors) > 0 {
err := store.claimErrors[0]
store.claimErrors = store.claimErrors[1:]
return nil, err
}
if store.claimed || store.job == nil { if store.claimed || store.job == nil {
return nil, nil return nil, nil
@@ -518,6 +518,111 @@
gap: var(--space-3); gap: var(--space-3);
} }
.wizardFinishPanel {
display: grid;
gap: var(--space-4);
justify-items: stretch;
padding: var(--space-2) 0 0;
min-height: min(14rem, 32dvh);
align-content: center;
}
.wizardFinishShell {
display: grid;
gap: var(--space-4);
width: min(100%, 40rem);
padding: 0;
border: 0;
border-radius: 0;
background: transparent;
box-shadow: none;
}
.wizardFinishStatusRow {
display: grid;
grid-template-columns: auto minmax(0, 1fr);
gap: var(--space-3);
align-items: start;
}
.wizardFinishIndicator {
width: 2.5rem;
height: 2.5rem;
display: inline-flex;
align-items: center;
justify-content: center;
border-radius: 999px;
border: 1px solid color-mix(in srgb, var(--bootstrap-accent) 18%, transparent);
background: color-mix(in srgb, var(--bootstrap-accent) 8%, transparent);
}
.wizardFinishIndicator[data-status="failed"] {
border-color: color-mix(in srgb, var(--color-danger-border, var(--color-border)) 44%, transparent);
background: color-mix(in srgb, var(--color-danger-surface, var(--color-surface-secondary)) 36%, transparent);
}
.wizardFinishSpinner {
width: 1.25rem;
height: 1.25rem;
border-radius: 999px;
border: 2px solid color-mix(in srgb, var(--bootstrap-accent) 18%, transparent);
border-top-color: var(--bootstrap-accent);
animation: wizardFinishSpin 900ms linear infinite;
}
.wizardFinishIndicator[data-status="failed"] .wizardFinishSpinner {
border: 2px solid color-mix(in srgb, var(--color-danger-border, var(--color-border)) 22%, transparent);
border-top-color: var(--color-danger-text, var(--color-text));
animation: none;
transform: rotate(45deg);
border-radius: var(--radius-sm);
width: 1rem;
height: 1rem;
}
.wizardFinishCopy {
display: grid;
gap: var(--space-1);
min-width: 0;
}
.wizardFinishTitle {
@include text-title;
margin: 0;
}
.wizardFinishDescription,
.wizardFinishMessage,
.wizardFinishHint {
margin: 0;
color: var(--color-text-muted);
}
.wizardFinishMessage[data-status="failed"] {
color: var(--color-danger-text, var(--color-text));
}
.wizardFinishHint {
@include text-caption;
}
.wizardFinishActions {
display: flex;
gap: var(--space-3);
flex-wrap: wrap;
padding-top: var(--space-1);
}
@keyframes wizardFinishSpin {
from {
transform: rotate(0deg);
}
to {
transform: rotate(360deg);
}
}
@include respond-down(tablet) { @include respond-down(tablet) {
.summaryGrid, .summaryGrid,
.wizardBody { .wizardBody {
@@ -188,6 +188,7 @@ export const WorkspaceHome = (props: WorkspaceHomeProps): JSX.Element => {
const [isBootstrapStateResolved, setIsBootstrapStateResolved] = createSignal(false); const [isBootstrapStateResolved, setIsBootstrapStateResolved] = createSignal(false);
const [isBootstrapComplete, setIsBootstrapComplete] = createSignal(false); const [isBootstrapComplete, setIsBootstrapComplete] = createSignal(false);
const [isWizardOpen, setIsWizardOpen] = createSignal(false); const [isWizardOpen, setIsWizardOpen] = createSignal(false);
const [isFinishingBootstrapFlow, setIsFinishingBootstrapFlow] = createSignal(false);
const [currentStepIndex, setCurrentStepIndex] = createSignal(0); const [currentStepIndex, setCurrentStepIndex] = createSignal(0);
const installation = createMemo(() => appShellData.installation()); const installation = createMemo(() => appShellData.installation());
const materializationState = createMemo<MaterializationState>(() => { const materializationState = createMemo<MaterializationState>(() => {
@@ -209,6 +210,9 @@ export const WorkspaceHome = (props: WorkspaceHomeProps): JSX.Element => {
() => materializationState() === "pending" || materializationState() === "running", () => materializationState() === "pending" || materializationState() === "running",
); );
const hasMaterializationFailed = createMemo(() => materializationState() === "failed"); const hasMaterializationFailed = createMemo(() => materializationState() === "failed");
const showBootstrapFinishingState = createMemo(
() => isFinishingBootstrapFlow() && (isMaterializationInFlight() || hasMaterializationFailed()),
);
const materializationStatusLabel = createMemo(() => { const materializationStatusLabel = createMemo(() => {
switch (materializationState()) { switch (materializationState()) {
case "pending": case "pending":
@@ -263,14 +267,28 @@ export const WorkspaceHome = (props: WorkspaceHomeProps): JSX.Element => {
} }
if (!isBootstrapPersisted()) { if (!isBootstrapPersisted()) {
setIsFinishingBootstrapFlow(false);
resetWizardState(); resetWizardState();
} }
setIsBootstrapComplete(isBootstrapPersisted() && !isMaterializationInFlight()); setIsBootstrapComplete(isBootstrapPersisted() && !isMaterializationInFlight());
setIsWizardOpen(!isBootstrapPersisted()); setIsWizardOpen(!isBootstrapPersisted() || showBootstrapFinishingState());
setIsBootstrapStateResolved(true); setIsBootstrapStateResolved(true);
}); });
createEffect(() => {
if (!isFinishingBootstrapFlow()) {
return;
}
if (isMaterializationInFlight() || hasMaterializationFailed()) {
return;
}
setIsFinishingBootstrapFlow(false);
setIsWizardOpen(false);
});
createEffect(() => { createEffect(() => {
if (!isBootstrapPersisted() || !isMaterializationInFlight()) { if (!isBootstrapPersisted() || !isMaterializationInFlight()) {
return; return;
@@ -321,7 +339,7 @@ export const WorkspaceHome = (props: WorkspaceHomeProps): JSX.Element => {
const currentStepState = createMemo<BootstrapSubmissionState>(() => stepState[currentStep().id]); const currentStepState = createMemo<BootstrapSubmissionState>(() => stepState[currentStep().id]);
const isFirstStep = (): boolean => currentStepIndex() === 0; const isFirstStep = (): boolean => currentStepIndex() === 0;
const isLastStep = (): boolean => currentStepIndex() === bootstrapStepDefinitions.length - 1; const isLastStep = (): boolean => currentStepIndex() === bootstrapStepDefinitions.length - 1;
const canDismissWizard = (): boolean => isBootstrapPersisted(); const canDismissWizard = (): boolean => isBootstrapPersisted() && !isMaterializationInFlight();
const resetWizardState = (): void => { const resetWizardState = (): void => {
setInstanceForm({ ...defaultInstanceForm }); setInstanceForm({ ...defaultInstanceForm });
@@ -335,6 +353,7 @@ export const WorkspaceHome = (props: WorkspaceHomeProps): JSX.Element => {
structure: initialSubmissionState(), structure: initialSubmissionState(),
}); });
setCurrentStepIndex(0); setCurrentStepIndex(0);
setIsFinishingBootstrapFlow(false);
}; };
const submitStep = async (step: BootstrapStepKey, payload: unknown): Promise<boolean> => { const submitStep = async (step: BootstrapStepKey, payload: unknown): Promise<boolean> => {
@@ -395,8 +414,10 @@ export const WorkspaceHome = (props: WorkspaceHomeProps): JSX.Element => {
if (isLastStep()) { if (isLastStep()) {
await appShellData.reload(); await appShellData.reload();
const shouldShowFinishingState = isBootstrapPersisted() && (isMaterializationInFlight() || hasMaterializationFailed());
setIsFinishingBootstrapFlow(shouldShowFinishingState);
setIsBootstrapComplete(isBootstrapPersisted() && !isMaterializationInFlight()); setIsBootstrapComplete(isBootstrapPersisted() && !isMaterializationInFlight());
setIsWizardOpen(!isBootstrapPersisted()); setIsWizardOpen(!isBootstrapPersisted() || shouldShowFinishingState);
setIsBootstrapStateResolved(true); setIsBootstrapStateResolved(true);
return; return;
} }
@@ -462,29 +483,16 @@ export const WorkspaceHome = (props: WorkspaceHomeProps): JSX.Element => {
<section class={styles.hero} data-slot="workspace-home-hero"> <section class={styles.hero} data-slot="workspace-home-hero">
<h1 class={styles.title}>{isBootstrapPersisted() ? appShellData.activeServer().name : bootstrapTargetLabel()}</h1> <h1 class={styles.title}>{isBootstrapPersisted() ? appShellData.activeServer().name : bootstrapTargetLabel()}</h1>
<Show when={isBootstrapStateResolved() && isBootstrapPersisted() && materializationState() !== "succeeded"}> <Show when={isBootstrapStateResolved() && !isBootstrapPersisted()}>
<div class={styles.heroStatus}>
<div class={styles.statusBadge} data-status={materializationState()}>
{materializationStatusLabel()}
</div>
<Show when={materializationMessage()}>
<p class={styles.heroStatusMessage} data-status={materializationState()}>
{materializationMessage()}
</p>
</Show>
</div>
</Show>
<Show when={isBootstrapStateResolved() && !isBootstrapComplete()}>
<div class={styles.heroActions}> <div class={styles.heroActions}>
<button <button
type="button" type="button"
class={styles.primaryButton} class={styles.primaryButton}
disabled={isBootstrapPersisted()}
onClick={(): void => { onClick={(): void => {
setIsWizardOpen(true); setIsWizardOpen(true);
}} }}
> >
{isBootstrapPersisted() ? "Bootstrap saved" : "Open bootstrap wizard"} Open bootstrap wizard
</button> </button>
</div> </div>
</Show> </Show>
@@ -516,6 +524,52 @@ export const WorkspaceHome = (props: WorkspaceHomeProps): JSX.Element => {
</Show> </Show>
</header> </header>
<Show
when={!showBootstrapFinishingState()}
fallback={
<div class={styles.wizardFinishPanel} data-slot="bootstrap-wizard-finishing-state">
<div class={styles.wizardFinishShell}>
<div class={styles.wizardFinishStatusRow}>
<div class={styles.wizardFinishIndicator} data-status={materializationState()} aria-hidden="true">
<div class={styles.wizardFinishSpinner} />
</div>
<div class={styles.wizardFinishCopy}>
<span class={styles.wizardStepEyebrow}>Bootstrap status</span>
<h3 class={styles.wizardFinishTitle}>Finishing setup</h3>
<p class={styles.wizardFinishDescription}>
We saved your initial bootstrap. The server is finishing the last background setup steps now.
</p>
</div>
</div>
<div class={styles.statusBadge} data-status={materializationState()}>
{materializationStatusLabel()}
</div>
<Show when={materializationMessage()}>
<p class={styles.wizardFinishMessage} data-status={materializationState()}>
{materializationMessage()}
</p>
</Show>
<Show when={isMaterializationInFlight()}>
<p class={styles.wizardFinishHint}>This window will close automatically when setup is complete.</p>
</Show>
</div>
<Show when={hasMaterializationFailed()}>
<div class={styles.wizardFinishActions}>
<button
type="button"
class={styles.secondaryButton}
onClick={(): void => {
setIsFinishingBootstrapFlow(false);
setIsWizardOpen(false);
}}
>
Close
</button>
</div>
</Show>
</div>
}
>
<div class={styles.wizardBody}> <div class={styles.wizardBody}>
<aside class={styles.wizardSidebar} data-slot="bootstrap-wizard-sidebar"> <aside class={styles.wizardSidebar} data-slot="bootstrap-wizard-sidebar">
<nav class={styles.wizardSteps} aria-label="Bootstrap steps"> <nav class={styles.wizardSteps} aria-label="Bootstrap steps">
@@ -716,6 +770,7 @@ export const WorkspaceHome = (props: WorkspaceHomeProps): JSX.Element => {
</div> </div>
</div> </div>
</Show>
</section> </section>
</div> </div>
</Portal> </Portal>