Merge branch 'Features/Backend/Posix-DB-Projection'

This commit is contained in:
MangoPig
2026-06-21 22:03:43 +01:00
6 changed files with 785 additions and 0 deletions

52
Backend/cmd/posix/main.go Normal file
View File

@@ -0,0 +1,52 @@
package main
import (
"context"
"fmt"
"log"
"os"
"moku-backend/internal/config"
"moku-backend/internal/database"
"moku-backend/internal/posixproj"
)
func main() {
command := "rebuild"
if len(os.Args) > 1 {
command = os.Args[1]
}
switch command {
case "rebuild":
if err := rebuildProjection(context.Background()); err != nil {
log.Fatalf("rebuild POSIX projection: %v", err)
}
default:
log.Fatalf("unsupported posix command %q (supported: rebuild)", command)
}
}
func rebuildProjection(ctx context.Context) error {
cfg := config.Load()
db, err := database.NewPostgres(cfg.PostgresURL)
if err != nil {
return fmt.Errorf("connect database: %w", err)
}
defer db.Close()
summary, err := posixproj.NewProjector(db, cfg.POSIXRoot).RebuildWithSummary(ctx)
if err != nil {
return err
}
fmt.Printf(
"POSIX projection rebuilt from %s\n total nodes: %d\n directories: %d\n files: %d\n",
cfg.POSIXRoot,
summary.TotalNodes,
summary.DirectoryCount,
summary.FileCount,
)
return nil
}

View File

@@ -0,0 +1,45 @@
-- +goose Up
CREATE TYPE posix_node_kind AS ENUM ('directory', 'file');
CREATE TABLE IF NOT EXISTS posix_nodes (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
path TEXT NOT NULL UNIQUE,
parent_path TEXT,
name TEXT NOT NULL,
depth INTEGER NOT NULL,
node_kind posix_node_kind NOT NULL,
logical_type TEXT NOT NULL DEFAULT 'generic',
file_role TEXT,
resource_id TEXT,
resource_name TEXT,
resource_slug TEXT,
installation_id TEXT,
organization_id TEXT,
organization_slug TEXT,
department_slug TEXT,
team_slug TEXT,
project_slug TEXT,
personal_slug TEXT,
content_json JSONB,
size_bytes BIGINT NOT NULL DEFAULT 0,
checksum TEXT,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX IF NOT EXISTS idx_posix_nodes_parent_path ON posix_nodes (parent_path);
CREATE INDEX IF NOT EXISTS idx_posix_nodes_logical_type ON posix_nodes (logical_type);
CREATE INDEX IF NOT EXISTS idx_posix_nodes_project_slug ON posix_nodes (project_slug);
CREATE INDEX IF NOT EXISTS idx_posix_nodes_department_slug ON posix_nodes (department_slug);
CREATE INDEX IF NOT EXISTS idx_posix_nodes_team_slug ON posix_nodes (team_slug);
-- +goose Down
DROP INDEX IF EXISTS idx_posix_nodes_team_slug;
DROP INDEX IF EXISTS idx_posix_nodes_department_slug;
DROP INDEX IF EXISTS idx_posix_nodes_project_slug;
DROP INDEX IF EXISTS idx_posix_nodes_logical_type;
DROP INDEX IF EXISTS idx_posix_nodes_parent_path;
DROP TABLE IF EXISTS posix_nodes;
DROP TYPE IF EXISTS posix_node_kind;

View File

@@ -14,6 +14,7 @@ import (
"github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5"
"moku-backend/internal/database" "moku-backend/internal/database"
"moku-backend/internal/posixproj"
) )
const ( const (
@@ -413,6 +414,10 @@ func (service *Service) SaveStructure(ctx context.Context, input SaveStructureIn
return StructureRecord{}, err return StructureRecord{}, err
} }
if err := posixproj.NewProjector(service.db, service.posixRoot).Rebuild(ctx); err != nil {
return StructureRecord{}, fmt.Errorf("rebuild POSIX projection: %w", err)
}
return StructureRecord{ return StructureRecord{
Installation: installation, Installation: installation,
Organization: organization, Organization: organization,

View File

@@ -0,0 +1,531 @@
package posixproj
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"moku-backend/internal/database"
)
const rootProjectionPath = "/"
type Projector struct {
db *database.DB
root string
}
type RebuildSummary struct {
TotalNodes int
DirectoryCount int
FileCount int
}
type NodeKind string
const (
NodeKindDirectory NodeKind = "directory"
NodeKindFile NodeKind = "file"
)
type Scope struct {
InstallationID string
OrganizationID string
OrganizationSlug string
DepartmentSlug string
TeamSlug string
ProjectSlug string
PersonalSlug string
}
type Node struct {
Path string
ParentPath *string
Name string
Depth int
NodeKind NodeKind
LogicalType string
FileRole string
ResourceID string
ResourceName string
ResourceSlug string
InstallationID string
OrganizationID string
OrganizationSlug string
DepartmentSlug string
TeamSlug string
ProjectSlug string
PersonalSlug string
ContentJSON []byte
SizeBytes int64
Checksum string
}
func NewProjector(db *database.DB, root string) *Projector {
return &Projector{db: db, root: strings.TrimSpace(root)}
}
func (projector *Projector) Rebuild(ctx context.Context) error {
_, err := projector.RebuildWithSummary(ctx)
return err
}
func (projector *Projector) RebuildWithSummary(ctx context.Context) (RebuildSummary, error) {
if projector == nil || projector.db == nil || projector.db.Pool == nil {
return RebuildSummary{}, nil
}
nodes, err := ScanRoot(projector.root)
if err != nil {
return RebuildSummary{}, err
}
summary := summarizeNodes(nodes)
tx, err := projector.db.Pool.Begin(ctx)
if err != nil {
return RebuildSummary{}, err
}
defer func() {
_ = tx.Rollback(ctx)
}()
if _, err := tx.Exec(ctx, `DELETE FROM posix_nodes;`); err != nil {
return RebuildSummary{}, fmt.Errorf("clear posix_nodes: %w", err)
}
for _, node := range nodes {
if _, err := tx.Exec(ctx, `
INSERT INTO posix_nodes (
path,
parent_path,
name,
depth,
node_kind,
logical_type,
file_role,
resource_id,
resource_name,
resource_slug,
installation_id,
organization_id,
organization_slug,
department_slug,
team_slug,
project_slug,
personal_slug,
content_json,
size_bytes,
checksum
) VALUES (
$1, $2, $3, $4, $5::posix_node_kind, $6, $7, $8, $9, $10,
$11, $12, $13, $14, $15, $16, $17, $18::jsonb, $19, $20
);
`,
node.Path,
node.ParentPath,
node.Name,
node.Depth,
string(node.NodeKind),
node.LogicalType,
node.FileRole,
node.ResourceID,
node.ResourceName,
node.ResourceSlug,
node.InstallationID,
node.OrganizationID,
node.OrganizationSlug,
node.DepartmentSlug,
node.TeamSlug,
node.ProjectSlug,
node.PersonalSlug,
node.ContentJSON,
node.SizeBytes,
node.Checksum,
); err != nil {
return RebuildSummary{}, fmt.Errorf("insert posix node %s: %w", node.Path, err)
}
}
if err := tx.Commit(ctx); err != nil {
return RebuildSummary{}, err
}
return summary, nil
}
func ScanRoot(root string) ([]Node, error) {
rootPath := strings.TrimSpace(root)
if rootPath == "" {
return nil, nil
}
info, err := os.Stat(rootPath)
if err != nil {
return nil, fmt.Errorf("stat POSIX root: %w", err)
}
if !info.IsDir() {
return nil, fmt.Errorf("POSIX root is not a directory: %s", rootPath)
}
rootScope, err := loadRootScope(rootPath)
if err != nil {
return nil, err
}
nodes := []Node{{
Path: rootProjectionPath,
ParentPath: nil,
Name: filepath.Base(rootPath),
Depth: 0,
NodeKind: NodeKindDirectory,
LogicalType: "tenant_root",
InstallationID: rootScope.InstallationID,
OrganizationID: rootScope.OrganizationID,
OrganizationSlug: rootScope.OrganizationSlug,
}}
err = filepath.WalkDir(rootPath, func(path string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if path == rootPath {
return nil
}
relPath, err := filepath.Rel(rootPath, path)
if err != nil {
return err
}
relPath = filepath.ToSlash(relPath)
if relPath == "." {
return nil
}
node, err := buildNode(rootPath, relPath, entry, rootScope)
if err != nil {
return err
}
nodes = append(nodes, node)
return nil
})
if err != nil {
return nil, fmt.Errorf("scan POSIX root: %w", err)
}
return nodes, nil
}
func loadRootScope(rootPath string) (Scope, error) {
settingsPath := filepath.Join(rootPath, "settings.json")
content, err := os.ReadFile(settingsPath)
if err != nil {
if errorsIsNotExist(err) {
return Scope{}, nil
}
return Scope{}, fmt.Errorf("read root settings.json: %w", err)
}
var payload map[string]any
if err := json.Unmarshal(content, &payload); err != nil {
return Scope{}, fmt.Errorf("decode root settings.json: %w", err)
}
installation, _ := payload["installation"].(map[string]any)
organization, _ := payload["organization"].(map[string]any)
return Scope{
InstallationID: stringValue(installation["id"]),
OrganizationID: stringValue(organization["id"]),
OrganizationSlug: stringValue(organization["slug"]),
}, nil
}
func buildNode(rootPath, relPath string, entry fs.DirEntry, rootScope Scope) (Node, error) {
scope := deriveScope(relPath, rootScope)
parentPath := projectionParentPath(relPath)
logicalType, fileRole := classifyPath(relPath, entry.IsDir())
node := Node{
Path: relPath,
ParentPath: parentPath,
Name: entry.Name(),
Depth: strings.Count(relPath, "/") + 1,
NodeKind: NodeKindDirectory,
LogicalType: logicalType,
FileRole: fileRole,
InstallationID: scope.InstallationID,
OrganizationID: scope.OrganizationID,
OrganizationSlug: scope.OrganizationSlug,
DepartmentSlug: scope.DepartmentSlug,
TeamSlug: scope.TeamSlug,
ProjectSlug: scope.ProjectSlug,
PersonalSlug: scope.PersonalSlug,
}
if entry.IsDir() {
return node, nil
}
absPath := filepath.Join(rootPath, filepath.FromSlash(relPath))
content, err := os.ReadFile(absPath)
if err != nil {
return Node{}, fmt.Errorf("read POSIX file %s: %w", relPath, err)
}
hash := sha256.Sum256(content)
node.NodeKind = NodeKindFile
node.SizeBytes = int64(len(content))
node.Checksum = hex.EncodeToString(hash[:])
if strings.EqualFold(filepath.Ext(entry.Name()), ".json") {
var payload map[string]any
if err := json.Unmarshal(content, &payload); err == nil {
jsonContent, err := json.Marshal(payload)
if err != nil {
return Node{}, fmt.Errorf("remarshal POSIX file %s: %w", relPath, err)
}
node.ContentJSON = jsonContent
node.ResourceID = stringValue(payload["id"])
node.ResourceName = stringValue(payload["name"])
node.ResourceSlug = stringValue(payload["slug"])
if node.ResourceID == "" && fileRole == "settings" && logicalType == "tenant" {
installation, _ := payload["installation"].(map[string]any)
organization, _ := payload["organization"].(map[string]any)
node.ResourceID = stringValue(installation["id"])
node.ResourceName = stringValue(installation["name"])
node.InstallationID = stringValue(installation["id"])
node.OrganizationID = stringValue(organization["id"])
node.OrganizationSlug = firstNonEmpty(node.OrganizationSlug, stringValue(organization["slug"]))
}
if node.ResourceID == "" && fileRole == "users" {
node.ResourceName = firstNonEmpty(node.ResourceName, parentEntityName(logicalType, scope))
}
}
}
if node.ResourceSlug == "" {
node.ResourceSlug = inferredResourceSlug(logicalType, scope)
}
return node, nil
}
func deriveScope(relPath string, rootScope Scope) Scope {
scope := rootScope
parts := strings.Split(relPath, "/")
if len(parts) >= 2 && parts[0] == "departments" && strings.HasPrefix(parts[1], "department-") {
scope.DepartmentSlug = strings.TrimPrefix(parts[1], "department-")
}
if len(parts) >= 4 && parts[0] == "departments" && parts[2] == "teams" && strings.HasPrefix(parts[3], "team-") {
scope.DepartmentSlug = strings.TrimPrefix(parts[1], "department-")
scope.TeamSlug = strings.TrimPrefix(parts[3], "team-")
}
if len(parts) >= 2 && parts[0] == "projects" && strings.HasPrefix(parts[1], "project-") {
scope.ProjectSlug = strings.TrimPrefix(parts[1], "project-")
}
if len(parts) >= 3 && parts[0] == "users" && parts[1] == "personals" && strings.HasPrefix(parts[2], "personal-") {
scope.PersonalSlug = strings.TrimPrefix(parts[2], "personal-")
}
return scope
}
func classifyPath(relPath string, isDir bool) (logicalType, fileRole string) {
parts := strings.Split(relPath, "/")
name := parts[len(parts)-1]
if !isDir {
fileRole = strings.TrimSuffix(name, filepath.Ext(name))
}
switch {
case relPath == "settings.json":
return "tenant", "settings"
case relPath == "layout.json":
return "tenant", "layout"
case len(parts) >= 1 && parts[0] == "catalog":
if isDir {
if len(parts) == 1 {
return "catalog", ""
}
if len(parts) >= 2 && parts[1] == "packs" {
if len(parts) == 2 {
return "catalog_packs", ""
}
if len(parts) == 3 {
return "catalog_pack", ""
}
if len(parts) >= 4 && parts[3] == "entries" {
return "catalog_pack_entries", ""
}
return "catalog_entry", ""
}
if len(parts) >= 2 && parts[1] == "standalone" {
if len(parts) == 2 {
return "catalog_standalone", ""
}
return "catalog_entry", ""
}
}
return "catalog", fileRole
case len(parts) >= 2 && parts[0] == "departments" && strings.HasPrefix(parts[1], "department-"):
if isDir {
if len(parts) == 2 {
return "department", ""
}
if len(parts) == 3 && parts[2] == "teams" {
return "department_teams", ""
}
if len(parts) >= 4 && strings.HasPrefix(parts[3], "team-") {
return "team", ""
}
}
if len(parts) >= 4 && strings.HasPrefix(parts[3], "team-") {
return "team", fileRole
}
return "department", fileRole
case len(parts) >= 2 && parts[0] == "projects" && strings.HasPrefix(parts[1], "project-"):
if isDir {
if len(parts) == 2 {
return "project", ""
}
if len(parts) == 3 && parts[2] == "tree" {
return "project_tree", ""
}
if strings.Contains(relPath, "/tree/") || strings.HasSuffix(relPath, "/tree") {
if strings.HasPrefix(name, "folder-") {
return "folder", ""
}
if strings.HasPrefix(name, "item-") {
return "item", ""
}
}
}
if strings.Contains(relPath, "/tree/") {
if strings.HasPrefix(parts[len(parts)-2], "item-") {
return "item", fileRole
}
if strings.HasPrefix(parts[len(parts)-2], "folder-") {
return "folder", fileRole
}
}
return "project", fileRole
case len(parts) >= 1 && parts[0] == "users":
if isDir {
if len(parts) == 1 {
return "users", ""
}
if len(parts) == 2 && parts[1] == "personals" {
return "personals", ""
}
if len(parts) >= 3 && parts[1] == "personals" && strings.HasPrefix(parts[2], "personal-") {
return "personal", ""
}
if strings.Contains(relPath, "/tree/") || strings.HasSuffix(relPath, "/tree") {
if strings.HasPrefix(name, "folder-") {
return "folder", ""
}
if strings.HasPrefix(name, "item-") {
return "item", ""
}
}
}
if len(parts) >= 3 && parts[1] == "personals" && strings.HasPrefix(parts[2], "personal-") {
if strings.Contains(relPath, "/tree/") {
if strings.HasPrefix(parts[len(parts)-2], "item-") {
return "item", fileRole
}
if strings.HasPrefix(parts[len(parts)-2], "folder-") {
return "folder", fileRole
}
}
return "personal", fileRole
}
return "users", fileRole
default:
if isDir {
return "directory", ""
}
return "file", fileRole
}
}
func projectionParentPath(relPath string) *string {
if relPath == "" || relPath == rootProjectionPath {
return nil
}
parent := filepath.ToSlash(filepath.Dir(relPath))
if parent == "." || parent == "" {
root := rootProjectionPath
return &root
}
return &parent
}
func inferredResourceSlug(logicalType string, scope Scope) string {
switch logicalType {
case "department":
return scope.DepartmentSlug
case "team":
return scope.TeamSlug
case "project":
return scope.ProjectSlug
case "personal":
return scope.PersonalSlug
default:
return ""
}
}
func parentEntityName(logicalType string, scope Scope) string {
switch logicalType {
case "department":
return scope.DepartmentSlug
case "team":
return scope.TeamSlug
case "project":
return scope.ProjectSlug
case "personal":
return scope.PersonalSlug
default:
return ""
}
}
func stringValue(value any) string {
stringValue, _ := value.(string)
return strings.TrimSpace(stringValue)
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
trimmed := strings.TrimSpace(value)
if trimmed != "" {
return trimmed
}
}
return ""
}
func summarizeNodes(nodes []Node) RebuildSummary {
summary := RebuildSummary{TotalNodes: len(nodes)}
for _, node := range nodes {
switch node.NodeKind {
case NodeKindDirectory:
summary.DirectoryCount++
case NodeKindFile:
summary.FileCount++
}
}
return summary
}
func errorsIsNotExist(err error) bool {
return err != nil && os.IsNotExist(err)
}

View File

@@ -0,0 +1,148 @@
package posixproj
import (
"encoding/json"
"os"
"path/filepath"
"testing"
)
func TestScanRootBuildsProjectedNodesFromBootstrapShape(t *testing.T) {
root := filepath.Join(t.TempDir(), "POSIX")
mustMkdirAll(t, filepath.Join(root, "catalog", "packs"))
mustMkdirAll(t, filepath.Join(root, "catalog", "standalone"))
mustMkdirAll(t, filepath.Join(root, "departments", "department-primary-department", "teams", "team-primary-team"))
mustMkdirAll(t, filepath.Join(root, "projects", "project-primary-project", "tree"))
mustMkdirAll(t, filepath.Join(root, "users", "personals"))
mustWriteJSON(t, filepath.Join(root, "settings.json"), map[string]any{
"installation": map[string]any{
"id": "installation-1",
"name": "MangoPig",
"isBootstrapped": true,
},
"organization": map[string]any{
"id": "org-1",
"name": "Primary Organization",
"slug": "primary-organization",
},
})
mustWriteJSON(t, filepath.Join(root, "layout.json"), map[string]any{
"type": "tenant-layout",
"home": map[string]any{"defaultProjectSlug": "primary-project"},
})
mustWriteJSON(t, filepath.Join(root, "departments", "department-primary-department", "settings.json"), map[string]any{
"id": "dept-1",
"name": "Primary Department",
"slug": "primary-department",
"type": "department",
})
mustWriteJSON(t, filepath.Join(root, "departments", "department-primary-department", "users.json"), map[string]any{
"users": []map[string]any{{"id": "admin-1"}},
})
mustWriteJSON(t, filepath.Join(root, "departments", "department-primary-department", "teams", "team-primary-team", "settings.json"), map[string]any{
"id": "team-1",
"name": "Primary Team",
"slug": "primary-team",
"type": "team",
})
mustWriteJSON(t, filepath.Join(root, "projects", "project-primary-project", "settings.json"), map[string]any{
"id": "project-1",
"name": "Primary Project",
"slug": "primary-project",
"type": "project",
})
mustWriteJSON(t, filepath.Join(root, "projects", "project-primary-project", "home.json"), map[string]any{
"type": "project-home",
"project": "primary-project",
})
mustWriteJSON(t, filepath.Join(root, "users", "settings.json"), map[string]any{
"primaryAdminId": "admin-1",
})
mustWriteJSON(t, filepath.Join(root, "users", "data.json"), map[string]any{
"users": []map[string]any{{"id": "admin-1", "email": "ronald@example.com"}},
})
nodes, err := ScanRoot(root)
if err != nil {
t.Fatalf("ScanRoot() error = %v", err)
}
index := make(map[string]Node, len(nodes))
for _, node := range nodes {
index[node.Path] = node
}
rootNode, ok := index[rootProjectionPath]
if !ok {
t.Fatalf("expected synthetic root node")
}
if rootNode.LogicalType != "tenant_root" {
t.Fatalf("expected root logical type tenant_root, got %q", rootNode.LogicalType)
}
if rootNode.OrganizationSlug != "primary-organization" {
t.Fatalf("expected root organization slug primary-organization, got %q", rootNode.OrganizationSlug)
}
tenantSettings := index["settings.json"]
if tenantSettings.LogicalType != "tenant" || tenantSettings.FileRole != "settings" {
t.Fatalf("unexpected tenant settings classification: %#v", tenantSettings)
}
if tenantSettings.InstallationID != "installation-1" {
t.Fatalf("expected installation id installation-1, got %q", tenantSettings.InstallationID)
}
deptSettings := index["departments/department-primary-department/settings.json"]
if deptSettings.DepartmentSlug != "primary-department" {
t.Fatalf("expected department slug primary-department, got %q", deptSettings.DepartmentSlug)
}
if deptSettings.ResourceID != "dept-1" {
t.Fatalf("expected department resource id dept-1, got %q", deptSettings.ResourceID)
}
teamSettings := index["departments/department-primary-department/teams/team-primary-team/settings.json"]
if teamSettings.TeamSlug != "primary-team" {
t.Fatalf("expected team slug primary-team, got %q", teamSettings.TeamSlug)
}
projectSettings := index["projects/project-primary-project/settings.json"]
if projectSettings.ProjectSlug != "primary-project" {
t.Fatalf("expected project slug primary-project, got %q", projectSettings.ProjectSlug)
}
if projectSettings.ResourceName != "Primary Project" {
t.Fatalf("expected project resource name Primary Project, got %q", projectSettings.ResourceName)
}
projectTree := index["projects/project-primary-project/tree"]
if projectTree.LogicalType != "project_tree" || projectTree.NodeKind != NodeKindDirectory {
t.Fatalf("unexpected project tree node: %#v", projectTree)
}
usersData := index["users/data.json"]
if usersData.LogicalType != "users" || usersData.FileRole != "data" {
t.Fatalf("unexpected users data classification: %#v", usersData)
}
if usersData.Checksum == "" || usersData.SizeBytes == 0 {
t.Fatalf("expected users/data.json checksum and size to be populated: %#v", usersData)
}
}
func mustMkdirAll(t *testing.T, path string) {
t.Helper()
if err := os.MkdirAll(path, 0o755); err != nil {
t.Fatalf("MkdirAll(%q) error = %v", path, err)
}
}
func mustWriteJSON(t *testing.T, path string, payload any) {
t.Helper()
bytes, err := json.MarshalIndent(payload, "", " ")
if err != nil {
t.Fatalf("MarshalIndent(%q) error = %v", path, err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(path, bytes, 0o644); err != nil {
t.Fatalf("WriteFile(%q) error = %v", path, err)
}
}

View File

@@ -21,6 +21,10 @@ migrate-rebuild:
migrate-status: migrate-status:
cd '{{backend_dir}}' && go run ./cmd/migrate status cd '{{backend_dir}}' && go run ./cmd/migrate status
# Rebuild the POSIX-to-DB projection from the current POSIX root.
posix-rebuild:
cd '{{backend_dir}}' && go run ./cmd/posix rebuild
# Format backend Go source files. # Format backend Go source files.
fmt: fmt:
cd '{{backend_dir}}' && gofmt -w ./cmd ./db ./internal cd '{{backend_dir}}' && gofmt -w ./cmd ./db ./internal