Feat: add POSIX DB projection

This commit is contained in:
MangoPig
2026-06-21 22:02:59 +01:00
parent 9b4f1ce197
commit 3c7a73853d
6 changed files with 785 additions and 0 deletions

View File

@@ -0,0 +1,531 @@
package posixproj
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"moku-backend/internal/database"
)
const rootProjectionPath = "/"
type Projector struct {
db *database.DB
root string
}
type RebuildSummary struct {
TotalNodes int
DirectoryCount int
FileCount int
}
type NodeKind string
const (
NodeKindDirectory NodeKind = "directory"
NodeKindFile NodeKind = "file"
)
type Scope struct {
InstallationID string
OrganizationID string
OrganizationSlug string
DepartmentSlug string
TeamSlug string
ProjectSlug string
PersonalSlug string
}
type Node struct {
Path string
ParentPath *string
Name string
Depth int
NodeKind NodeKind
LogicalType string
FileRole string
ResourceID string
ResourceName string
ResourceSlug string
InstallationID string
OrganizationID string
OrganizationSlug string
DepartmentSlug string
TeamSlug string
ProjectSlug string
PersonalSlug string
ContentJSON []byte
SizeBytes int64
Checksum string
}
func NewProjector(db *database.DB, root string) *Projector {
return &Projector{db: db, root: strings.TrimSpace(root)}
}
func (projector *Projector) Rebuild(ctx context.Context) error {
_, err := projector.RebuildWithSummary(ctx)
return err
}
func (projector *Projector) RebuildWithSummary(ctx context.Context) (RebuildSummary, error) {
if projector == nil || projector.db == nil || projector.db.Pool == nil {
return RebuildSummary{}, nil
}
nodes, err := ScanRoot(projector.root)
if err != nil {
return RebuildSummary{}, err
}
summary := summarizeNodes(nodes)
tx, err := projector.db.Pool.Begin(ctx)
if err != nil {
return RebuildSummary{}, err
}
defer func() {
_ = tx.Rollback(ctx)
}()
if _, err := tx.Exec(ctx, `DELETE FROM posix_nodes;`); err != nil {
return RebuildSummary{}, fmt.Errorf("clear posix_nodes: %w", err)
}
for _, node := range nodes {
if _, err := tx.Exec(ctx, `
INSERT INTO posix_nodes (
path,
parent_path,
name,
depth,
node_kind,
logical_type,
file_role,
resource_id,
resource_name,
resource_slug,
installation_id,
organization_id,
organization_slug,
department_slug,
team_slug,
project_slug,
personal_slug,
content_json,
size_bytes,
checksum
) VALUES (
$1, $2, $3, $4, $5::posix_node_kind, $6, $7, $8, $9, $10,
$11, $12, $13, $14, $15, $16, $17, $18::jsonb, $19, $20
);
`,
node.Path,
node.ParentPath,
node.Name,
node.Depth,
string(node.NodeKind),
node.LogicalType,
node.FileRole,
node.ResourceID,
node.ResourceName,
node.ResourceSlug,
node.InstallationID,
node.OrganizationID,
node.OrganizationSlug,
node.DepartmentSlug,
node.TeamSlug,
node.ProjectSlug,
node.PersonalSlug,
node.ContentJSON,
node.SizeBytes,
node.Checksum,
); err != nil {
return RebuildSummary{}, fmt.Errorf("insert posix node %s: %w", node.Path, err)
}
}
if err := tx.Commit(ctx); err != nil {
return RebuildSummary{}, err
}
return summary, nil
}
func ScanRoot(root string) ([]Node, error) {
rootPath := strings.TrimSpace(root)
if rootPath == "" {
return nil, nil
}
info, err := os.Stat(rootPath)
if err != nil {
return nil, fmt.Errorf("stat POSIX root: %w", err)
}
if !info.IsDir() {
return nil, fmt.Errorf("POSIX root is not a directory: %s", rootPath)
}
rootScope, err := loadRootScope(rootPath)
if err != nil {
return nil, err
}
nodes := []Node{{
Path: rootProjectionPath,
ParentPath: nil,
Name: filepath.Base(rootPath),
Depth: 0,
NodeKind: NodeKindDirectory,
LogicalType: "tenant_root",
InstallationID: rootScope.InstallationID,
OrganizationID: rootScope.OrganizationID,
OrganizationSlug: rootScope.OrganizationSlug,
}}
err = filepath.WalkDir(rootPath, func(path string, entry fs.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if path == rootPath {
return nil
}
relPath, err := filepath.Rel(rootPath, path)
if err != nil {
return err
}
relPath = filepath.ToSlash(relPath)
if relPath == "." {
return nil
}
node, err := buildNode(rootPath, relPath, entry, rootScope)
if err != nil {
return err
}
nodes = append(nodes, node)
return nil
})
if err != nil {
return nil, fmt.Errorf("scan POSIX root: %w", err)
}
return nodes, nil
}
func loadRootScope(rootPath string) (Scope, error) {
settingsPath := filepath.Join(rootPath, "settings.json")
content, err := os.ReadFile(settingsPath)
if err != nil {
if errorsIsNotExist(err) {
return Scope{}, nil
}
return Scope{}, fmt.Errorf("read root settings.json: %w", err)
}
var payload map[string]any
if err := json.Unmarshal(content, &payload); err != nil {
return Scope{}, fmt.Errorf("decode root settings.json: %w", err)
}
installation, _ := payload["installation"].(map[string]any)
organization, _ := payload["organization"].(map[string]any)
return Scope{
InstallationID: stringValue(installation["id"]),
OrganizationID: stringValue(organization["id"]),
OrganizationSlug: stringValue(organization["slug"]),
}, nil
}
func buildNode(rootPath, relPath string, entry fs.DirEntry, rootScope Scope) (Node, error) {
scope := deriveScope(relPath, rootScope)
parentPath := projectionParentPath(relPath)
logicalType, fileRole := classifyPath(relPath, entry.IsDir())
node := Node{
Path: relPath,
ParentPath: parentPath,
Name: entry.Name(),
Depth: strings.Count(relPath, "/") + 1,
NodeKind: NodeKindDirectory,
LogicalType: logicalType,
FileRole: fileRole,
InstallationID: scope.InstallationID,
OrganizationID: scope.OrganizationID,
OrganizationSlug: scope.OrganizationSlug,
DepartmentSlug: scope.DepartmentSlug,
TeamSlug: scope.TeamSlug,
ProjectSlug: scope.ProjectSlug,
PersonalSlug: scope.PersonalSlug,
}
if entry.IsDir() {
return node, nil
}
absPath := filepath.Join(rootPath, filepath.FromSlash(relPath))
content, err := os.ReadFile(absPath)
if err != nil {
return Node{}, fmt.Errorf("read POSIX file %s: %w", relPath, err)
}
hash := sha256.Sum256(content)
node.NodeKind = NodeKindFile
node.SizeBytes = int64(len(content))
node.Checksum = hex.EncodeToString(hash[:])
if strings.EqualFold(filepath.Ext(entry.Name()), ".json") {
var payload map[string]any
if err := json.Unmarshal(content, &payload); err == nil {
jsonContent, err := json.Marshal(payload)
if err != nil {
return Node{}, fmt.Errorf("remarshal POSIX file %s: %w", relPath, err)
}
node.ContentJSON = jsonContent
node.ResourceID = stringValue(payload["id"])
node.ResourceName = stringValue(payload["name"])
node.ResourceSlug = stringValue(payload["slug"])
if node.ResourceID == "" && fileRole == "settings" && logicalType == "tenant" {
installation, _ := payload["installation"].(map[string]any)
organization, _ := payload["organization"].(map[string]any)
node.ResourceID = stringValue(installation["id"])
node.ResourceName = stringValue(installation["name"])
node.InstallationID = stringValue(installation["id"])
node.OrganizationID = stringValue(organization["id"])
node.OrganizationSlug = firstNonEmpty(node.OrganizationSlug, stringValue(organization["slug"]))
}
if node.ResourceID == "" && fileRole == "users" {
node.ResourceName = firstNonEmpty(node.ResourceName, parentEntityName(logicalType, scope))
}
}
}
if node.ResourceSlug == "" {
node.ResourceSlug = inferredResourceSlug(logicalType, scope)
}
return node, nil
}
func deriveScope(relPath string, rootScope Scope) Scope {
scope := rootScope
parts := strings.Split(relPath, "/")
if len(parts) >= 2 && parts[0] == "departments" && strings.HasPrefix(parts[1], "department-") {
scope.DepartmentSlug = strings.TrimPrefix(parts[1], "department-")
}
if len(parts) >= 4 && parts[0] == "departments" && parts[2] == "teams" && strings.HasPrefix(parts[3], "team-") {
scope.DepartmentSlug = strings.TrimPrefix(parts[1], "department-")
scope.TeamSlug = strings.TrimPrefix(parts[3], "team-")
}
if len(parts) >= 2 && parts[0] == "projects" && strings.HasPrefix(parts[1], "project-") {
scope.ProjectSlug = strings.TrimPrefix(parts[1], "project-")
}
if len(parts) >= 3 && parts[0] == "users" && parts[1] == "personals" && strings.HasPrefix(parts[2], "personal-") {
scope.PersonalSlug = strings.TrimPrefix(parts[2], "personal-")
}
return scope
}
func classifyPath(relPath string, isDir bool) (logicalType, fileRole string) {
parts := strings.Split(relPath, "/")
name := parts[len(parts)-1]
if !isDir {
fileRole = strings.TrimSuffix(name, filepath.Ext(name))
}
switch {
case relPath == "settings.json":
return "tenant", "settings"
case relPath == "layout.json":
return "tenant", "layout"
case len(parts) >= 1 && parts[0] == "catalog":
if isDir {
if len(parts) == 1 {
return "catalog", ""
}
if len(parts) >= 2 && parts[1] == "packs" {
if len(parts) == 2 {
return "catalog_packs", ""
}
if len(parts) == 3 {
return "catalog_pack", ""
}
if len(parts) >= 4 && parts[3] == "entries" {
return "catalog_pack_entries", ""
}
return "catalog_entry", ""
}
if len(parts) >= 2 && parts[1] == "standalone" {
if len(parts) == 2 {
return "catalog_standalone", ""
}
return "catalog_entry", ""
}
}
return "catalog", fileRole
case len(parts) >= 2 && parts[0] == "departments" && strings.HasPrefix(parts[1], "department-"):
if isDir {
if len(parts) == 2 {
return "department", ""
}
if len(parts) == 3 && parts[2] == "teams" {
return "department_teams", ""
}
if len(parts) >= 4 && strings.HasPrefix(parts[3], "team-") {
return "team", ""
}
}
if len(parts) >= 4 && strings.HasPrefix(parts[3], "team-") {
return "team", fileRole
}
return "department", fileRole
case len(parts) >= 2 && parts[0] == "projects" && strings.HasPrefix(parts[1], "project-"):
if isDir {
if len(parts) == 2 {
return "project", ""
}
if len(parts) == 3 && parts[2] == "tree" {
return "project_tree", ""
}
if strings.Contains(relPath, "/tree/") || strings.HasSuffix(relPath, "/tree") {
if strings.HasPrefix(name, "folder-") {
return "folder", ""
}
if strings.HasPrefix(name, "item-") {
return "item", ""
}
}
}
if strings.Contains(relPath, "/tree/") {
if strings.HasPrefix(parts[len(parts)-2], "item-") {
return "item", fileRole
}
if strings.HasPrefix(parts[len(parts)-2], "folder-") {
return "folder", fileRole
}
}
return "project", fileRole
case len(parts) >= 1 && parts[0] == "users":
if isDir {
if len(parts) == 1 {
return "users", ""
}
if len(parts) == 2 && parts[1] == "personals" {
return "personals", ""
}
if len(parts) >= 3 && parts[1] == "personals" && strings.HasPrefix(parts[2], "personal-") {
return "personal", ""
}
if strings.Contains(relPath, "/tree/") || strings.HasSuffix(relPath, "/tree") {
if strings.HasPrefix(name, "folder-") {
return "folder", ""
}
if strings.HasPrefix(name, "item-") {
return "item", ""
}
}
}
if len(parts) >= 3 && parts[1] == "personals" && strings.HasPrefix(parts[2], "personal-") {
if strings.Contains(relPath, "/tree/") {
if strings.HasPrefix(parts[len(parts)-2], "item-") {
return "item", fileRole
}
if strings.HasPrefix(parts[len(parts)-2], "folder-") {
return "folder", fileRole
}
}
return "personal", fileRole
}
return "users", fileRole
default:
if isDir {
return "directory", ""
}
return "file", fileRole
}
}
func projectionParentPath(relPath string) *string {
if relPath == "" || relPath == rootProjectionPath {
return nil
}
parent := filepath.ToSlash(filepath.Dir(relPath))
if parent == "." || parent == "" {
root := rootProjectionPath
return &root
}
return &parent
}
func inferredResourceSlug(logicalType string, scope Scope) string {
switch logicalType {
case "department":
return scope.DepartmentSlug
case "team":
return scope.TeamSlug
case "project":
return scope.ProjectSlug
case "personal":
return scope.PersonalSlug
default:
return ""
}
}
func parentEntityName(logicalType string, scope Scope) string {
switch logicalType {
case "department":
return scope.DepartmentSlug
case "team":
return scope.TeamSlug
case "project":
return scope.ProjectSlug
case "personal":
return scope.PersonalSlug
default:
return ""
}
}
func stringValue(value any) string {
stringValue, _ := value.(string)
return strings.TrimSpace(stringValue)
}
func firstNonEmpty(values ...string) string {
for _, value := range values {
trimmed := strings.TrimSpace(value)
if trimmed != "" {
return trimmed
}
}
return ""
}
func summarizeNodes(nodes []Node) RebuildSummary {
summary := RebuildSummary{TotalNodes: len(nodes)}
for _, node := range nodes {
switch node.NodeKind {
case NodeKindDirectory:
summary.DirectoryCount++
case NodeKindFile:
summary.FileCount++
}
}
return summary
}
func errorsIsNotExist(err error) bool {
return err != nil && os.IsNotExist(err)
}

View File

@@ -0,0 +1,148 @@
package posixproj
import (
"encoding/json"
"os"
"path/filepath"
"testing"
)
func TestScanRootBuildsProjectedNodesFromBootstrapShape(t *testing.T) {
root := filepath.Join(t.TempDir(), "POSIX")
mustMkdirAll(t, filepath.Join(root, "catalog", "packs"))
mustMkdirAll(t, filepath.Join(root, "catalog", "standalone"))
mustMkdirAll(t, filepath.Join(root, "departments", "department-primary-department", "teams", "team-primary-team"))
mustMkdirAll(t, filepath.Join(root, "projects", "project-primary-project", "tree"))
mustMkdirAll(t, filepath.Join(root, "users", "personals"))
mustWriteJSON(t, filepath.Join(root, "settings.json"), map[string]any{
"installation": map[string]any{
"id": "installation-1",
"name": "MangoPig",
"isBootstrapped": true,
},
"organization": map[string]any{
"id": "org-1",
"name": "Primary Organization",
"slug": "primary-organization",
},
})
mustWriteJSON(t, filepath.Join(root, "layout.json"), map[string]any{
"type": "tenant-layout",
"home": map[string]any{"defaultProjectSlug": "primary-project"},
})
mustWriteJSON(t, filepath.Join(root, "departments", "department-primary-department", "settings.json"), map[string]any{
"id": "dept-1",
"name": "Primary Department",
"slug": "primary-department",
"type": "department",
})
mustWriteJSON(t, filepath.Join(root, "departments", "department-primary-department", "users.json"), map[string]any{
"users": []map[string]any{{"id": "admin-1"}},
})
mustWriteJSON(t, filepath.Join(root, "departments", "department-primary-department", "teams", "team-primary-team", "settings.json"), map[string]any{
"id": "team-1",
"name": "Primary Team",
"slug": "primary-team",
"type": "team",
})
mustWriteJSON(t, filepath.Join(root, "projects", "project-primary-project", "settings.json"), map[string]any{
"id": "project-1",
"name": "Primary Project",
"slug": "primary-project",
"type": "project",
})
mustWriteJSON(t, filepath.Join(root, "projects", "project-primary-project", "home.json"), map[string]any{
"type": "project-home",
"project": "primary-project",
})
mustWriteJSON(t, filepath.Join(root, "users", "settings.json"), map[string]any{
"primaryAdminId": "admin-1",
})
mustWriteJSON(t, filepath.Join(root, "users", "data.json"), map[string]any{
"users": []map[string]any{{"id": "admin-1", "email": "ronald@example.com"}},
})
nodes, err := ScanRoot(root)
if err != nil {
t.Fatalf("ScanRoot() error = %v", err)
}
index := make(map[string]Node, len(nodes))
for _, node := range nodes {
index[node.Path] = node
}
rootNode, ok := index[rootProjectionPath]
if !ok {
t.Fatalf("expected synthetic root node")
}
if rootNode.LogicalType != "tenant_root" {
t.Fatalf("expected root logical type tenant_root, got %q", rootNode.LogicalType)
}
if rootNode.OrganizationSlug != "primary-organization" {
t.Fatalf("expected root organization slug primary-organization, got %q", rootNode.OrganizationSlug)
}
tenantSettings := index["settings.json"]
if tenantSettings.LogicalType != "tenant" || tenantSettings.FileRole != "settings" {
t.Fatalf("unexpected tenant settings classification: %#v", tenantSettings)
}
if tenantSettings.InstallationID != "installation-1" {
t.Fatalf("expected installation id installation-1, got %q", tenantSettings.InstallationID)
}
deptSettings := index["departments/department-primary-department/settings.json"]
if deptSettings.DepartmentSlug != "primary-department" {
t.Fatalf("expected department slug primary-department, got %q", deptSettings.DepartmentSlug)
}
if deptSettings.ResourceID != "dept-1" {
t.Fatalf("expected department resource id dept-1, got %q", deptSettings.ResourceID)
}
teamSettings := index["departments/department-primary-department/teams/team-primary-team/settings.json"]
if teamSettings.TeamSlug != "primary-team" {
t.Fatalf("expected team slug primary-team, got %q", teamSettings.TeamSlug)
}
projectSettings := index["projects/project-primary-project/settings.json"]
if projectSettings.ProjectSlug != "primary-project" {
t.Fatalf("expected project slug primary-project, got %q", projectSettings.ProjectSlug)
}
if projectSettings.ResourceName != "Primary Project" {
t.Fatalf("expected project resource name Primary Project, got %q", projectSettings.ResourceName)
}
projectTree := index["projects/project-primary-project/tree"]
if projectTree.LogicalType != "project_tree" || projectTree.NodeKind != NodeKindDirectory {
t.Fatalf("unexpected project tree node: %#v", projectTree)
}
usersData := index["users/data.json"]
if usersData.LogicalType != "users" || usersData.FileRole != "data" {
t.Fatalf("unexpected users data classification: %#v", usersData)
}
if usersData.Checksum == "" || usersData.SizeBytes == 0 {
t.Fatalf("expected users/data.json checksum and size to be populated: %#v", usersData)
}
}
func mustMkdirAll(t *testing.T, path string) {
t.Helper()
if err := os.MkdirAll(path, 0o755); err != nil {
t.Fatalf("MkdirAll(%q) error = %v", path, err)
}
}
func mustWriteJSON(t *testing.T, path string, payload any) {
t.Helper()
bytes, err := json.MarshalIndent(payload, "", " ")
if err != nil {
t.Fatalf("MarshalIndent(%q) error = %v", path, err)
}
bytes = append(bytes, '\n')
if err := os.WriteFile(path, bytes, 0o644); err != nil {
t.Fatalf("WriteFile(%q) error = %v", path, err)
}
}