Feat: add POSIX DB projection
This commit is contained in:
MangoPig
531
Backend/internal/posixproj/projector.go
Normal file
531
Backend/internal/posixproj/projector.go
Normal file
@@ -0,0 +1,531 @@
|
||||
package posixproj
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"moku-backend/internal/database"
|
||||
)
|
||||
|
||||
const rootProjectionPath = "/"
|
||||
|
||||
type Projector struct {
|
||||
db *database.DB
|
||||
root string
|
||||
}
|
||||
|
||||
type RebuildSummary struct {
|
||||
TotalNodes int
|
||||
DirectoryCount int
|
||||
FileCount int
|
||||
}
|
||||
|
||||
type NodeKind string
|
||||
|
||||
const (
|
||||
NodeKindDirectory NodeKind = "directory"
|
||||
NodeKindFile NodeKind = "file"
|
||||
)
|
||||
|
||||
type Scope struct {
|
||||
InstallationID string
|
||||
OrganizationID string
|
||||
OrganizationSlug string
|
||||
DepartmentSlug string
|
||||
TeamSlug string
|
||||
ProjectSlug string
|
||||
PersonalSlug string
|
||||
}
|
||||
|
||||
type Node struct {
|
||||
Path string
|
||||
ParentPath *string
|
||||
Name string
|
||||
Depth int
|
||||
NodeKind NodeKind
|
||||
LogicalType string
|
||||
FileRole string
|
||||
ResourceID string
|
||||
ResourceName string
|
||||
ResourceSlug string
|
||||
InstallationID string
|
||||
OrganizationID string
|
||||
OrganizationSlug string
|
||||
DepartmentSlug string
|
||||
TeamSlug string
|
||||
ProjectSlug string
|
||||
PersonalSlug string
|
||||
ContentJSON []byte
|
||||
SizeBytes int64
|
||||
Checksum string
|
||||
}
|
||||
|
||||
func NewProjector(db *database.DB, root string) *Projector {
|
||||
return &Projector{db: db, root: strings.TrimSpace(root)}
|
||||
}
|
||||
|
||||
func (projector *Projector) Rebuild(ctx context.Context) error {
|
||||
_, err := projector.RebuildWithSummary(ctx)
|
||||
return err
|
||||
}
|
||||
|
||||
func (projector *Projector) RebuildWithSummary(ctx context.Context) (RebuildSummary, error) {
|
||||
if projector == nil || projector.db == nil || projector.db.Pool == nil {
|
||||
return RebuildSummary{}, nil
|
||||
}
|
||||
|
||||
nodes, err := ScanRoot(projector.root)
|
||||
if err != nil {
|
||||
return RebuildSummary{}, err
|
||||
}
|
||||
|
||||
summary := summarizeNodes(nodes)
|
||||
|
||||
tx, err := projector.db.Pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return RebuildSummary{}, err
|
||||
}
|
||||
defer func() {
|
||||
_ = tx.Rollback(ctx)
|
||||
}()
|
||||
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM posix_nodes;`); err != nil {
|
||||
return RebuildSummary{}, fmt.Errorf("clear posix_nodes: %w", err)
|
||||
}
|
||||
|
||||
for _, node := range nodes {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO posix_nodes (
|
||||
path,
|
||||
parent_path,
|
||||
name,
|
||||
depth,
|
||||
node_kind,
|
||||
logical_type,
|
||||
file_role,
|
||||
resource_id,
|
||||
resource_name,
|
||||
resource_slug,
|
||||
installation_id,
|
||||
organization_id,
|
||||
organization_slug,
|
||||
department_slug,
|
||||
team_slug,
|
||||
project_slug,
|
||||
personal_slug,
|
||||
content_json,
|
||||
size_bytes,
|
||||
checksum
|
||||
) VALUES (
|
||||
$1, $2, $3, $4, $5::posix_node_kind, $6, $7, $8, $9, $10,
|
||||
$11, $12, $13, $14, $15, $16, $17, $18::jsonb, $19, $20
|
||||
);
|
||||
`,
|
||||
node.Path,
|
||||
node.ParentPath,
|
||||
node.Name,
|
||||
node.Depth,
|
||||
string(node.NodeKind),
|
||||
node.LogicalType,
|
||||
node.FileRole,
|
||||
node.ResourceID,
|
||||
node.ResourceName,
|
||||
node.ResourceSlug,
|
||||
node.InstallationID,
|
||||
node.OrganizationID,
|
||||
node.OrganizationSlug,
|
||||
node.DepartmentSlug,
|
||||
node.TeamSlug,
|
||||
node.ProjectSlug,
|
||||
node.PersonalSlug,
|
||||
node.ContentJSON,
|
||||
node.SizeBytes,
|
||||
node.Checksum,
|
||||
); err != nil {
|
||||
return RebuildSummary{}, fmt.Errorf("insert posix node %s: %w", node.Path, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return RebuildSummary{}, err
|
||||
}
|
||||
|
||||
return summary, nil
|
||||
}
|
||||
|
||||
func ScanRoot(root string) ([]Node, error) {
|
||||
rootPath := strings.TrimSpace(root)
|
||||
if rootPath == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
info, err := os.Stat(rootPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("stat POSIX root: %w", err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return nil, fmt.Errorf("POSIX root is not a directory: %s", rootPath)
|
||||
}
|
||||
|
||||
rootScope, err := loadRootScope(rootPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
nodes := []Node{{
|
||||
Path: rootProjectionPath,
|
||||
ParentPath: nil,
|
||||
Name: filepath.Base(rootPath),
|
||||
Depth: 0,
|
||||
NodeKind: NodeKindDirectory,
|
||||
LogicalType: "tenant_root",
|
||||
InstallationID: rootScope.InstallationID,
|
||||
OrganizationID: rootScope.OrganizationID,
|
||||
OrganizationSlug: rootScope.OrganizationSlug,
|
||||
}}
|
||||
|
||||
err = filepath.WalkDir(rootPath, func(path string, entry fs.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if path == rootPath {
|
||||
return nil
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(rootPath, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
relPath = filepath.ToSlash(relPath)
|
||||
if relPath == "." {
|
||||
return nil
|
||||
}
|
||||
|
||||
node, err := buildNode(rootPath, relPath, entry, rootScope)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
nodes = append(nodes, node)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan POSIX root: %w", err)
|
||||
}
|
||||
|
||||
return nodes, nil
|
||||
}
|
||||
|
||||
func loadRootScope(rootPath string) (Scope, error) {
|
||||
settingsPath := filepath.Join(rootPath, "settings.json")
|
||||
content, err := os.ReadFile(settingsPath)
|
||||
if err != nil {
|
||||
if errorsIsNotExist(err) {
|
||||
return Scope{}, nil
|
||||
}
|
||||
return Scope{}, fmt.Errorf("read root settings.json: %w", err)
|
||||
}
|
||||
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(content, &payload); err != nil {
|
||||
return Scope{}, fmt.Errorf("decode root settings.json: %w", err)
|
||||
}
|
||||
|
||||
installation, _ := payload["installation"].(map[string]any)
|
||||
organization, _ := payload["organization"].(map[string]any)
|
||||
|
||||
return Scope{
|
||||
InstallationID: stringValue(installation["id"]),
|
||||
OrganizationID: stringValue(organization["id"]),
|
||||
OrganizationSlug: stringValue(organization["slug"]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func buildNode(rootPath, relPath string, entry fs.DirEntry, rootScope Scope) (Node, error) {
|
||||
scope := deriveScope(relPath, rootScope)
|
||||
parentPath := projectionParentPath(relPath)
|
||||
logicalType, fileRole := classifyPath(relPath, entry.IsDir())
|
||||
|
||||
node := Node{
|
||||
Path: relPath,
|
||||
ParentPath: parentPath,
|
||||
Name: entry.Name(),
|
||||
Depth: strings.Count(relPath, "/") + 1,
|
||||
NodeKind: NodeKindDirectory,
|
||||
LogicalType: logicalType,
|
||||
FileRole: fileRole,
|
||||
InstallationID: scope.InstallationID,
|
||||
OrganizationID: scope.OrganizationID,
|
||||
OrganizationSlug: scope.OrganizationSlug,
|
||||
DepartmentSlug: scope.DepartmentSlug,
|
||||
TeamSlug: scope.TeamSlug,
|
||||
ProjectSlug: scope.ProjectSlug,
|
||||
PersonalSlug: scope.PersonalSlug,
|
||||
}
|
||||
|
||||
if entry.IsDir() {
|
||||
return node, nil
|
||||
}
|
||||
|
||||
absPath := filepath.Join(rootPath, filepath.FromSlash(relPath))
|
||||
content, err := os.ReadFile(absPath)
|
||||
if err != nil {
|
||||
return Node{}, fmt.Errorf("read POSIX file %s: %w", relPath, err)
|
||||
}
|
||||
|
||||
hash := sha256.Sum256(content)
|
||||
node.NodeKind = NodeKindFile
|
||||
node.SizeBytes = int64(len(content))
|
||||
node.Checksum = hex.EncodeToString(hash[:])
|
||||
|
||||
if strings.EqualFold(filepath.Ext(entry.Name()), ".json") {
|
||||
var payload map[string]any
|
||||
if err := json.Unmarshal(content, &payload); err == nil {
|
||||
jsonContent, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return Node{}, fmt.Errorf("remarshal POSIX file %s: %w", relPath, err)
|
||||
}
|
||||
node.ContentJSON = jsonContent
|
||||
node.ResourceID = stringValue(payload["id"])
|
||||
node.ResourceName = stringValue(payload["name"])
|
||||
node.ResourceSlug = stringValue(payload["slug"])
|
||||
if node.ResourceID == "" && fileRole == "settings" && logicalType == "tenant" {
|
||||
installation, _ := payload["installation"].(map[string]any)
|
||||
organization, _ := payload["organization"].(map[string]any)
|
||||
node.ResourceID = stringValue(installation["id"])
|
||||
node.ResourceName = stringValue(installation["name"])
|
||||
node.InstallationID = stringValue(installation["id"])
|
||||
node.OrganizationID = stringValue(organization["id"])
|
||||
node.OrganizationSlug = firstNonEmpty(node.OrganizationSlug, stringValue(organization["slug"]))
|
||||
}
|
||||
if node.ResourceID == "" && fileRole == "users" {
|
||||
node.ResourceName = firstNonEmpty(node.ResourceName, parentEntityName(logicalType, scope))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if node.ResourceSlug == "" {
|
||||
node.ResourceSlug = inferredResourceSlug(logicalType, scope)
|
||||
}
|
||||
|
||||
return node, nil
|
||||
}
|
||||
|
||||
func deriveScope(relPath string, rootScope Scope) Scope {
|
||||
scope := rootScope
|
||||
parts := strings.Split(relPath, "/")
|
||||
if len(parts) >= 2 && parts[0] == "departments" && strings.HasPrefix(parts[1], "department-") {
|
||||
scope.DepartmentSlug = strings.TrimPrefix(parts[1], "department-")
|
||||
}
|
||||
if len(parts) >= 4 && parts[0] == "departments" && parts[2] == "teams" && strings.HasPrefix(parts[3], "team-") {
|
||||
scope.DepartmentSlug = strings.TrimPrefix(parts[1], "department-")
|
||||
scope.TeamSlug = strings.TrimPrefix(parts[3], "team-")
|
||||
}
|
||||
if len(parts) >= 2 && parts[0] == "projects" && strings.HasPrefix(parts[1], "project-") {
|
||||
scope.ProjectSlug = strings.TrimPrefix(parts[1], "project-")
|
||||
}
|
||||
if len(parts) >= 3 && parts[0] == "users" && parts[1] == "personals" && strings.HasPrefix(parts[2], "personal-") {
|
||||
scope.PersonalSlug = strings.TrimPrefix(parts[2], "personal-")
|
||||
}
|
||||
return scope
|
||||
}
|
||||
|
||||
func classifyPath(relPath string, isDir bool) (logicalType, fileRole string) {
|
||||
parts := strings.Split(relPath, "/")
|
||||
name := parts[len(parts)-1]
|
||||
if !isDir {
|
||||
fileRole = strings.TrimSuffix(name, filepath.Ext(name))
|
||||
}
|
||||
|
||||
switch {
|
||||
case relPath == "settings.json":
|
||||
return "tenant", "settings"
|
||||
case relPath == "layout.json":
|
||||
return "tenant", "layout"
|
||||
case len(parts) >= 1 && parts[0] == "catalog":
|
||||
if isDir {
|
||||
if len(parts) == 1 {
|
||||
return "catalog", ""
|
||||
}
|
||||
if len(parts) >= 2 && parts[1] == "packs" {
|
||||
if len(parts) == 2 {
|
||||
return "catalog_packs", ""
|
||||
}
|
||||
if len(parts) == 3 {
|
||||
return "catalog_pack", ""
|
||||
}
|
||||
if len(parts) >= 4 && parts[3] == "entries" {
|
||||
return "catalog_pack_entries", ""
|
||||
}
|
||||
return "catalog_entry", ""
|
||||
}
|
||||
if len(parts) >= 2 && parts[1] == "standalone" {
|
||||
if len(parts) == 2 {
|
||||
return "catalog_standalone", ""
|
||||
}
|
||||
return "catalog_entry", ""
|
||||
}
|
||||
}
|
||||
return "catalog", fileRole
|
||||
case len(parts) >= 2 && parts[0] == "departments" && strings.HasPrefix(parts[1], "department-"):
|
||||
if isDir {
|
||||
if len(parts) == 2 {
|
||||
return "department", ""
|
||||
}
|
||||
if len(parts) == 3 && parts[2] == "teams" {
|
||||
return "department_teams", ""
|
||||
}
|
||||
if len(parts) >= 4 && strings.HasPrefix(parts[3], "team-") {
|
||||
return "team", ""
|
||||
}
|
||||
}
|
||||
if len(parts) >= 4 && strings.HasPrefix(parts[3], "team-") {
|
||||
return "team", fileRole
|
||||
}
|
||||
return "department", fileRole
|
||||
case len(parts) >= 2 && parts[0] == "projects" && strings.HasPrefix(parts[1], "project-"):
|
||||
if isDir {
|
||||
if len(parts) == 2 {
|
||||
return "project", ""
|
||||
}
|
||||
if len(parts) == 3 && parts[2] == "tree" {
|
||||
return "project_tree", ""
|
||||
}
|
||||
if strings.Contains(relPath, "/tree/") || strings.HasSuffix(relPath, "/tree") {
|
||||
if strings.HasPrefix(name, "folder-") {
|
||||
return "folder", ""
|
||||
}
|
||||
if strings.HasPrefix(name, "item-") {
|
||||
return "item", ""
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.Contains(relPath, "/tree/") {
|
||||
if strings.HasPrefix(parts[len(parts)-2], "item-") {
|
||||
return "item", fileRole
|
||||
}
|
||||
if strings.HasPrefix(parts[len(parts)-2], "folder-") {
|
||||
return "folder", fileRole
|
||||
}
|
||||
}
|
||||
return "project", fileRole
|
||||
case len(parts) >= 1 && parts[0] == "users":
|
||||
if isDir {
|
||||
if len(parts) == 1 {
|
||||
return "users", ""
|
||||
}
|
||||
if len(parts) == 2 && parts[1] == "personals" {
|
||||
return "personals", ""
|
||||
}
|
||||
if len(parts) >= 3 && parts[1] == "personals" && strings.HasPrefix(parts[2], "personal-") {
|
||||
return "personal", ""
|
||||
}
|
||||
if strings.Contains(relPath, "/tree/") || strings.HasSuffix(relPath, "/tree") {
|
||||
if strings.HasPrefix(name, "folder-") {
|
||||
return "folder", ""
|
||||
}
|
||||
if strings.HasPrefix(name, "item-") {
|
||||
return "item", ""
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(parts) >= 3 && parts[1] == "personals" && strings.HasPrefix(parts[2], "personal-") {
|
||||
if strings.Contains(relPath, "/tree/") {
|
||||
if strings.HasPrefix(parts[len(parts)-2], "item-") {
|
||||
return "item", fileRole
|
||||
}
|
||||
if strings.HasPrefix(parts[len(parts)-2], "folder-") {
|
||||
return "folder", fileRole
|
||||
}
|
||||
}
|
||||
return "personal", fileRole
|
||||
}
|
||||
return "users", fileRole
|
||||
default:
|
||||
if isDir {
|
||||
return "directory", ""
|
||||
}
|
||||
return "file", fileRole
|
||||
}
|
||||
}
|
||||
|
||||
func projectionParentPath(relPath string) *string {
|
||||
if relPath == "" || relPath == rootProjectionPath {
|
||||
return nil
|
||||
}
|
||||
parent := filepath.ToSlash(filepath.Dir(relPath))
|
||||
if parent == "." || parent == "" {
|
||||
root := rootProjectionPath
|
||||
return &root
|
||||
}
|
||||
return &parent
|
||||
}
|
||||
|
||||
func inferredResourceSlug(logicalType string, scope Scope) string {
|
||||
switch logicalType {
|
||||
case "department":
|
||||
return scope.DepartmentSlug
|
||||
case "team":
|
||||
return scope.TeamSlug
|
||||
case "project":
|
||||
return scope.ProjectSlug
|
||||
case "personal":
|
||||
return scope.PersonalSlug
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func parentEntityName(logicalType string, scope Scope) string {
|
||||
switch logicalType {
|
||||
case "department":
|
||||
return scope.DepartmentSlug
|
||||
case "team":
|
||||
return scope.TeamSlug
|
||||
case "project":
|
||||
return scope.ProjectSlug
|
||||
case "personal":
|
||||
return scope.PersonalSlug
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func stringValue(value any) string {
|
||||
stringValue, _ := value.(string)
|
||||
return strings.TrimSpace(stringValue)
|
||||
}
|
||||
|
||||
func firstNonEmpty(values ...string) string {
|
||||
for _, value := range values {
|
||||
trimmed := strings.TrimSpace(value)
|
||||
if trimmed != "" {
|
||||
return trimmed
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func summarizeNodes(nodes []Node) RebuildSummary {
|
||||
summary := RebuildSummary{TotalNodes: len(nodes)}
|
||||
for _, node := range nodes {
|
||||
switch node.NodeKind {
|
||||
case NodeKindDirectory:
|
||||
summary.DirectoryCount++
|
||||
case NodeKindFile:
|
||||
summary.FileCount++
|
||||
}
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
func errorsIsNotExist(err error) bool {
|
||||
return err != nil && os.IsNotExist(err)
|
||||
}
|
||||
Reference in New Issue
Block a user