First commit

This commit is contained in:
MangoPig
2026-02-21 21:55:42 +00:00
commit 060629ca94
27 changed files with 2739 additions and 0 deletions
+314
View File
@@ -0,0 +1,314 @@
#Requires -RunAsAdministrator
<#
.SYNOPSIS
JingTian LLM - Windows Sync Setup
.DESCRIPTION
Sets up rclone for automatic file syncing between this Windows machine
and the JingTian Ubuntu server. Run once, then syncing happens automatically.
.NOTES
Run via Setup.bat (double-click) or directly with admin privileges.
#>
param(
[string]$SyncRoot = $PSScriptRoot,
[string]$Password = ""
)
$ErrorActionPreference = "Stop"
# ============================================================================
# Helpers
# ============================================================================
function Write-Step {
param([int]$Number, [string]$Title)
Write-Host ""
Write-Host "[$Number/6] $Title" -ForegroundColor Cyan
Write-Host ("-" * 50)
}
function Read-Config {
param([string]$Path)
$config = @{}
$section = ""
$listItems = @()
$listSection = ""
foreach ($line in Get-Content $Path) {
$line = $line.Trim()
if ($line -eq "" -or $line.StartsWith("#")) { continue }
if ($line -match '^\[(.+)\]$') {
# Save any pending list
if ($listSection -and $listItems.Count -gt 0) {
$config[$listSection] = $listItems
}
$section = $Matches[1]
# Check if this section is a list section (bidirectional)
if ($section -eq "bidirectional") {
$listSection = $section
$listItems = @()
} else {
$listSection = ""
}
continue
}
if ($listSection) {
$listItems += $line
} elseif ($line -match '^(.+?)=(.+)$') {
$key = "$section.$($Matches[1].Trim())"
$config[$key] = $Matches[2].Trim()
}
}
# Save final list section
if ($listSection -and $listItems.Count -gt 0) {
$config[$listSection] = $listItems
}
return $config
}
# ============================================================================
# Main Setup
# ============================================================================
Write-Host ""
Write-Host "========================================" -ForegroundColor Yellow
Write-Host " JingTian LLM - Sync Setup" -ForegroundColor Yellow
Write-Host "========================================" -ForegroundColor Yellow
Write-Host ""
# Determine paths
$configPath = Join-Path $SyncRoot "config.ini"
$rclonePath = Join-Path $SyncRoot "rclone"
$rcloneExe = Join-Path $rclonePath "rclone.exe"
$rcloneConf = Join-Path $rclonePath "rclone.conf"
$logsPath = Join-Path $SyncRoot "logs"
$syncScript = Join-Path $SyncRoot "sync.ps1"
$ignoreFile = Join-Path $SyncRoot ".sync_ignore"
# BenjaminTeam is two levels up from Code/Sync/
$benjaminTeam = (Resolve-Path (Join-Path $SyncRoot "..\..\.." )).Path
Write-Host "Sync root: $SyncRoot"
Write-Host "BenjaminTeam: $benjaminTeam"
Write-Host ""
# Read config
if (-not (Test-Path $configPath)) {
Write-Host "ERROR: config.ini not found at $configPath" -ForegroundColor Red
exit 1
}
$config = Read-Config $configPath
$serverHost = $config["server.host"]
$serverUser = $config["server.user"]
$serverPort = $config["server.port"]
$remotePath = $config["server.remote_path"]
$remoteName = $config["sync.remote_name"]
$interval = $config["sync.interval_minutes"]
Write-Host "Server: ${serverUser}@${serverHost}:${serverPort}" -ForegroundColor Gray
Write-Host "Remote: ${remoteName}:${remotePath}" -ForegroundColor Gray
# --------------------------------------------------------------------------
# Step 1: Install rclone
# --------------------------------------------------------------------------
Write-Step 1 "Installing rclone"
if (Test-Path $rcloneExe) {
$version = & $rcloneExe version 2>&1 | Select-Object -First 1
Write-Host "rclone already installed: $version" -ForegroundColor Green
} else {
Write-Host "Downloading rclone..."
$zipUrl = "https://downloads.rclone.org/rclone-current-windows-amd64.zip"
$zipPath = Join-Path $env:TEMP "rclone.zip"
$extractPath = Join-Path $env:TEMP "rclone-extract"
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
Invoke-WebRequest -Uri $zipUrl -OutFile $zipPath -UseBasicParsing
Write-Host "Extracting..."
if (Test-Path $extractPath) { Remove-Item $extractPath -Recurse -Force }
Expand-Archive -Path $zipPath -DestinationPath $extractPath -Force
# rclone extracts into a subfolder like rclone-v1.68.2-windows-amd64/
$rcloneSubDir = Get-ChildItem $extractPath -Directory | Select-Object -First 1
if (-not (Test-Path $rclonePath)) { New-Item -ItemType Directory -Path $rclonePath -Force | Out-Null }
Copy-Item (Join-Path $rcloneSubDir.FullName "rclone.exe") $rcloneExe -Force
# Cleanup
Remove-Item $zipPath -Force -ErrorAction SilentlyContinue
Remove-Item $extractPath -Recurse -Force -ErrorAction SilentlyContinue
$version = & $rcloneExe version 2>&1 | Select-Object -First 1
Write-Host "Installed: $version" -ForegroundColor Green
}
# --------------------------------------------------------------------------
# Step 2: Configure rclone (password-based SFTP)
# --------------------------------------------------------------------------
Write-Step 2 "Configuring rclone remote"
if ($Password -ne "") {
$plainPassword = $Password
Write-Host "Using provided password" -ForegroundColor Green
} else {
Write-Host "Enter the password for ${serverUser}@${serverHost}:" -ForegroundColor Yellow
$securePassword = Read-Host -AsSecureString
$plainPassword = [Runtime.InteropServices.Marshal]::PtrToStringAuto(
[Runtime.InteropServices.Marshal]::SecureStringToBSTR($securePassword)
)
}
# Obscure the password using rclone
$obscuredPassword = & $rcloneExe obscure $plainPassword 2>&1
$plainPassword = $null # Clear from memory
$rcloneConfContent = @"
[$remoteName]
type = sftp
host = $serverHost
user = $serverUser
port = $serverPort
pass = $obscuredPassword
shell_type = unix
md5sum_command = md5sum
sha1sum_command = sha1sum
"@
# Write without BOM (rclone can't parse BOM)
$utf8NoBom = New-Object System.Text.UTF8Encoding $false
[System.IO.File]::WriteAllText($rcloneConf, $rcloneConfContent, $utf8NoBom)
Write-Host "rclone config written to $rcloneConf" -ForegroundColor Green
# --------------------------------------------------------------------------
# Step 3: Test connection
# --------------------------------------------------------------------------
Write-Step 3 "Testing connection to server"
try {
$testOutput = & $rcloneExe lsd "${remoteName}:${remotePath}" --config $rcloneConf 2>&1
if ($LASTEXITCODE -ne 0) {
Write-Host "ERROR: Connection failed!" -ForegroundColor Red
Write-Host $testOutput
Write-Host ""
Write-Host "Check the server IP, username, and password in config.ini" -ForegroundColor Yellow
exit 1
}
Write-Host "Connection successful! Remote directories:" -ForegroundColor Green
$testOutput | ForEach-Object { Write-Host " $_" -ForegroundColor Gray }
} catch {
Write-Host "ERROR: Connection failed: $_" -ForegroundColor Red
exit 1
}
# --------------------------------------------------------------------------
# Step 4: Generate .sync_ignore
# --------------------------------------------------------------------------
Write-Step 4 "Generating sync exclusion rules"
$ignoreContent = @"
# JingTian Sync Ignore Rules
# rclone filter syntax - one rule per line
# See: https://rclone.org/filtering/
# The _LLM directory (scripts, logs, config - managed via git, not sync)
- _LLM/**
# Dotfiles and hidden items
- .*
# Office temp/lock files
- ~`$*
- *.tmp
- ~*.tmp
# OS generated files
- Thumbs.db
- desktop.ini
- .DS_Store
- ehthumbs.db
# Windows shortcuts
- *.lnk
"@
# Write without BOM (rclone can't parse BOM)
$utf8NoBom = New-Object System.Text.UTF8Encoding $false
[System.IO.File]::WriteAllText($ignoreFile, $ignoreContent, $utf8NoBom)
Write-Host "Sync ignore rules written to $ignoreFile" -ForegroundColor Green
# --------------------------------------------------------------------------
# Step 5: Create logs directory
# --------------------------------------------------------------------------
Write-Step 5 "Setting up logging"
if (-not (Test-Path $logsPath)) {
New-Item -ItemType Directory -Path $logsPath -Force | Out-Null
}
Write-Host "Logs directory: $logsPath" -ForegroundColor Green
# --------------------------------------------------------------------------
# Step 6: Create scheduled task
# --------------------------------------------------------------------------
Write-Step 6 "Creating scheduled task"
$taskName = "JingTian-Sync"
# Remove existing task if present
$existing = Get-ScheduledTask -TaskName $taskName -ErrorAction SilentlyContinue
if ($existing) {
Write-Host "Removing existing task '$taskName'..." -ForegroundColor Yellow
Unregister-ScheduledTask -TaskName $taskName -Confirm:$false
}
# Build the action - run sync.ps1 via PowerShell
$action = New-ScheduledTaskAction `
-Execute "powershell.exe" `
-Argument "-ExecutionPolicy Bypass -WindowStyle Hidden -File `"$syncScript`" -SyncRoot `"$SyncRoot`"" `
-WorkingDirectory $SyncRoot
# Trigger: every N minutes, indefinitely
$trigger = New-ScheduledTaskTrigger -Once -At (Get-Date) `
-RepetitionInterval (New-TimeSpan -Minutes ([int]$interval)) `
-RepetitionDuration (New-TimeSpan -Days 9999)
# Settings
$settings = New-ScheduledTaskSettingsSet `
-AllowStartIfOnBatteries `
-DontStopIfGoingOnBatteries `
-StartWhenAvailable `
-RunOnlyIfNetworkAvailable `
-MultipleInstances IgnoreNew
# Register as SYSTEM so it runs even when user is logged out
Register-ScheduledTask `
-TaskName $taskName `
-Action $action `
-Trigger $trigger `
-Settings $settings `
-RunLevel Highest `
-User "SYSTEM" `
-Description "JingTian LLM - Sync files with Ubuntu server every $interval minutes" `
| Out-Null
Write-Host "Scheduled task '$taskName' created (every $interval minutes)" -ForegroundColor Green
# ============================================================================
# Done
# ============================================================================
Write-Host ""
Write-Host "========================================" -ForegroundColor Green
Write-Host " Setup Complete!" -ForegroundColor Green
Write-Host "========================================" -ForegroundColor Green
Write-Host ""
Write-Host "Sync will run automatically every $interval minutes." -ForegroundColor White
Write-Host "Logs are saved to: $logsPath" -ForegroundColor White
Write-Host ""
Write-Host "To sync manually, run:" -ForegroundColor Gray
Write-Host " powershell -File `"$syncScript`" -SyncRoot `"$SyncRoot`"" -ForegroundColor Gray
Write-Host ""
+22
View File
@@ -0,0 +1,22 @@
# JingTian Sync Configuration
# Edit these values to match your server setup.
[server]
host=SERVER_IP
user=SERVER_USER
port=22
remote_path=/data/jingtian/BenjaminTeam
[sync]
# How often the sync runs (in minutes)
interval_minutes=5
# rclone remote name (used internally)
remote_name=jingtian
[bidirectional]
# Files that sync BOTH ways (Ubuntu -> Windows AND Windows -> Ubuntu).
# These are files the pipeline generates on Ubuntu that Mr. Choi can also edit.
# One path per line, relative to BenjaminTeam/.
# Lines starting with # are comments.
Admin/Tracker.xlsx
+240
View File
@@ -0,0 +1,240 @@
<#
.SYNOPSIS
JingTian LLM - File Sync Script
.DESCRIPTION
Syncs files between Windows (BenjaminTeam/) and the Ubuntu server.
- PULL first: Ubuntu -> Windows for bidirectional files (pipeline outputs)
- PUSH second: Windows -> Ubuntu (one-way, Windows is source of truth)
Order matters: pull gets pipeline updates, then push sends everything
(including those updates + Mr. Choi's edits) back as source of truth.
Called automatically by Windows Task Scheduler every N minutes.
#>
param(
[string]$SyncRoot = $PSScriptRoot
)
$ErrorActionPreference = "Continue"
# ============================================================================
# Helpers
# ============================================================================
function Read-Config {
param([string]$Path)
$config = @{}
$section = ""
$listItems = @()
$listSection = ""
foreach ($line in Get-Content $Path) {
$line = $line.Trim()
if ($line -eq "" -or $line.StartsWith("#")) { continue }
if ($line -match '^\[(.+)\]$') {
if ($listSection -and $listItems.Count -gt 0) {
$config[$listSection] = $listItems
}
$section = $Matches[1]
if ($section -eq "bidirectional") {
$listSection = $section
$listItems = @()
} else {
$listSection = ""
}
continue
}
if ($listSection) {
$listItems += $line
} elseif ($line -match '^(.+?)=(.+)$') {
$key = "$section.$($Matches[1].Trim())"
$config[$key] = $Matches[2].Trim()
}
}
if ($listSection -and $listItems.Count -gt 0) {
$config[$listSection] = $listItems
}
return $config
}
function Write-Log {
param([string]$Message, [string]$Level = "INFO")
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
$entry = "[$timestamp] [$Level] $Message"
Add-Content -Path $logFile -Value $entry
if ($Level -eq "ERROR") {
Write-Host $entry -ForegroundColor Red
}
}
# ============================================================================
# Setup
# ============================================================================
$configPath = Join-Path $SyncRoot "config.ini"
$rcloneExe = Join-Path $SyncRoot "rclone\rclone.exe"
$rcloneConf = Join-Path $SyncRoot "rclone\rclone.conf"
$logsPath = Join-Path $SyncRoot "logs"
$ignoreFile = Join-Path $SyncRoot ".sync_ignore"
# BenjaminTeam is three levels up from Code/Sync/ (_LLM -> BenjaminTeam)
$benjaminTeam = (Resolve-Path (Join-Path $SyncRoot "..\..\.." )).Path
# Log file: one per day
$logDate = Get-Date -Format "yyyy-MM-dd"
$logFile = Join-Path $logsPath "sync-$logDate.log"
# Ensure logs directory
if (-not (Test-Path $logsPath)) {
New-Item -ItemType Directory -Path $logsPath -Force | Out-Null
}
# Preflight checks
if (-not (Test-Path $configPath)) {
Write-Log "config.ini not found at $configPath" "ERROR"
exit 1
}
if (-not (Test-Path $rcloneExe)) {
Write-Log "rclone.exe not found at $rcloneExe. Run Setup.bat first." "ERROR"
exit 1
}
if (-not (Test-Path $rcloneConf)) {
Write-Log "rclone.conf not found at $rcloneConf. Run Setup.bat first." "ERROR"
exit 1
}
# Read config
$config = Read-Config $configPath
$remoteName = $config["sync.remote_name"]
$remotePath = $config["server.remote_path"]
$bidirectionalFiles = @()
if ($config.ContainsKey("bidirectional")) {
$bidirectionalFiles = $config["bidirectional"]
}
$remoteBase = "${remoteName}:${remotePath}"
Write-Log "=== Sync started ==="
Write-Log "BenjaminTeam: $benjaminTeam"
Write-Log "Remote: $remoteBase"
# ============================================================================
# Phase 1: PULL bidirectional files (Ubuntu -> Windows)
# Runs FIRST so pipeline updates reach Windows before push sends everything back.
# Uses --update: only copies if remote file is newer than local.
# This means Mr. Choi's recent edits are preserved (his file is newer).
# ============================================================================
if ($bidirectionalFiles.Count -gt 0) {
Write-Log "--- Phase 1: PULL bidirectional files (Ubuntu -> Windows) ---"
foreach ($relPath in $bidirectionalFiles) {
# Normalize path separators
$relPath = $relPath.Replace("/", "\")
$remoteSrc = "${remoteBase}/$($relPath.Replace('\', '/'))"
$localDest = Join-Path $benjaminTeam $relPath
$localDir = Split-Path $localDest -Parent
# Ensure local directory exists
if (-not (Test-Path $localDir)) {
New-Item -ItemType Directory -Path $localDir -Force | Out-Null
}
Write-Log "Pulling: $relPath"
try {
$pullOutput = & $rcloneExe copyto $remoteSrc $localDest `
--config $rcloneConf `
--update `
--contimeout 30s `
--timeout 60s `
--retries 3 `
--log-level NOTICE 2>&1
$pullExitCode = $LASTEXITCODE
if ($pullExitCode -eq 0) {
Write-Log " Pulled OK: $relPath"
} else {
# Exit code 3 = directory not found / file doesn't exist yet (OK)
if ($pullExitCode -eq 3) {
Write-Log " Not found on server (yet): $relPath"
} else {
Write-Log " Pull failed (exit code: $pullExitCode): $relPath" "ERROR"
foreach ($line in $pullOutput) {
Write-Log " $line" "ERROR"
}
}
}
} catch {
Write-Log " Pull exception for ${relPath}: $_" "ERROR"
}
}
} else {
Write-Log "No bidirectional files configured, skipping pull phase"
}
# ============================================================================
# Phase 2: PUSH (Windows -> Ubuntu)
# Windows is source of truth. rclone sync mirrors local to remote.
# Bidirectional files are EXCLUDED from push — they are managed by the
# pipeline on Ubuntu and pulled down in Phase 1.
# ============================================================================
Write-Log "--- Phase 2: PUSH (Windows -> Ubuntu) ---"
$pushArgs = @(
"sync"
$benjaminTeam
$remoteBase
"--config", $rcloneConf
"--transfers", "4"
"--checkers", "8"
"--contimeout", "30s"
"--timeout", "120s"
"--retries", "3"
"--log-level", "NOTICE"
)
# Add filter file if it exists
if (Test-Path $ignoreFile) {
$pushArgs += "--filter-from"
$pushArgs += $ignoreFile
}
# Exclude bidirectional files from push (pipeline manages these on Ubuntu)
foreach ($relPath in $bidirectionalFiles) {
$excludePath = $relPath.Replace("\", "/")
$pushArgs += "--exclude"
$pushArgs += $excludePath
}
$pushStart = Get-Date
Write-Log "Running: rclone sync (push)"
try {
$pushOutput = & $rcloneExe @pushArgs 2>&1
$pushExitCode = $LASTEXITCODE
$pushDuration = ((Get-Date) - $pushStart).TotalSeconds
if ($pushExitCode -eq 0) {
Write-Log "Push completed in ${pushDuration}s"
} else {
Write-Log "Push failed (exit code: $pushExitCode) in ${pushDuration}s" "ERROR"
foreach ($line in $pushOutput) {
Write-Log " $line" "ERROR"
}
}
} catch {
Write-Log "Push exception: $_" "ERROR"
}
# ============================================================================
# Done
# ============================================================================
Write-Log "=== Sync finished ==="