feat: initial public release (MAESTRO)
This commit is contained in:
oss-sync
@@ -0,0 +1,363 @@
|
||||
/**
|
||||
* Git URL install handler for skills.
|
||||
* POST /api/skills/install-from-url
|
||||
*
|
||||
* Two modes:
|
||||
* - Preview: POST { url } → returns detected skills + scan findings
|
||||
* - Install: POST { url, selectedSkills: [...] } → installs selected skills
|
||||
*/
|
||||
|
||||
import type { Request, Response } from 'express';
|
||||
import {
|
||||
existsSync, mkdirSync, mkdtempSync, rmSync, cpSync,
|
||||
readdirSync, lstatSync, readFileSync, writeFileSync,
|
||||
} from 'fs';
|
||||
import { join, relative } from 'path';
|
||||
import { execFileSync } from 'child_process';
|
||||
import { tmpdir } from 'os';
|
||||
import matter from 'gray-matter';
|
||||
import type { SkillCatalog } from '../engine/skills.js';
|
||||
import { VALID_SKILL_NAME } from '../engine/skills.js';
|
||||
import { scanSkillContent, scanSkillDirectory, maxSeverity, type ScanFinding } from '../engine/skills-scanner.js';
|
||||
import { logger } from '../logger.js';
|
||||
|
||||
// ── Types ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export interface GitInstallDeps {
|
||||
skillCatalog: SkillCatalog;
|
||||
auditLog?: (jobId: string | null, action: string, actor: string, detail: object) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface DetectedSkill {
|
||||
name: string;
|
||||
description: string;
|
||||
relativePath: string;
|
||||
fullPath: string;
|
||||
isDir: boolean;
|
||||
findings: ScanFinding[];
|
||||
maxSeverity: 'high' | 'medium' | 'none';
|
||||
}
|
||||
|
||||
// ── Constants ───────────────────────────────────────────────────────────────
|
||||
|
||||
const MAX_REPO_SIZE_BYTES = 50 * 1024 * 1024; // 50 MB
|
||||
const GIT_CLONE_TIMEOUT_MS = 30_000;
|
||||
const SKIP_DIRS = new Set(['.git', 'node_modules', '.github', '.vscode']);
|
||||
|
||||
// ── Helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Walk a cloned directory looking for skills:
|
||||
* - Directories containing SKILL.md
|
||||
* - Standalone .md files with frontmatter `name`
|
||||
*/
|
||||
export function detectSkillsInDir(rootDir: string): DetectedSkill[] {
|
||||
const results: DetectedSkill[] = [];
|
||||
|
||||
function walk(dir: string): void {
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = readdirSync(dir);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
const fullPath = join(dir, entry);
|
||||
|
||||
let stat;
|
||||
try {
|
||||
stat = lstatSync(fullPath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip symlinks entirely (security)
|
||||
if (stat.isSymbolicLink()) continue;
|
||||
|
||||
if (stat.isDirectory()) {
|
||||
if (SKIP_DIRS.has(entry)) continue;
|
||||
|
||||
// Check if this directory is a skill (has SKILL.md)
|
||||
const skillMdPath = join(fullPath, 'SKILL.md');
|
||||
if (existsSync(skillMdPath)) {
|
||||
try {
|
||||
const skillStat = lstatSync(skillMdPath);
|
||||
if (skillStat.isSymbolicLink()) continue; // Skip symlinked SKILL.md
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
const raw = readFileSync(skillMdPath, 'utf-8');
|
||||
const { data } = matter(raw);
|
||||
if (data && typeof data.name === 'string' && data.name && VALID_SKILL_NAME.test(data.name)) {
|
||||
const findings = scanSkillDirectory(fullPath);
|
||||
results.push({
|
||||
name: data.name,
|
||||
description: typeof data.description === 'string' ? data.description : '',
|
||||
relativePath: relative(rootDir, fullPath),
|
||||
fullPath,
|
||||
isDir: true,
|
||||
findings,
|
||||
maxSeverity: maxSeverity(findings),
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Unreadable SKILL.md — skip
|
||||
}
|
||||
// Don't recurse into skill directories (they're self-contained)
|
||||
continue;
|
||||
}
|
||||
|
||||
// Not a skill dir — recurse
|
||||
walk(fullPath);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Standalone .md file
|
||||
if (stat.isFile() && entry.endsWith('.md')) {
|
||||
try {
|
||||
const raw = readFileSync(fullPath, 'utf-8');
|
||||
const { data } = matter(raw);
|
||||
if (data && typeof data.name === 'string' && data.name && VALID_SKILL_NAME.test(data.name)) {
|
||||
const findings = scanSkillContent(raw);
|
||||
results.push({
|
||||
name: data.name,
|
||||
description: typeof data.description === 'string' ? data.description : '',
|
||||
relativePath: relative(rootDir, fullPath),
|
||||
fullPath,
|
||||
isDir: false,
|
||||
findings,
|
||||
maxSeverity: maxSeverity(findings),
|
||||
});
|
||||
}
|
||||
} catch {
|
||||
// Unreadable .md — skip
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
walk(rootDir);
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate that a URL is safe for git clone (SSRF defense).
|
||||
* Only HTTPS URLs are allowed.
|
||||
*/
|
||||
export function validateUrl(url: string): string | null {
|
||||
if (!url || typeof url !== 'string') return 'url is required';
|
||||
const trimmed = url.trim();
|
||||
if (!trimmed.startsWith('https://')) {
|
||||
return 'Only https:// URLs are allowed (http://, file://, ssh://, git:// and local paths are rejected for security)';
|
||||
}
|
||||
// Reject control characters (newlines, NUL, etc.), shell metacharacters,
|
||||
// quotes and whitespace. The clone now runs via execFile (no shell), so this
|
||||
// is defense-in-depth, but it also prevents a newline+quote breakout if the
|
||||
// URL is ever reused in a shell context.
|
||||
if (/[\u0000-\u001f\u007f;&|`$"'\\<>(){}\s]/.test(trimmed)) {
|
||||
return 'URL contains disallowed characters';
|
||||
}
|
||||
// Must parse as a real https URL.
|
||||
let parsed: URL;
|
||||
try {
|
||||
parsed = new URL(trimmed);
|
||||
} catch {
|
||||
return 'URL is not a valid URL';
|
||||
}
|
||||
if (parsed.protocol !== 'https:') {
|
||||
return 'Only https:// URLs are allowed';
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── Handler ─────────────────────────────────────────────────────────────────
|
||||
|
||||
export function handleInstallFromUrl(deps: GitInstallDeps): (req: Request, res: Response) => Promise<void> {
|
||||
return async (req: Request, res: Response): Promise<void> => {
|
||||
const { url, scope: rawScope, selectedSkills } = req.body ?? {};
|
||||
|
||||
// 1. Validate URL
|
||||
const urlError = validateUrl(url);
|
||||
if (urlError) {
|
||||
res.status(400).json({ error: urlError });
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. Validate scope
|
||||
const scope: 'system' | 'user' = rawScope === 'system' ? 'system' : 'user';
|
||||
|
||||
// 3. System scope requires admin
|
||||
const user = req.user as Express.User | undefined;
|
||||
if (scope === 'system' && (!user || user.role !== 'admin')) {
|
||||
res.status(403).json({ error: 'System-scope install requires admin role' });
|
||||
return;
|
||||
}
|
||||
|
||||
const userId = user?.id ?? 'anonymous';
|
||||
|
||||
// 4. Clone to temp directory
|
||||
const tmpBase = mkdtempSync(join(tmpdir(), 'skill-git-'));
|
||||
const cloneDir = join(tmpBase, 'repo');
|
||||
|
||||
try {
|
||||
try {
|
||||
// execFile (no shell): url and cloneDir are passed as literal argv
|
||||
// entries, so shell metacharacters in `url` cannot inject commands.
|
||||
// `--` terminates option parsing so a `url` starting with `-` cannot
|
||||
// be treated as a git flag (defense-in-depth; validateUrl already
|
||||
// requires an https:// prefix).
|
||||
execFileSync(
|
||||
'git',
|
||||
['clone', '--depth', '1', '--no-recurse-submodules', '--no-checkout', '--', url, cloneDir],
|
||||
{ timeout: GIT_CLONE_TIMEOUT_MS, stdio: 'pipe' },
|
||||
);
|
||||
execFileSync(
|
||||
'git',
|
||||
['-C', cloneDir, 'checkout', 'HEAD', '--', '.'],
|
||||
{ timeout: GIT_CLONE_TIMEOUT_MS, stdio: 'pipe' },
|
||||
);
|
||||
} catch (cloneErr: unknown) {
|
||||
const isTimeout = cloneErr instanceof Error && 'killed' in cloneErr && (cloneErr as any).killed;
|
||||
if (isTimeout) {
|
||||
res.status(408).json({ error: 'Git clone timed out (30s limit)' });
|
||||
return;
|
||||
}
|
||||
const msg = cloneErr instanceof Error ? cloneErr.message : String(cloneErr);
|
||||
res.status(400).json({ error: `Git clone failed: ${msg.slice(0, 300)}` });
|
||||
return;
|
||||
}
|
||||
|
||||
// 5. Verify clone exists
|
||||
if (!existsSync(cloneDir)) {
|
||||
res.status(400).json({ error: 'Git clone produced no output directory' });
|
||||
return;
|
||||
}
|
||||
|
||||
// 6. Size check
|
||||
try {
|
||||
const duOutput = execFileSync('du', ['-sb', cloneDir], { encoding: 'utf-8', timeout: 10_000 });
|
||||
const sizeBytes = parseInt(duOutput.split('\t')[0], 10);
|
||||
if (sizeBytes > MAX_REPO_SIZE_BYTES) {
|
||||
res.status(400).json({
|
||||
error: `Repository too large: ${Math.round(sizeBytes / 1024 / 1024)}MB exceeds 50MB limit`,
|
||||
});
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// du failed — continue (non-critical)
|
||||
logger.warn('[skills-git-install] du -sb failed, skipping size check');
|
||||
}
|
||||
|
||||
// 7. Detect skills
|
||||
const detected = detectSkillsInDir(cloneDir);
|
||||
|
||||
if (detected.length === 0) {
|
||||
res.status(400).json({
|
||||
error: 'No skills found in repository. Skills must be directories with SKILL.md or standalone .md files with frontmatter "name".',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// 8. Preview mode — explicitly requested via preview flag
|
||||
const previewMode = req.body?.preview === true;
|
||||
if (previewMode) {
|
||||
const preview = detected.map(s => ({
|
||||
name: s.name,
|
||||
description: s.description,
|
||||
relativePath: s.relativePath,
|
||||
isDir: s.isDir,
|
||||
findings: s.findings,
|
||||
maxSeverity: s.maxSeverity,
|
||||
}));
|
||||
res.json({ preview, totalDetected: detected.length });
|
||||
return;
|
||||
}
|
||||
|
||||
// 9. Install mode — selectedSkills or all detected
|
||||
const selectedSet = Array.isArray(selectedSkills) && selectedSkills.length > 0
|
||||
? new Set(selectedSkills.filter((s: unknown) => typeof s === 'string'))
|
||||
: new Set(detected.map(s => s.name));
|
||||
if (selectedSet.size === 0) {
|
||||
res.status(400).json({ error: 'No skills detected in repository' });
|
||||
return;
|
||||
}
|
||||
|
||||
// Resolve target directory
|
||||
const targetDir = scope === 'system'
|
||||
? deps.skillCatalog.getSystemDir()
|
||||
: deps.skillCatalog.getUserSkillDir(userId);
|
||||
|
||||
if (!existsSync(targetDir)) {
|
||||
mkdirSync(targetDir, { recursive: true });
|
||||
}
|
||||
|
||||
const installed: string[] = [];
|
||||
const errors: string[] = [];
|
||||
|
||||
for (const skill of detected) {
|
||||
if (!selectedSet.has(skill.name)) continue;
|
||||
|
||||
try {
|
||||
if (skill.isDir) {
|
||||
// Copy entire skill directory
|
||||
const destDir = join(targetDir, skill.name);
|
||||
cpSync(skill.fullPath, destDir, { recursive: true });
|
||||
} else {
|
||||
// Single .md file → create as directory format ({name}/SKILL.md)
|
||||
const content = readFileSync(skill.fullPath, 'utf-8');
|
||||
const destDir = join(targetDir, skill.name);
|
||||
mkdirSync(destDir, { recursive: true });
|
||||
writeFileSync(join(destDir, 'SKILL.md'), content, 'utf-8');
|
||||
}
|
||||
installed.push(skill.name);
|
||||
} catch (err: unknown) {
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
errors.push(`${skill.name}: ${msg.slice(0, 200)}`);
|
||||
logger.warn(`[skills-git-install] failed to install skill=${skill.name} err=${msg}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Check for requested skills that weren't found in the repo
|
||||
for (const name of selectedSet) {
|
||||
if (!installed.includes(name) && !errors.some(e => e.startsWith(`${name}:`))) {
|
||||
errors.push(`${name}: not found in repository`);
|
||||
}
|
||||
}
|
||||
|
||||
// 10. Invalidate cache
|
||||
if (scope === 'system') {
|
||||
deps.skillCatalog.refreshSystem();
|
||||
} else {
|
||||
deps.skillCatalog.invalidate(userId);
|
||||
}
|
||||
|
||||
// 11. Audit log
|
||||
if (deps.auditLog && installed.length > 0) {
|
||||
deps.auditLog(null, 'skill_install_from_url', userId, {
|
||||
url,
|
||||
scope,
|
||||
installed,
|
||||
errors: errors.length > 0 ? errors : undefined,
|
||||
}).catch(err => {
|
||||
logger.warn(`[skills-git-install] audit log failed err=${err}`);
|
||||
});
|
||||
}
|
||||
|
||||
logger.info(`[skills-git-install] installed=${installed.length} errors=${errors.length} scope=${scope} user=${userId}`);
|
||||
|
||||
// 12. Return result
|
||||
res.json({ installed, errors: errors.length > 0 ? errors : undefined });
|
||||
|
||||
} finally {
|
||||
// Always clean up temp directory
|
||||
try {
|
||||
rmSync(tmpBase, { recursive: true, force: true });
|
||||
} catch (cleanupErr) {
|
||||
logger.warn(`[skills-git-install] tmpdir cleanup failed: ${cleanupErr}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user