This commit is contained in:
2026-04-25 02:32:22 +09:00
parent 52aaa4d1f6
commit 8a8c0c9713
38 changed files with 1033 additions and 38 deletions

View File

@@ -1,49 +0,0 @@
import { parseTja } from './parse';
import { factorize } from './factorize';
import { readdirSync, readFileSync, writeFileSync, existsSync, mkdirSync } from 'fs';
import { join, dirname } from 'path';
import iconv from 'iconv-lite';
const [,, workingDir, dataDir] = process.argv;
const featurePath = join(workingDir, 'factors.json');
if (existsSync(featurePath)) {
console.log('factors.json already exists. Skipping extraction.');
process.exit(0);
}
// 작업 디렉토리 존재 확인 및 생성
if (!existsSync(workingDir)) {
mkdirSync(workingDir, { recursive: true });
}
const tjaDir = join(dataDir, 'tja');
const results: any[] = [];
for (const file of readdirSync(tjaDir)) {
if (!file.endsWith('.tja')) continue;
const songno = file.replace(/\D/g, '');
const buffer = readFileSync(join(tjaDir, file));
let content = iconv.decode(buffer, 'shift-jis', { stripBOM: true });
content = content.replace(/\uFFFD/g, '').replace(/[\u{0080}-\u{FFFF}]/gu, '').replace(/\r\n/g, '\n');
let parsed = parseTja(content);
if (!parsed) {
content = iconv.decode(buffer, 'utf-8', { stripBOM: true });
content = content.replace(/\uFFFD/g, '').replace(/[\u{0080}-\u{FFFF}]/gu, '').replace(/\r\n/g, '\n');
parsed = parseTja(content);
}
if (!parsed) continue;
for (const diff of ['oni', 'edit'] as const) {
if (!parsed[diff]) continue;
const factors = factorize(parsed[diff]!);
results.push({ songno, diff: diff === 'oni' ? 'oni' : 'ura', ...factors });
}
}
writeFileSync(featurePath, JSON.stringify(results, null, 2));
console.log(`Features extracted to ${featurePath}`);

View File

@@ -1,114 +0,0 @@
import { Course, Bar, Note, HitNote } from 'tja-parser';
export namespace Factor {
export function filterHitNotes(course: Course) {
const notes: HitNote[] = [];
course.noteGroups.forEach((g) => {
if (g instanceof Bar) {
for (const note of g.getNotes()) {
if (note instanceof HitNote) {
notes.push(note);
}
}
}
});
return notes;
}
export function getAllNotes(course: Course) {
const notes: { type: number, time: number }[] = [];
for (const group of course.noteGroups) {
if (group instanceof Bar) {
notes.push(...group.getNotes().map((note) => ({
//@ts-expect-error
type: note.type,
time: note.getTimingMS()
})));
}
}
return notes.sort((a, b) => a.time - b.time).filter(n => [1, 2, 3, 4].includes(n.type));
}
// 밀도 관련
export function getAverageDensity(notes: HitNote[]) {
return notes.length / (notes[notes.length - 1].getTimingMS() - notes[0].getTimingMS()) * 1000
}
export function getPeakDensity(notes: HitNote[]) {
let peakDensity = 0;
for (let i = 0; i < notes.length; i++) {
let noteCount = 1;
for (let j = i + 1; j < notes.length && (notes[j].getTimingMS() - notes[i].getTimingMS() <= 1000); j++) {
noteCount++;
}
if (peakDensity < noteCount) {
peakDensity = noteCount;
}
}
return peakDensity;
}
// BPM 관련
export function getAverageBPM(notes: HitNote[]) {
const averageBPM = notes.reduce((p, note) => p + note.getBPM().valueOf(), 0) / notes.length;
return averageBPM;
}
export function getBpmChange(notes: HitNote[]) {
let bpmChange = 0;
for (let i = 1; i < notes.length; i++) {
if (Math.abs(notes[i].getBPM() - notes[i - 1].getBPM()) > 1.5) {
bpmChange++;
}
};
return bpmChange;
}
// 복잡성
export function getComplexity(notes: HitNote[]) {
let complexity = 0;
for (let i = 2; i < notes.length; i++) {
let localComplexity = 0;
// ddk 또는 dkk류면 1, 아니면 0.5
if (
(notes[i].type % 2 === notes[i - 1].type % 2 && notes[i - 1].type % 2 !== notes[i - 2].type % 2) ||
(notes[i].type % 2 !== notes[i - 1].type % 2 && notes[i - 1].type % 2 === notes[i - 2].type % 2)
) {
localComplexity = 1;
} else {
localComplexity = 0.5
}
// 시간 차가 짧을 수록 complexity 증가
localComplexity *= (1 / (notes[i].getTimingMS() - notes[i - 1].getTimingMS()) + 1 / (notes[i - 1].getTimingMS() - notes[i - 2].getTimingMS()))
complexity += localComplexity;
}
return complexity / notes.length;
}
// 스크롤 변화
export function getScrollChange(notes: HitNote[]) {
let bpmChange = 0;
for (let i = 1; i < notes.length; i++) {
if (Math.abs(notes[i].getBPM() * notes[i].getScroll() - notes[i - 1].getBPM() * notes[i - 1].getScroll()) > 1.5) {
bpmChange++;
}
};
return bpmChange;
}
}
export function factorize(course: Course) {
const notes = Factor.filterHitNotes(course)
return {
note_count: notes.length,
density_avg: Factor.getAverageDensity(notes),
density_peak: Factor.getPeakDensity(notes),
bpm_avg: Factor.getAverageBPM(notes),
bpm_change: Factor.getBpmChange(notes),
complexity: Factor.getComplexity(notes),
scroll_change: Factor.getScrollChange(notes)
};
}

View File

@@ -1,51 +0,0 @@
import tjaParser, { Bar, Branch, Course } from 'tja-parser'
export function parseTja(tja: string): Partial<Record<'oni' | 'edit', Course>> | null {
try {
const song = tjaParser.Song.parse(tja);
let oni: Course | undefined = undefined;
let edit: Course | undefined = undefined;
if (song.course?.oni) {
const noteGroups = song.course.oni.noteGroups;
oni = song.course.oni;
oni.noteGroups = []
for (const noteGroup of noteGroups) {
if (noteGroup instanceof Bar) {
oni.pushNoteGroups(noteGroup)
}
else if (noteGroup instanceof Branch) {
const bar = noteGroup.master || noteGroup.advanced || noteGroup.normal;
if (bar) {
oni.pushNoteGroups(...bar)
}
}
}
}
if (song.course?.edit) {
const noteGroups = song.course.edit.noteGroups;
edit = song.course.edit;
edit.noteGroups = []
for (const noteGroup of noteGroups) {
if (noteGroup instanceof Bar) {
edit.pushNoteGroups(noteGroup)
}
else if (noteGroup instanceof Branch) {
const bar = noteGroup.master || noteGroup.advanced || noteGroup.normal;
if (bar) {
edit.pushNoteGroups(...bar)
}
}
}
}
return { oni, edit }
}
catch (err){
console.error(err)
return null;
}
}

View File

@@ -1,97 +0,0 @@
import { parseTja } from './parse';
import { factorize } from './factorize';
import { join } from 'path';
import { readFileSync, writeFileSync, unlinkSync, existsSync } from 'fs';
import { execSync } from 'child_process';
/**
* 예측 엔진: 모델 실행을 위해 Python 환경을 호출
*/
function runInference(workingDir: string, factors: Record<string, number>): number {
const tempFactorPath = join(workingDir, `temp_feat_${Date.now()}.json`);
writeFileSync(tempFactorPath, JSON.stringify(factors));
const pythonScript = `
import json, torch, sys
import numpy as np
import torch.nn as nn
# 모델 정의 (학습된 구조와 동일해야 함)
class DifficultyNet(nn.Module):
def __init__(self, input_dim):
super().__init__()
self.net = nn.Sequential(
nn.Linear(input_dim, 128),
nn.BatchNorm1d(128),
nn.ReLU(),
nn.Dropout(0.4),
nn.Linear(128, 64),
nn.ReLU(),
nn.Dropout(0.3),
nn.Linear(64, 32),
nn.ReLU(),
nn.Linear(32, 1),
nn.Sigmoid()
)
def forward(self, x): return self.net(x)
def predict():
try:
with open('${tempFactorPath}', 'r') as f: factors = json.load(f)
with open('${join(workingDir, 'scaler.json')}', 'r') as f: s = json.load(f)
X = np.array([[factors[col] for col in s['cols']]], dtype=np.float32)
# 정규화 (Robust/Standard 지원)
if 'center' in s: X_scaled = (X - np.array(s['center'])) / np.array(s['scale'])
else: X_scaled = (X - np.array(s.get('mean', 0))) / np.array(s.get('std', s.get('scale', 1)))
model = DifficultyNet(len(s['cols']))
model.load_state_dict(torch.load('${join(workingDir, 'model.pth')}', map_location='cpu'))
model.eval()
with torch.no_grad():
input_tensor = torch.from_numpy(X_scaled).float()
pred = model(input_tensor) * 12 + 0.5
print(float(pred.item()))
except Exception as e:
print(f"ERROR: {e}", file=sys.stderr)
sys.exit(1)
if __name__ == "__main__": predict()
`;
const tempPyPath = join(workingDir, `inference_${Date.now()}.py`);
writeFileSync(tempPyPath, pythonScript);
try {
const result = execSync(`python3 ${tempPyPath}`).toString().trim();
return parseFloat(result);
} finally {
if (existsSync(tempFactorPath)) unlinkSync(tempFactorPath);
if (existsSync(tempPyPath)) unlinkSync(tempPyPath);
}
}
function main() {
const [workingDir, tjaPath, difficulty = 'oni'] = process.argv.slice(2);
if (!workingDir || !tjaPath) {
console.log("Usage: bun run script/predict.ts <workingDir> <tjaPath> [difficulty]");
process.exit(1);
}
const tjaContent = readFileSync(tjaPath, 'utf-8');
const parsed = parseTja(tjaContent);
const chart = parsed[difficulty.toLowerCase() as 'oni' | 'edit'];
if (!chart) {
console.error(`Error: Difficulty '${difficulty}' not found.`);
process.exit(1);
}
const score = runInference(workingDir, factorize(chart));
console.log(`\n🎯 Predicted Difficulty: ${score.toFixed(2)}`);
}
main();

65
script/preprocess.ts Normal file
View File

@@ -0,0 +1,65 @@
import Bun from 'bun';
import path from 'node:path';
import { parseArgs } from 'node:util';
import fs, { mkdirSync } from 'node:fs';
import { Song } from 'tja-parser';
import { featurize } from '../preprocess/featurize';
import { parseTja } from '../preprocess/parse'
const { values } = parseArgs({
args: Bun.argv,
options: {
outputDir: {
type: "string"
},
dataDir: {
type: "string"
}
},
strict: true,
allowPositionals: true,
})
if (!values.dataDir || !values.outputDir) {
console.error("--outputDir --dataDir");
process.exit(1);
}
const outputDir = values.outputDir ?? '';
if (!fs.existsSync(outputDir)) mkdirSync(outputDir)
const dataDir = values.dataDir ?? '';
const tjaDir = path.join(dataDir, 'tja');
const files = fs.readdirSync(tjaDir);
const features: ({ songno: string, difficulty: 'oni' | 'ura' } & {})[] = [];
for (const file of files) {
const tja = fs.readFileSync(path.join(tjaDir, file), 'utf-8');
const songno = path.basename(file, '.tja');
try {
const parsed = parseTja(tja);
const oni = parsed?.oni;
const edit = parsed?.edit;
if (oni) {
features.push({
songno,
difficulty: 'oni',
...featurize(oni)
})
}
if (edit) {
features.push({
songno,
difficulty: 'ura',
...featurize(edit)
})
}
}
catch (err) {
console.error(err);
console.error(file);
}
}
const featurePath = path.join(outputDir, 'features.json');
fs.writeFileSync(featurePath, JSON.stringify(features, null, 2), 'utf-8');

View File

@@ -1,152 +0,0 @@
import pandas as pd
import json, os, sys
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
from sklearn.preprocessing import StandardScaler
MAX_MARGIN_LIMIT = 10
class DifficultyNet(nn.Module):
def __init__(self, input_dim):
super(DifficultyNet, self).__init__()
self.net = nn.Sequential(
nn.Linear(input_dim, 128),
nn.BatchNorm1d(128),
nn.ReLU(),
nn.Dropout(0.4),
nn.Linear(128, 64),
nn.ReLU(),
nn.Dropout(0.3),
nn.Linear(64, 32),
nn.ReLU(),
nn.Linear(32, 1),
nn.Sigmoid()
)
def forward(self, x):
return self.net(x)
def train(script_dir, working_dir, data_dir, train_count, val_count, margin, val_iterations):
device = torch.device("mps" if torch.backends.mps.is_available() else "cpu")
print(f"🚀 Using Device: {device}")
# 데이터 로드 및 병합
factors_path = os.path.join(working_dir, 'factors.json')
if not os.path.exists(factors_path):
os.system(f"bun run {os.path.join(script_dir, 'factorize.ts')} {data_dir} {working_dir}")
df_feat = pd.read_json(factors_path)
df_meas = pd.read_csv(os.path.join(data_dir, 'measure.csv'))
df_meas['diff'] = df_meas['diff'].replace('ura', 'edit')
df_feat['songno'], df_meas['songno'] = df_feat['songno'].astype(int), df_meas['songno'].astype(int)
df = pd.merge(df_feat, df_meas, on=['songno', 'diff'])
exclude_cols = ['songno', 'diff', 'title', 'course', '상수', 'predicted_measure', 'error']
X_cols = [c for c in df_feat.columns if c not in exclude_cols]
scaler = StandardScaler()
df[X_cols] = scaler.fit_transform(df[X_cols])
with open(os.path.join(working_dir, 'scaler.json'), 'w') as f:
json.dump({'mean': scaler.mean_.tolist(), 'std': scaler.scale_.tolist(), 'cols': X_cols}, f)
model_path = os.path.join(working_dir, 'model.pth')
margin = float(margin)
val_iterations = int(val_iterations)
model = DifficultyNet(len(X_cols)).to(device)
attempt = 1
while True:
print(f"\n[Attempt {attempt}] " + "-"*40)
if os.path.exists(model_path):
try: model.load_state_dict(torch.load(model_path, map_location=device))
except: pass
train_df = df.sample(n=min(int(train_count), len(df)))
X_train = torch.FloatTensor(train_df[X_cols].values).to(device)
y_train = torch.FloatTensor(((train_df['상수'].values - 1) / 11)).unsqueeze(1).to(device)
base_lr = 0.001
optimizer = optim.Adam(model.parameters(), lr=base_lr, weight_decay=1e-5)
criterion = nn.L1Loss()
model.train()
epoch = 0
while True:
optimizer.zero_grad()
preds_train = model(X_train)
loss = criterion(preds_train, y_train) # MSELoss 권장
loss.backward()
optimizer.step()
# 현재 학습 데이터에 대한 오차 분석
with torch.no_grad():
diff = torch.abs((preds_train * 12 + 0.5) - (y_train * 12 + 0.5))
train_mae = torch.mean(diff).item()
train_max = torch.max(diff).item()
if epoch % 1000 == 0:
print(f" - Ep {epoch:5d} | MAE: {train_mae:.4f} | MAX: {train_max:.4f}")
# ✅ 조건 강화: MAE뿐만 아니라 MAX도 어느 정도 잡혔을 때만 검증으로 이동
if train_mae < margin and train_max < (margin * MAX_MARGIN_LIMIT):
print(f" ✅ Train Goal Reached (MAE: {train_mae} < {margin:.4f}, MAX: {train_max:.4f} < {(margin * MAX_MARGIN_LIMIT):.4f}). Moving to Val.")
break
epoch += 1
if epoch > 50000:
print(" ⚠️ Timed out. Resampling...")
break
if train_mae >= margin:
torch.save(model.state_dict(), model_path)
attempt += 1
continue
# 3. 검증 단계
print(f"3. Validating {val_iterations} iterations...")
model.eval()
all_passed = True
with torch.no_grad():
for i in range(1, val_iterations + 1):
val_df = df.sample(n=min(int(val_count), len(df)))
X_val = torch.FloatTensor(val_df[X_cols].values).to(device)
y_val_raw = val_df['상수'].values
preds = model(X_val) * 12 + 0.5
y_val_tensor = torch.FloatTensor(y_val_raw).unsqueeze(1).to(device)
diff_tensor = torch.abs(preds - y_val_tensor)
mae = torch.mean(diff_tensor).item()
max_error = torch.max(diff_tensor).item()
# csv 저장
# preds를 CPU 넘파이로 변환하고 1차원으로 펴주는 과정이 포함되어야 합니다.
preds_np = preds.detach().cpu().numpy().flatten() if torch.is_tensor(preds) else preds
res_df = val_df[['songno', 'diff', '상수']].copy()
res_df['predicted_measure'] = preds_np
res_df['error'] = np.abs(preds_np - y_val_raw)
output_file = os.path.join(working_dir, f'validate_result_{i}.csv')
res_df.to_csv(output_file, index=False)
if mae <= margin and max_error <= (margin * MAX_MARGIN_LIMIT):
print(f" [Iter {i}] ✅ PASS (MAE: {mae:.4f}, MAX: {max_error:.4f})")
else:
print(f" [Iter {i}] ❌ FAIL (MAE: {mae:.4f} > {margin} || MAX: {max_error:.4f} > {margin * MAX_MARGIN_LIMIT})")
all_passed = False
break
if all_passed:
print(f"\n✨ Final Success!")
torch.save(model.state_dict(), model_path)
return
else:
torch.save(model.state_dict(), model_path)
attempt += 1
if __name__ == "__main__":
if len(sys.argv) < 8: sys.exit(1)
train(*sys.argv[1:8])

View File

@@ -1,118 +0,0 @@
import { parseTja } from './parse';
import { factorize, Factors } from './factorize';
import { join } from 'path';
import { readFileSync, writeFileSync, readdirSync } from 'fs';
import { parse } from 'csv-parse/sync';
async function train() {
const args = process.argv.slice(2);
const workingDir = args[0];
const scriptDir = args[1];
const dataDir = args[2];
const trainCount = parseInt(args[3]) || 100;
const validateCount = parseInt(args[4]) || 20;
const margin = parseFloat(args[5]) || 0.1;
const factorJsonPath = join(scriptDir, 'factor.json');
// 항상 script/ 위치 참조
let weights: Factors = JSON.parse(readFileSync(factorJsonPath, 'utf-8'));
// 결과물(로그 등)을 저장할 경로 (필요 시 활용)
const logPath = join(workingDir, 'training_log.txt');
const measureCsv = readFileSync(join(dataDir, 'measure.csv'), 'utf-8');
const records = parse(measureCsv, { columns: true, skip_empty_lines: true });
const tjaFiles = readdirSync(join(dataDir, 'tja')).filter(f => f.endsWith('.tja'));
const shuffled = tjaFiles.sort(() => 0.5 - Math.random());
const trainFiles = shuffled.slice(0, trainCount);
const validateFiles = shuffled.slice(trainCount, trainCount + validateCount);
console.log(`Training with ${trainFiles.length} files...`);
// 학습 로직 (Sigmoid + [1, 12] scaling)
const sigmoid = (x: number) => 1 / (1 + Math.exp(-x));
let error = Infinity;
let iterations = 0;
while (error > margin && iterations < 10) { // Spec에 따라 10번 반복
let totalError = 0;
let count = 0;
for (const file of trainFiles) {
const songno = file.replace(/\D/g, '');
const tjaContent = readFileSync(join(dataDir, 'tja', file), 'utf-8');
const parsed = parseTja(tjaContent);
if (!parsed) continue;
for (const diff of ['oni', 'edit'] as const) {
const course = parsed[diff];
if (!course) continue;
const target = records.find((r: any) => r.songno === songno && (r.diff === (diff === 'oni' ? 'oni' : 'ura')));
if (!target) continue;
const factors = factorize(course);
// 가중치 적용 전 정규화 (임의 값으로 가정)
const normalizedFactors = {
physical_density: Math.min(factors.physical_density / 20, 1),
stamina_requirement: Math.min(factors.stamina_requirement, 1),
pattern_complexity: Math.min(factors.pattern_complexity, 1),
rhythmic_complexity: Math.min(factors.rhythmic_complexity, 1),
reading_gimmick: Math.min(factors.reading_gimmick, 1)
};
const rawPrediction = Object.keys(normalizedFactors).reduce((sum, key) =>
sum + (normalizedFactors[key as keyof Factors] * weights[key as keyof Factors]), 0);
const prediction = (sigmoid(rawPrediction) * 11) + 1; // [1, 12]
const targetValue = parseFloat(target.);
const diff_val = targetValue - prediction;
totalError += Math.abs(diff_val);
count++;
// 가중치 업데이트 (간단한 경사 하강법)
for (const key in weights) {
const k = key as keyof Factors;
weights[k] += diff_val * normalizedFactors[k] * 0.01;
}
}
}
error = totalError / (count || 1);
console.log(`Iteration ${iterations}: Mean Error = ${error.toFixed(4)}`);
iterations++;
}
writeFileSync(factorJsonPath, JSON.stringify(weights, null, 2));
writeFileSync(join(workingDir, 'training_result.json'), JSON.stringify({ finalError: error, weights }, null, 2));
console.log(`Training complete. Weights saved to ${factorJsonPath}, result saved to ${workingDir}`);
// 검증 로직
console.log('\nValidation Results:');
for (const file of validateFiles) {
const songno = file.replace(/\D/g, '');
const tjaContent = readFileSync(join(dataDir, 'tja', file), 'utf-8');
const parsed = parseTja(tjaContent);
if (!parsed) continue;
for (const diff of ['oni', 'edit'] as const) {
const course = parsed[diff];
if (!course) continue;
const target = records.find((r: any) => r.songno === songno && (r.diff === (diff === 'oni' ? 'oni' : 'ura')));
if (!target) {
console.log(`[!] No match for ${songno} diff=${diff === 'oni' ? 'oni' : 'ura'}`);
continue;
}
const factors = factorize(course);
const prediction = Object.keys(factors).reduce((sum, key) =>
sum + (factors[key as keyof Factors] * weights[key as keyof Factors]), 0);
console.log(`[${songno}] Target: ${target.}, Predicted: ${prediction.toFixed(2)}, Diff: ${Math.abs(parseFloat(target.) - prediction).toFixed(2)}`);
}
}
}
train();